diff --git a/fsst/src/main/java/io/github/dfa1/vortex/fsst/LossyPerfectHashTable.java b/fsst/src/main/java/io/github/dfa1/vortex/fsst/LossyPerfectHashTable.java new file mode 100644 index 00000000..bcdf104b --- /dev/null +++ b/fsst/src/main/java/io/github/dfa1/vortex/fsst/LossyPerfectHashTable.java @@ -0,0 +1,143 @@ +package io.github.dfa1.vortex.fsst; + +import java.util.List; + +/// Lossy perfect hash table resolving 3-8 byte FSST matches from the first three bytes of an input +/// word (the FSST paper's Algorithm 4, §5.1 "Predicated Scalar Compression"). +/// +/// Each slot holds at most one candidate symbol. A lookup hashes the input word's first three bytes +/// to exactly one slot, reads it (no probing, no chaining), and confirms the match with a single +/// masked 8-byte compare: the input word's high bits beyond the candidate's length are masked off, +/// then compared against the candidate's packed bytes. A slot miss or a failed compare is a "no +/// match" — this is what "lossy" means: a genuine 3-8 byte match can occasionally miss because a +/// higher-gain symbol won its slot on a hash collision. That is harmless, because greedy parsing +/// then falls back to a shorter match ([ShortCodeTable]) or an escape — never to wrong output. +final class LossyPerfectHashTable { + + /// Number of slots. The paper specifies 4096; the well-regarded Rust reference + /// `spiraldb/fsst` uses 2048, citing avoidance of L1D cache-line splits (each slot is small, so + /// 2048 slots keep the whole table within a handful of cache lines that a scan touches + /// repeatedly), and 2048 is this project's benchmark target (`vortex-jni`) — so we deliberately + /// match the reference here rather than the paper's literal 4096. A power of two lets the hash + /// mask instead of taking a modulo, which the hot-loop rule (no per-element modulo/division) + /// requires. + private static final int SLOTS = 2048; + + /// Mask selecting a slot index from a hash; valid because [#SLOTS] is a power of two. + private static final int SLOT_MASK = SLOTS - 1; + + /// Multiply-xor mixing constant, the same shape as the old encoder's `SymbolTable.indexHash` + /// (the golden-ratio 64-bit odd constant `2^64 / phi`). Multiplying by a large odd constant and + /// folding the high bits down spreads the low three input bytes across the whole word so the + /// slot mask sees well-mixed bits, avoiding clustering when many symbols share a low byte. + private static final long HASH_MULTIPLIER = 0x9E3779B97F4A7C15L; + + /// Low three bytes of the input word — the only bytes the hash keys on. + private static final long PREFIX_MASK = 0x00FF_FFFFL; + + /// Per-slot packed symbol bytes, LSB-first ([Symbol] convention). Meaningful only where the + /// corresponding [#occupied] entry is set. + private final long[] packedBytes; + + /// Per-slot mask `~0L >>> ignoredBits` (`ignoredBits = 64 - 8 * length`) applied to an input + /// word before comparing, clearing the high bytes past the candidate's length. + private final long[] keepMask; + + /// Per-slot symbol length in bytes, 3-8. + private final int[] lengths; + + /// Per-slot code (the symbol's list index), meaningful only where [#occupied] is set. + private final int[] codes; + + /// Whether each slot holds a candidate. + private final boolean[] occupied; + + private LossyPerfectHashTable(long[] packedBytes, long[] keepMask, int[] lengths, int[] codes, + boolean[] occupied) { + this.packedBytes = packedBytes; + this.keepMask = keepMask; + this.lengths = lengths; + this.codes = codes; + this.occupied = occupied; + } + + /// Builds the table from the trained symbols in descending-gain order, keeping only those of + /// length 3-8 (shorter symbols are the [ShortCodeTable]'s job and are skipped here). The list + /// index is each symbol's code, matching the parallel-array convention + /// [Decompressor#of(long[], int[])] uses. + /// + /// Insertion is a single forward pass with first-writer-wins on collision. WHY this is + /// load-bearing: the caller passes symbols in descending gain order, so when two symbols' first + /// three bytes hash to the same slot, the one seen first (the higher-gain one) keeps the slot + /// and the later (lower-gain) one is skipped, never overwritten. This is exactly the paper's + /// rule that the more valuable symbol wins a lossy collision. This class must NOT re-sort its + /// input — it consumes whatever order the caller gives and inserts once. + /// + /// @param symbolsByGainDescending the trained symbols, code = list index, gain-descending + /// @return a hash table resolving 3-8 byte matches with first-writer-wins on collision + static LossyPerfectHashTable of(List symbolsByGainDescending) { + long[] packedBytes = new long[SLOTS]; + long[] keepMask = new long[SLOTS]; + int[] lengths = new int[SLOTS]; + int[] codes = new int[SLOTS]; + boolean[] occupied = new boolean[SLOTS]; + for (int code = 0; code < symbolsByGainDescending.size(); code++) { + Symbol symbol = symbolsByGainDescending.get(code); + if (symbol.length() < 3) { + continue; // Length 1-2 belongs to ShortCodeTable. + } + int slot = slotFor(symbol.packedBytes()); + if (occupied[slot]) { + continue; // First writer (higher gain) wins; skip the collision. + } + occupied[slot] = true; + packedBytes[slot] = symbol.packedBytes(); + keepMask[slot] = keepMaskFor(symbol.length()); + lengths[slot] = symbol.length(); + codes[slot] = code; + } + return new LossyPerfectHashTable(packedBytes, keepMask, lengths, codes, occupied); + } + + /// Looks up `word` and reports whether a stored 3-8 byte symbol really matches it. + /// + /// The input word's first three bytes select one slot; the candidate there matches only if the + /// masked compare passes (`(word & keepMask) == candidate.packedBytes`), which rules out both + /// hash collisions with unrelated bytes and empty slots. On a miss the returned [HashMatch] has + /// `hit == false` and the caller falls back to [ShortCodeTable]. + /// + /// @param word an 8-byte little-endian input word starting at the current match position, with + /// any bytes past the remaining input already zero-padded by the caller + /// @return the match result: `hit`, and when hit the matched `code` and `length` + HashMatch lookup(long word) { + int slot = slotFor(word); + boolean hit = occupied[slot] && (word & keepMask[slot]) == packedBytes[slot]; + return new HashMatch(hit, codes[slot], lengths[slot]); + } + + /// Computes the slot index a word hashes to, keyed on its first three bytes. Package-visible so + /// tests can construct deliberate hash collisions against a stable, inspectable hash rather than + /// blindly brute-forcing one — a stable hash is easier to reason about and to test. + /// + /// @param word an input word; only its low three bytes are hashed + /// @return the slot index in `0 .. SLOTS - 1` + static int slotFor(long word) { + long mixed = (word & PREFIX_MASK) * HASH_MULTIPLIER; + return (int) (mixed >>> 32) & SLOT_MASK; + } + + private static long keepMaskFor(int length) { + int ignoredBits = 64 - 8 * length; + return ~0L >>> ignoredBits; + } + + /// Result of a [LossyPerfectHashTable#lookup(long)]: whether a 3-8 byte symbol matched and, if + /// so, its code and length. When `hit` is false, `code` and `length` are unspecified and the + /// caller must ignore them. + /// + /// @param hit whether a stored symbol really matched the input word + /// @param code the matched symbol code, valid only when `hit` + /// @param length the matched symbol length in bytes (3-8), valid only when `hit` + record HashMatch(boolean hit, int code, int length) { + } +} diff --git a/fsst/src/main/java/io/github/dfa1/vortex/fsst/Matcher.java b/fsst/src/main/java/io/github/dfa1/vortex/fsst/Matcher.java new file mode 100644 index 00000000..21885bbd --- /dev/null +++ b/fsst/src/main/java/io/github/dfa1/vortex/fsst/Matcher.java @@ -0,0 +1,87 @@ +package io.github.dfa1.vortex.fsst; + +import java.util.List; + +/// Branch-free longest-match matcher composing a [LossyPerfectHashTable] (3-8 byte candidates) with +/// a [ShortCodeTable] (0/1/2 byte candidates), the FSST paper's Algorithm 4. +/// +/// This replaces the old encoder's `longestMatch`, which probed symbol lengths 8 down to 1 in a +/// per-position loop (up to eight sequential hash lookups per input byte). Here a single hash-table +/// lookup plus one masked compare yields the 3-8 byte candidate, a single array read yields the +/// 0/1/2 byte candidate, and one conditional select picks the longer — no loop over candidate +/// lengths, no per-length branch. That uniform body is exactly what the hot-loop rule (no +/// modulo/division/variable-target branch per element) needs to stay JIT-vectorizable, which the +/// old eight-iteration loop could not deliver. +public final class Matcher { + + /// Longest symbol length that can be resolved, in bytes. + private static final int MAX_SYMBOL_LENGTH = 8; + + private final LossyPerfectHashTable hashTable; + private final ShortCodeTable shortCodes; + + private Matcher(LossyPerfectHashTable hashTable, ShortCodeTable shortCodes) { + this.hashTable = hashTable; + this.shortCodes = shortCodes; + } + + /// Builds a matcher from the trained symbols in descending-gain order. + /// + /// The list index is each symbol's code, matching the parallel-array convention + /// [Decompressor#of(long[], int[])] uses. The whole list is handed to both tables: each keeps + /// only the entries of its own length class ([ShortCodeTable] takes lengths 1-2, + /// [LossyPerfectHashTable] takes lengths 3-8), so a symbol's code stays equal to its index in + /// the original list without any placeholder padding. The caller's gain-descending order is + /// preserved, since it is load-bearing for the hash table's first-writer-wins collision + /// handling. The input is not re-sorted. + /// + /// @param symbolsByGainDescending the trained symbols, code = list index, gain-descending + /// @return a matcher resolving the longest 0-8 byte match at any input position + public static Matcher of(List symbolsByGainDescending) { + return new Matcher( + LossyPerfectHashTable.of(symbolsByGainDescending), + ShortCodeTable.of(symbolsByGainDescending)); + } + + /// Returns the longest match at the current input position packed as `code << 8 | length`. + /// + /// The hash table is consulted first for a 3-8 byte candidate; on a real hit that candidate is + /// returned, otherwise the result falls back to the short-code table's 0/1/2 byte answer. A + /// length of 0 (and code [ShortCodeTable#NO_CODE]) means no symbol matched and the caller must + /// escape the current byte. Callers that want the parts separately can use + /// [#codeOf(int)] and [#lengthOf(int)]. + /// + /// @param word an 8-byte little-endian input word starting at the current match position, with + /// any bytes past the remaining input already zero-padded by the caller + /// @return the longest match as `code << 8 | length`; length 0 signals "no match, escape" + public int longestMatch(long word) { + LossyPerfectHashTable.HashMatch hashMatch = hashTable.lookup(word); + if (hashMatch.hit()) { + return hashMatch.code() << 8 | hashMatch.length(); + } + return shortCodes.codeFor(word) << 8 | shortCodes.lengthFor(word); + } + + /// Extracts the code from a packed [#longestMatch(long)] result. + /// + /// @param packedMatch a `code << 8 | length` value from [#longestMatch(long)] + /// @return the matched symbol code, or [ShortCodeTable#NO_CODE] when the length is 0 + public static int codeOf(int packedMatch) { + return packedMatch >> 8; + } + + /// Extracts the length from a packed [#longestMatch(long)] result. + /// + /// @param packedMatch a `code << 8 | length` value from [#longestMatch(long)] + /// @return the matched symbol length in bytes, 1-8, or 0 when there is no match + public static int lengthOf(int packedMatch) { + return packedMatch & 0xFF; + } + + /// Longest symbol length resolvable by this matcher, in bytes. + /// + /// @return the maximum symbol length, 8 + public static int maxSymbolLength() { + return MAX_SYMBOL_LENGTH; + } +} diff --git a/fsst/src/main/java/io/github/dfa1/vortex/fsst/ShortCodeTable.java b/fsst/src/main/java/io/github/dfa1/vortex/fsst/ShortCodeTable.java new file mode 100644 index 00000000..dff51251 --- /dev/null +++ b/fsst/src/main/java/io/github/dfa1/vortex/fsst/ShortCodeTable.java @@ -0,0 +1,94 @@ +package io.github.dfa1.vortex.fsst; + +import java.util.List; + +/// Direct-indexed table resolving the shortest FSST matches — length 0 (no match), 1, or 2 — +/// from the first two bytes of an input word (the FSST paper's Algorithm 4 `shortCodes`). +/// +/// The key is the input word's low 16 bits read as an unsigned little-endian value, i.e. the first +/// two bytes of the input at the current position (byte 0 in the low 8 bits, byte 1 next). That key +/// indexes one of 65536 slots, each holding a packed `code << 8 | length` giving the matched symbol +/// code and its length in one array read — no hashing, no probing. +/// +/// Only length-1 and length-2 symbols populate the table; longer symbols are the +/// [LossyPerfectHashTable]'s job. Each 16-bit key `hi:lo` is seeded so that, absent a longer match, +/// it resolves to the length-1 symbol for its low byte `lo` (if any). A length-2 symbol then +/// overwrites the specific `hi:lo` slot for its exact two bytes, taking precedence over the +/// length-1 fallback. A key whose low byte has no length-1 symbol and no length-2 symbol resolves +/// to [#NO_CODE] with length 0, telling the caller to escape that single byte. +final class ShortCodeTable { + + /// Number of slots — one per possible 16-bit (two-byte) key. + private static final int SLOTS = 1 << 16; + + /// Sentinel code meaning "no symbol matched"; the caller must escape the current byte. Real + /// codes are `0..254` (`0xFF` is the escape), so `-1` can never collide with a real code. + static final int NO_CODE = -1; + + /// Packed `code << 8 | length` per 16-bit key. A zero length marks "no match". + private final int[] slots; + + private ShortCodeTable(int[] slots) { + this.slots = slots; + } + + /// Builds the table from symbols in descending-gain order, keeping only the length-1 and + /// length-2 entries. The list index is each symbol's code, matching the parallel-array + /// convention [Decompressor#of(long[], int[])] uses (array index is the code). + /// + /// Length-1 symbols are seeded first across all 256 high-byte keys that share their low byte, + /// so any two-byte prefix falls back to its low byte's single-byte code; length-2 symbols then + /// overwrite their exact key, taking precedence. Input order beyond that does not matter here: + /// a length-2 symbol owns a unique key, so there is no gain-order contention within this table + /// (unlike [LossyPerfectHashTable], where collisions make insertion order load-bearing). + /// + /// @param symbolsByGainDescending the trained symbols, code = list index, gain-descending + /// @return a table resolving 0/1/2-byte matches for any two-byte input prefix + static ShortCodeTable of(List symbolsByGainDescending) { + int[] slots = new int[SLOTS]; + for (int code = 0; code < symbolsByGainDescending.size(); code++) { + Symbol symbol = symbolsByGainDescending.get(code); + if (symbol.length() == 1) { + int low = symbol.byteAt(0) & 0xFF; + int packed = code << 8 | 1; + for (int high = 0; high < 256; high++) { + int key = high << 8 | low; + if (length(slots[key]) == 0) { + slots[key] = packed; + } + } + } + } + for (int code = 0; code < symbolsByGainDescending.size(); code++) { + Symbol symbol = symbolsByGainDescending.get(code); + if (symbol.length() == 2) { + int key = (int) (symbol.packedBytes() & 0xFFFF); + slots[key] = code << 8 | 2; + } + } + return new ShortCodeTable(slots); + } + + /// Returns the code matched by the low two bytes of `word`, or [#NO_CODE] if none. One array + /// read, no branching. + /// + /// @param word an input word; only its low 16 bits (first two input bytes) are consulted + /// @return the matched symbol code, or [#NO_CODE] when there is no length-1 or length-2 match + int codeFor(long word) { + int packed = slots[(int) (word & 0xFFFF)]; + return packed == 0 ? NO_CODE : packed >>> 8; + } + + /// Returns the length of the symbol matched by the low two bytes of `word`: 2, 1, or 0 for no + /// match. One array read, no branching. + /// + /// @param word an input word; only its low 16 bits (first two input bytes) are consulted + /// @return the matched symbol length in bytes, or 0 when there is no match + int lengthFor(long word) { + return length(slots[(int) (word & 0xFFFF)]); + } + + private static int length(int packed) { + return packed & 0xFF; + } +} diff --git a/fsst/src/test/java/io/github/dfa1/vortex/fsst/LossyPerfectHashTableTest.java b/fsst/src/test/java/io/github/dfa1/vortex/fsst/LossyPerfectHashTableTest.java new file mode 100644 index 00000000..6cb59c72 --- /dev/null +++ b/fsst/src/test/java/io/github/dfa1/vortex/fsst/LossyPerfectHashTableTest.java @@ -0,0 +1,131 @@ +package io.github.dfa1.vortex.fsst; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class LossyPerfectHashTableTest { + + @Test + void lookup_realThreeByteSymbol_resolvesCode() { + // Given — a single length-3 symbol "abc" at code 0. + Symbol abc = Symbol.of("abc".getBytes(StandardCharsets.UTF_8), 0, 3); + LossyPerfectHashTable sut = LossyPerfectHashTable.of(List.of(abc)); + + // When — the input word starts with "abc" and carries an unrelated trailing byte that the + // masked compare must ignore (the symbol is only 3 bytes). + LossyPerfectHashTable.HashMatch result = sut.lookup(wordOf("abcZZZZZ")); + + // Then + assertThat(result.hit()).isTrue(); + assertThat(result.code()).isEqualTo(0); + assertThat(result.length()).isEqualTo(3); + } + + @Test + void lookup_eightByteSymbol_resolvesFullWord() { + // Given — a length-8 symbol fills the whole word, the upper length boundary. + Symbol full = Symbol.of("ABCDEFGH".getBytes(StandardCharsets.UTF_8), 0, 8); + LossyPerfectHashTable sut = LossyPerfectHashTable.of(List.of(full)); + + // When + LossyPerfectHashTable.HashMatch result = sut.lookup(wordOf("ABCDEFGH")); + + // Then + assertThat(result.hit()).isTrue(); + assertThat(result.length()).isEqualTo(8); + } + + @Test + void lookup_lengthOneAndTwoSymbolsSkipped() { + // Given — the hash table must ignore length 1-2 symbols (ShortCodeTable's job), so a table + // built only from them holds nothing to match. + Symbol a = Symbol.of("a".getBytes(StandardCharsets.UTF_8), 0, 1); + Symbol ab = Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2); + LossyPerfectHashTable sut = LossyPerfectHashTable.of(List.of(a, ab)); + + // When + LossyPerfectHashTable.HashMatch result = sut.lookup(wordOf("ab")); + + // Then + assertThat(result.hit()).isFalse(); + } + + @Test + void lookup_hashCollisionInGainOrder_higherGainWinsSlotLowerGainMisses() { + // Given — two distinct length-3 symbols whose first 3 bytes hash to the SAME slot. Inserted + // in descending-gain order (higher-gain first, per the load-bearing invariant), the first + // one must keep the slot and the second must be rejected — never overwrite it. + Symbol[] colliding = findCollidingPair(); + Symbol higherGain = colliding[0]; + Symbol lowerGain = colliding[1]; + LossyPerfectHashTable sut = LossyPerfectHashTable.of(List.of(higherGain, lowerGain)); + + // When — look up each symbol's own bytes. + LossyPerfectHashTable.HashMatch higher = sut.lookup(higherGain.packedBytes()); + LossyPerfectHashTable.HashMatch lower = sut.lookup(lowerGain.packedBytes()); + + // Then — the higher-gain (first-inserted) symbol wins; the lower-gain one correctly misses + // and does not corrupt or masquerade as the winner. + assertThat(higher.hit()).isTrue(); + assertThat(higher.code()).isEqualTo(0); + assertThat(lower.hit()).isFalse(); + } + + @Test + void lookup_hashCollisionButByteMismatch_reportsNoMatch() { + // Given — one stored length-3 symbol, and an input word that hashes to the same slot but + // does NOT byte-match it. Only the masked compare (not the hash alone) may gate a hit, so + // this adversarial word must report no match — proving there are no false positives. + Symbol stored = Symbol.of("abc".getBytes(StandardCharsets.UTF_8), 0, 3); + LossyPerfectHashTable sut = LossyPerfectHashTable.of(List.of(stored)); + long adversarial = findWordCollidingWithButNotEqualTo(stored.packedBytes()); + + // When + LossyPerfectHashTable.HashMatch result = sut.lookup(adversarial); + + // Then + assertThat(result.hit()).isFalse(); + } + + /// Finds two distinct 3-byte symbols whose first three bytes hash to the same slot, so the test + /// exercises a real collision against the actual (stable, package-visible) hash function rather + /// than a hand-picked guess that could drift if the hash changes. + private static Symbol[] findCollidingPair() { + int base = 0x414141; // "AAA", three printable bytes; iterate the third byte upward. + int firstSlot = LossyPerfectHashTable.slotFor(base); + for (int candidate = base + 1; candidate <= 0xFFFFFF; candidate++) { + if (LossyPerfectHashTable.slotFor(candidate) == firstSlot && candidate != base) { + return new Symbol[]{new Symbol(base, 3), new Symbol(candidate, 3)}; + } + } + throw new AssertionError("no colliding 3-byte prefix found — hash function changed?"); + } + + /// Finds a 3-byte word that hashes to the same slot as `stored` but differs from it, so the + /// masked compare (not the hash) is what must reject it. + private static long findWordCollidingWithButNotEqualTo(long stored) { + int storedPrefix = (int) (stored & 0x00FF_FFFFL); + int storedSlot = LossyPerfectHashTable.slotFor(storedPrefix); + for (int candidate = 0; candidate <= 0xFFFFFF; candidate++) { + if (candidate != storedPrefix && LossyPerfectHashTable.slotFor(candidate) == storedSlot) { + return candidate; + } + } + throw new AssertionError("no colliding-but-different word found — hash function changed?"); + } + + /// Packs the low bytes of `s` LSB-first into a word, matching the reader's little-endian + /// interpretation of the input at a match position. + private static long wordOf(String s) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + long word = 0; + for (int k = 0; k < bytes.length && k < 8; k++) { + word |= Byte.toUnsignedLong(bytes[k]) << (k * 8); + } + return word; + } +} diff --git a/fsst/src/test/java/io/github/dfa1/vortex/fsst/MatcherTest.java b/fsst/src/test/java/io/github/dfa1/vortex/fsst/MatcherTest.java new file mode 100644 index 00000000..bdee9fe5 --- /dev/null +++ b/fsst/src/test/java/io/github/dfa1/vortex/fsst/MatcherTest.java @@ -0,0 +1,98 @@ +package io.github.dfa1.vortex.fsst; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class MatcherTest { + + // A mixed-length table exercising every tier: code 0 = "ABCDEFGH" (8), code 1 = "abc" (3), + // code 2 = "ab" (2), code 3 = "a" (1). Gain-descending order is assumed by construction; the + // codes are the list indices. + private static final List SYMBOLS = List.of( + Symbol.of("ABCDEFGH".getBytes(StandardCharsets.UTF_8), 0, 8), + Symbol.of("abc".getBytes(StandardCharsets.UTF_8), 0, 3), + Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2), + Symbol.of("a".getBytes(StandardCharsets.UTF_8), 0, 1)); + + @Test + void longestMatch_eightByteSymbol_picksHashTableMatch() { + // Given + Matcher sut = Matcher.of(SYMBOLS); + + // When + int result = sut.longestMatch(wordOf("ABCDEFGH")); + + // Then — the longest (8-byte) symbol wins. + assertThat(Matcher.codeOf(result)).isEqualTo(0); + assertThat(Matcher.lengthOf(result)).isEqualTo(8); + } + + @Test + void longestMatch_threeByteSymbol_picksHashTableOverShorterShortCodes() { + // Given — input "abc..." matches "abc" (3), "ab" (2), and "a" (1); the 3-byte hash-table + // match must win over the 1/2-byte short-code candidates. + Matcher sut = Matcher.of(SYMBOLS); + + // When + int result = sut.longestMatch(wordOf("abcxxxxx")); + + // Then + assertThat(Matcher.codeOf(result)).isEqualTo(1); + assertThat(Matcher.lengthOf(result)).isEqualTo(3); + } + + @Test + void longestMatch_hashMissFallsBackToTwoByteShortCode() { + // Given — input "abz..." misses the 3-byte "abc" but matches "ab" (2); the matcher must + // fall back through the tiers to the short-code table's 2-byte answer. + Matcher sut = Matcher.of(SYMBOLS); + + // When + int result = sut.longestMatch(wordOf("abzxxxxx")); + + // Then + assertThat(Matcher.codeOf(result)).isEqualTo(2); + assertThat(Matcher.lengthOf(result)).isEqualTo(2); + } + + @Test + void longestMatch_hashMissAndTwoByteMissFallsBackToSingleByte() { + // Given — input "axz..." misses "abc" and "ab" but matches the length-1 "a". + Matcher sut = Matcher.of(SYMBOLS); + + // When + int result = sut.longestMatch(wordOf("axzxxxxx")); + + // Then + assertThat(Matcher.codeOf(result)).isEqualTo(3); + assertThat(Matcher.lengthOf(result)).isEqualTo(1); + } + + @Test + void longestMatch_unmatchedByte_signalsNoMatch() { + // Given — the byte 'z' matches nothing at any tier; the caller must escape it. + Matcher sut = Matcher.of(SYMBOLS); + + // When + int result = sut.longestMatch(wordOf("zzzzzzzz")); + + // Then + assertThat(Matcher.lengthOf(result)).isEqualTo(0); + assertThat(Matcher.codeOf(result)).isEqualTo(ShortCodeTable.NO_CODE); + } + + /// Packs the low bytes of `s` LSB-first into an 8-byte word, matching the reader's + /// little-endian interpretation of the input at a match position. + private static long wordOf(String s) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + long word = 0; + for (int k = 0; k < bytes.length && k < 8; k++) { + word |= Byte.toUnsignedLong(bytes[k]) << (k * 8); + } + return word; + } +} diff --git a/fsst/src/test/java/io/github/dfa1/vortex/fsst/ShortCodeTableTest.java b/fsst/src/test/java/io/github/dfa1/vortex/fsst/ShortCodeTableTest.java new file mode 100644 index 00000000..0fdffec9 --- /dev/null +++ b/fsst/src/test/java/io/github/dfa1/vortex/fsst/ShortCodeTableTest.java @@ -0,0 +1,104 @@ +package io.github.dfa1.vortex.fsst; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ShortCodeTableTest { + + @Test + void codeFor_singleByteSymbol_resolvesLengthOne() { + // Given — one length-1 symbol 'a' at code 0. + Symbol a = Symbol.of("a".getBytes(StandardCharsets.UTF_8), 0, 1); + ShortCodeTable sut = ShortCodeTable.of(List.of(a)); + long word = wordOf("ax"); // second byte irrelevant: no length-2 symbol. + + // When + int code = sut.codeFor(word); + int length = sut.lengthFor(word); + + // Then + assertThat(code).isEqualTo(0); + assertThat(length).isEqualTo(1); + } + + @Test + void codeFor_twoByteSymbol_resolvesLengthTwoAndBeatsSingleByteFallback() { + // Given — code 0 = 'a' (length 1), code 1 = "ab" (length 2). At input "ab" the length-2 + // symbol must win over the length-1 fallback for the low byte 'a'. + Symbol a = Symbol.of("a".getBytes(StandardCharsets.UTF_8), 0, 1); + Symbol ab = Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2); + ShortCodeTable sut = ShortCodeTable.of(List.of(a, ab)); + long word = wordOf("ab"); + + // When + int code = sut.codeFor(word); + int length = sut.lengthFor(word); + + // Then + assertThat(code).isEqualTo(1); + assertThat(length).isEqualTo(2); + } + + @Test + void codeFor_singleByteFallbackWhenTwoByteDiffers() { + // Given — "ab" is a length-2 symbol, but the input is "ax"; only the length-1 'a' fallback + // applies, so the low byte must still resolve to the single-byte code. + Symbol a = Symbol.of("a".getBytes(StandardCharsets.UTF_8), 0, 1); + Symbol ab = Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2); + ShortCodeTable sut = ShortCodeTable.of(List.of(a, ab)); + long word = wordOf("ax"); + + // When + int code = sut.codeFor(word); + int length = sut.lengthFor(word); + + // Then + assertThat(code).isEqualTo(0); + assertThat(length).isEqualTo(1); + } + + @Test + void codeFor_byteWithNoSymbol_resolvesNoMatch() { + // Given — only 'a' is in the table; the byte 'z' has no length-1 or length-2 symbol. + Symbol a = Symbol.of("a".getBytes(StandardCharsets.UTF_8), 0, 1); + ShortCodeTable sut = ShortCodeTable.of(List.of(a)); + long word = wordOf("zz"); + + // When + int code = sut.codeFor(word); + int length = sut.lengthFor(word); + + // Then — the caller must escape this byte. + assertThat(code).isEqualTo(ShortCodeTable.NO_CODE); + assertThat(length).isEqualTo(0); + } + + @Test + void codeFor_longSymbolsIgnored() { + // Given — a length-3 symbol must not populate this table (it is the hash table's job). + Symbol abc = Symbol.of("abc".getBytes(StandardCharsets.UTF_8), 0, 3); + ShortCodeTable sut = ShortCodeTable.of(List.of(abc)); + long word = wordOf("abc"); + + // When + int length = sut.lengthFor(word); + + // Then + assertThat(length).isEqualTo(0); + } + + /// Packs the low bytes of `s` LSB-first into a word, matching the reader's little-endian + /// interpretation of the input at a match position. + private static long wordOf(String s) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + long word = 0; + for (int k = 0; k < bytes.length && k < 8; k++) { + word |= Byte.toUnsignedLong(bytes[k]) << (k * 8); + } + return word; + } +}