From 0ba1a50de1612131e741c52fdd62bd32b220519a Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 20 Jul 2026 12:23:40 +0200 Subject: [PATCH] test(fsst): golden test for the paper's Figure 2 worked example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add PaperFigure2Test reproducing the FSST paper's Figure 2 worked example (Boncz/Neumann/Leis, PVLDB 13(11) 2020, p. 2653): corpus "tumcwitumvldb", maximum symbol length 3, maximum table size 5. After the paper's fourth iteration the symbol table is {tum, cwi, vld, b} and the figure's "Compressed" row shows tum|cwi|tum|vld|b, len = 5. The test hand-builds that exact table via Compressor.of(List) rather than CompressorBuilder.train(...) — training would re-derive a table through this codebase's own heuristics, which is circular. A golden test needs an external, human-verifiable fixed point, so the symbols and codes come straight off the figure (gain order tum=0, cwi=1, vld=2, b=3). It asserts the greedy compressor reproduces both the code sequence [tum,cwi,tum,vld,b] and length 5 (paper fidelity), and that the result round-trips to the original 13 bytes (primary correctness gate). The javadoc cites Figure 2 so a maintainer can re-verify against the primary source; the figure was re-fetched and confirmed from the authors' cwida/fsst reference repo (fsstcompression.pdf, p. 2653). Also tighten FsstEncodingEncoderTest .encode_repeatedText_learnsMultiByteSymbols_compressesBelowRaw from maxSymbolLength > 2 to > 4. On that test's input ("the quick brown fox jumps over the lazy dog" x100) the paper-faithful trainer deterministically learns ten length-8 symbols and one length-3 symbol (max 8) with the fixed 0x5EED encoder seed, so a > 4 floor is a stronger regression guard with ample headroom against a training change that quietly stopped growing symbols past a trigram. Co-Authored-By: Claude Sonnet 5 --- .../dfa1/vortex/fsst/PaperFigure2Test.java | 83 +++++++++++++++++++ .../encode/FsstEncodingEncoderTest.java | 8 +- 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 fsst/src/test/java/io/github/dfa1/vortex/fsst/PaperFigure2Test.java diff --git a/fsst/src/test/java/io/github/dfa1/vortex/fsst/PaperFigure2Test.java b/fsst/src/test/java/io/github/dfa1/vortex/fsst/PaperFigure2Test.java new file mode 100644 index 00000000..ab90789c --- /dev/null +++ b/fsst/src/test/java/io/github/dfa1/vortex/fsst/PaperFigure2Test.java @@ -0,0 +1,83 @@ +package io.github.dfa1.vortex.fsst; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/// Golden test reproducing the worked example in Figure 2 of the FSST paper (Boncz, Neumann, Leis, +/// "FSST: Fast Random Access String Compression", PVLDB 13(11), 2020, p. 2653, Figure 2: "Four +/// iterations of symbol table construction algorithm on the corpus 'tumcwitumvldb' with a maximum +/// symbol length of 3 and a maximum symbol table size of 5"). +/// +/// After the paper's fourth iteration the symbol table is `{tum, cwi, vld, b}` and the figure's +/// "Compressed" row at the bottom shows `tum | cwi | tum | vld | b`, len = 5 (down from the 13-byte +/// uncompressed corpus). This test hand-builds exactly that symbol table and asserts the greedy +/// longest-match compressor reproduces the figure's compressed code sequence and length, then that +/// the result round-trips back to the original 13 bytes. +/// +/// The symbol table is hand-constructed on purpose. Using [CompressorBuilder#train(byte[][])] would +/// re-derive a table through this codebase's own training heuristics — a circular check. A golden +/// test needs an external, human-verifiable fixed point, so the symbols and their codes come +/// straight off the paper's figure, not off our trainer. +/// +/// Symbol codes equal the symbol's index in the gain-descending list [Compressor#of(List)] receives. +/// Figure 2's iteration-4 table lists the symbols in gain order `tum` (gain 6), `cwi` (gain 3), +/// `vld` (gain 3), `b` (gain 1), so the codes assigned here are `tum`=0, `cwi`=1, `vld`=2, `b`=3. +class PaperFigure2Test { + + /// The paper's Figure 2 corpus, exactly as printed. ASCII, so US-ASCII and UTF-8 are equivalent. + private static final byte[] CORPUS = "tumcwitumvldb".getBytes(StandardCharsets.US_ASCII); + + /// Codes for the figure's compressed row `tum | cwi | tum | vld | b`, using the code assignment + /// documented on the class: tum=0, cwi=1, vld=2, b=3. + private static final byte[] EXPECTED_CODES = {0, 1, 0, 2, 3}; + + @Test + void compress_paperFigure2Corpus_producesFigure2CodeSequenceAndLength() { + // Given — the exact iteration-4 symbol table from Figure 2, in gain-descending order so + // each symbol's code matches its column in the figure (tum=0, cwi=1, vld=2, b=3). + Compressor sut = Compressor.of(List.of( + symbol("tum"), + symbol("cwi"), + symbol("vld"), + symbol("b"))); + byte[] out = new byte[CORPUS.length * 2]; + + // When — greedy longest-match compression of the 13-byte corpus. + long compressedLength = sut.compress(CORPUS, 0, CORPUS.length, out, 0); + + // Then — the figure's "Compressed ... len = 5" row: five codes [tum, cwi, tum, vld, b], no + // escapes, exactly the sequence the paper draws. + assertThat(compressedLength).isEqualTo(5L); + assertThat(java.util.Arrays.copyOf(out, (int) compressedLength)).isEqualTo(EXPECTED_CODES); + } + + @Test + void compress_thenDecompress_paperFigure2Corpus_roundTripsToOriginal() { + // Given — same hand-built Figure 2 table. + Compressor sut = Compressor.of(List.of( + symbol("tum"), + symbol("cwi"), + symbol("vld"), + symbol("b"))); + byte[] compressed = new byte[CORPUS.length * 2]; + + // When — compress, then decompress through the same table. + long compressedLength = sut.compress(CORPUS, 0, CORPUS.length, compressed, 0); + byte[] decompressed = new byte[CORPUS.length]; + long decompressedLength = sut.toDecompressor() + .decompress(compressed, 0, (int) compressedLength, decompressed, 0); + + // Then — the primary correctness gate: the exact original 13 bytes come back. + assertThat(decompressedLength).isEqualTo(CORPUS.length); + assertThat(decompressed).isEqualTo(CORPUS); + } + + private static Symbol symbol(String text) { + byte[] bytes = text.getBytes(StandardCharsets.US_ASCII); + return Symbol.of(bytes, 0, bytes.length); + } +} diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java index 55edd5a2..c57b0785 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java @@ -196,9 +196,13 @@ void encode_repeatedText_learnsMultiByteSymbols_compressesBelowRaw() { byte maxSymLen = maxSymbolLength(result); // Then — real compression (< half raw) proves multi-byte symbols are in play, and the - // table contains symbols longer than the old fixed 2-byte limit. + // table learns symbols well past a bigram. The paper-faithful trainer grows symbols to + // the 8-byte cap on this highly repetitive input (ten of eleven learned symbols are + // length 8, one is length 3, with the fixed 0x5EED encoder seed), so a `> 4` floor is a + // stronger regression guard than the old `> 2` with ample headroom — a training change + // that quietly stopped growing symbols past a trigram would now be caught. assertThat(compressedBytes).isLessThan(rawBytes / 2); - assertThat(maxSymLen).isGreaterThan((byte) 2); + assertThat(maxSymLen).isGreaterThan((byte) 4); } @Test