From 8933ecd5771532836a1261c8ca6341b509bfa493 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 19 Jul 2026 20:21:21 +0200 Subject: [PATCH] feat(fsst): module skeleton, Symbol, Decompressor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a standalone `vortex-fsst` module as the first step of #287 ("improve FSST implementation"). The current inline FSST encoder/decoder is a simplified first pass; comparing against vortex-jni surfaced real gaps in matching, training, and decode. Following the paper (Boncz/Neumann/Leis, VLDB 2020) and the Rust reference's split of a pure algorithm crate (spiraldb/fsst) from the thin wire adapter, and since research found no zero-dependency Java FSST library to reuse, the algorithm is isolated into its own module so it can be iterated on and benchmarked without the Vortex wire-format shell in the way. This PR is purely additive — no existing code is touched. It ports the lowest-risk piece first: the `Symbol` record (LSB-first byte packing, the same convention as the writer's existing SymbolCandidate) and a byte[]-based `Decompressor` implementing the paper's Algorithm 1 (escape-or-symbol decode loop). The MemorySegment-native hot path, branch-free matching, training, and the adapter rewire that makes writer/reader depend on this module all land in later PRs of the sequence. Unlike the build-tooling generators, this module ships to Maven Central (writer/reader will depend on it at runtime), so its POM omits `maven.deploy.skip`. Co-Authored-By: Claude Sonnet 5 --- fsst/pom.xml | 31 +++++ .../github/dfa1/vortex/fsst/Decompressor.java | 67 ++++++++++ .../io/github/dfa1/vortex/fsst/Symbol.java | 47 +++++++ .../dfa1/vortex/fsst/DecompressorTest.java | 119 ++++++++++++++++++ .../github/dfa1/vortex/fsst/SymbolTest.java | 74 +++++++++++ pom.xml | 6 + 6 files changed, 344 insertions(+) create mode 100644 fsst/pom.xml create mode 100644 fsst/src/main/java/io/github/dfa1/vortex/fsst/Decompressor.java create mode 100644 fsst/src/main/java/io/github/dfa1/vortex/fsst/Symbol.java create mode 100644 fsst/src/test/java/io/github/dfa1/vortex/fsst/DecompressorTest.java create mode 100644 fsst/src/test/java/io/github/dfa1/vortex/fsst/SymbolTest.java diff --git a/fsst/pom.xml b/fsst/pom.xml new file mode 100644 index 00000000..20a2a7cb --- /dev/null +++ b/fsst/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.12.4-SNAPSHOT + + + vortex-fsst + + vortex-fsst + Standalone FSST (Fast Static Symbol Table) string compression algorithm. Zero dependency on the Vortex + wire format or FFM memory model beyond the JDK's own java.lang.foreign — so the compression algorithm can be + iterated on and benchmarked independently. + + + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + diff --git a/fsst/src/main/java/io/github/dfa1/vortex/fsst/Decompressor.java b/fsst/src/main/java/io/github/dfa1/vortex/fsst/Decompressor.java new file mode 100644 index 00000000..6ca758e5 --- /dev/null +++ b/fsst/src/main/java/io/github/dfa1/vortex/fsst/Decompressor.java @@ -0,0 +1,67 @@ +package io.github.dfa1.vortex.fsst; + +/// Decodes an FSST-compressed byte stream against a trained symbol table (the FSST paper's +/// Algorithm 1). +/// +/// The table is held as parallel arrays indexed by code: `packedSymbols` gives each code's bytes +/// packed LSB-first into a `long` (the [Symbol] convention) and `lengths` gives its byte length. +/// Decoding walks the compressed stream one code at a time: the escape code `0xFF` copies the +/// next literal byte verbatim, any other code appends its symbol's bytes to the output. +public final class Decompressor { + + /// Escape code: emitted as `0xFF` followed by one literal byte, which is copied to the + /// output unchanged. Real symbol codes are therefore `0..254`. + public static final int ESCAPE = 0xFF; + + private final long[] packedSymbols; + private final int[] lengths; + + private Decompressor(long[] packedSymbols, int[] lengths) { + this.packedSymbols = packedSymbols; + this.lengths = lengths; + } + + /// Creates a decompressor over a trained symbol table. + /// + /// The two arrays are indexed by code and must be the same length; the arrays are referenced, + /// not copied, so the caller must not mutate them afterwards. + /// + /// @param packedSymbols each code's bytes packed LSB-first into a `long`, indexed by code + /// @param lengths each code's symbol length in bytes, indexed by code + /// @return a decompressor bound to the given table + public static Decompressor of(long[] packedSymbols, int[] lengths) { + return new Decompressor(packedSymbols, lengths); + } + + /// Decodes the compressed byte range `[start, end)` of `compressed` into `out`, writing the + /// decoded bytes starting at `outPos`. + /// + /// The caller is responsible for sizing `out` so every decoded byte fits: an `end - start` + /// code stream expands to at most `8 * (end - start)` output bytes (a full run of length-8 + /// symbols). This is an internal algorithm boundary, not an untrusted-input boundary, so no + /// defensive bounds checks are performed here. + /// + /// @param compressed the compressed code stream + /// @param start the index of the first code to decode, inclusive + /// @param end the index one past the last code to decode, exclusive + /// @param out the destination buffer for decoded bytes + /// @param outPos the index in `out` at which to start writing + /// @return the index in `out` one past the last byte written + public long decompress(byte[] compressed, int start, int end, byte[] out, long outPos) { + int pos = start; + int outIndex = (int) outPos; + while (pos < end) { + int code = compressed[pos++] & 0xFF; + if (code == ESCAPE) { + out[outIndex++] = compressed[pos++]; + } else { + long packed = packedSymbols[code]; + int length = lengths[code]; + for (int k = 0; k < length; k++) { + out[outIndex++] = (byte) (packed >>> (k * 8)); + } + } + } + return outIndex; + } +} diff --git a/fsst/src/main/java/io/github/dfa1/vortex/fsst/Symbol.java b/fsst/src/main/java/io/github/dfa1/vortex/fsst/Symbol.java new file mode 100644 index 00000000..f831433e --- /dev/null +++ b/fsst/src/main/java/io/github/dfa1/vortex/fsst/Symbol.java @@ -0,0 +1,47 @@ +package io.github.dfa1.vortex.fsst; + +/// A single FSST symbol: up to 8 bytes packed LSB-first into a `long` (byte 0 in the low byte, +/// byte `k` at bit position `k * 8`) paired with an explicit length in `1..8`. +/// +/// The LSB-first packing convention is shared everywhere a symbol crosses a boundary — the +/// training tables, the compressor, the decompressor, and eventually the `vortex.fsst` wire +/// format — so a symbol packed here reads back byte-for-byte identically wherever it lands. +/// +/// @param packedBytes the symbol's bytes, LSB-first (byte `k` at bit position `k * 8`) +/// @param length the symbol length in bytes, in `1..8` +public record Symbol(long packedBytes, int length) { + + /// Maximum symbol length in bytes, bounded by what fits in one `long`. + private static final int MAX_LENGTH = 8; + + /// Validates that `length` is a legal FSST symbol length. + /// + /// @throws IllegalArgumentException if `length` is not in `1..8` + public Symbol { + if (length < 1 || length > MAX_LENGTH) { + throw new IllegalArgumentException("symbol length must be in 1..8, was " + length); + } + } + + /// Packs `length` bytes of `data` starting at `offset` LSB-first into a new symbol. + /// + /// @param data the source bytes + /// @param offset the index of the symbol's first byte in `data` + /// @param length the symbol length in bytes, in `1..8` + /// @return a symbol holding the `length` bytes at `data[offset .. offset + length)` + public static Symbol of(byte[] data, int offset, int length) { + long value = 0; + for (int k = 0; k < length; k++) { + value |= Byte.toUnsignedLong(data[offset + k]) << (k * 8); + } + return new Symbol(value, length); + } + + /// Returns the `i`-th byte of this symbol (0-indexed, so byte 0 is the LSB). + /// + /// @param i the byte index, in `0 .. length - 1` + /// @return the `i`-th byte of the symbol + public byte byteAt(int i) { + return (byte) (packedBytes >>> (i * 8)); + } +} diff --git a/fsst/src/test/java/io/github/dfa1/vortex/fsst/DecompressorTest.java b/fsst/src/test/java/io/github/dfa1/vortex/fsst/DecompressorTest.java new file mode 100644 index 00000000..8914e0e7 --- /dev/null +++ b/fsst/src/test/java/io/github/dfa1/vortex/fsst/DecompressorTest.java @@ -0,0 +1,119 @@ +package io.github.dfa1.vortex.fsst; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +class DecompressorTest { + + @Nested + class SingleCode { + + @Test + void decompress_singleByteSymbol_appendsOneByte() { + // Given — code 0 maps to the single byte 'A'. + Decompressor sut = Decompressor.of(new long[]{'A'}, new int[]{1}); + byte[] out = new byte[1]; + + // When + long result = sut.decompress(new byte[]{0x00}, 0, 1, out, 0); + + // Then + assertThat(result).isEqualTo(1); + assertThat(out).isEqualTo("A".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void decompress_multiByteSymbol_appendsAllBytesLsbFirst() { + // Given — code 0 maps to the 4-byte symbol "abcd", packed LSB-first via Symbol.of. + Symbol abcd = Symbol.of("abcd".getBytes(StandardCharsets.UTF_8), 0, 4); + Decompressor sut = Decompressor.of(new long[]{abcd.packedBytes()}, new int[]{4}); + byte[] out = new byte[4]; + + // When + long result = sut.decompress(new byte[]{0x00}, 0, 1, out, 0); + + // Then + assertThat(result).isEqualTo(4); + assertThat(out).isEqualTo("abcd".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void decompress_escapeCode_copiesLiteralByte() { + // Given — no symbols; the escape code copies the following literal byte verbatim. + Decompressor sut = Decompressor.of(new long[0], new int[0]); + byte[] out = new byte[1]; + + // When + long result = sut.decompress(new byte[]{(byte) Decompressor.ESCAPE, 'Z'}, 0, 2, out, 0); + + // Then + assertThat(result).isEqualTo(1); + assertThat(out).isEqualTo("Z".getBytes(StandardCharsets.UTF_8)); + } + } + + @Nested + class MultipleCodes { + + @Test + void decompress_symbolsAndEscape_concatenateInOrder() { + // Given — code 0 = "ab" (2 bytes), code 1 = "c" (1 byte), then an escaped 'd'. + Symbol ab = Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2); + Decompressor sut = Decompressor.of(new long[]{ab.packedBytes(), 'c'}, new int[]{2, 1}); + byte[] compressed = {0x00, 0x01, (byte) Decompressor.ESCAPE, 'd'}; + byte[] out = new byte[4]; + + // When + long result = sut.decompress(compressed, 0, compressed.length, out, 0); + + // Then + assertThat(result).isEqualTo(4); + assertThat(out).isEqualTo("abcd".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void decompress_startEndSubrange_decodesOnlyTheGivenWindow() { + // Given — a stream with a leading and trailing code that must be ignored when only the + // middle window [1, 2) is requested. + Decompressor sut = Decompressor.of(new long[]{'X', 'Y', 'Z'}, new int[]{1, 1, 1}); + byte[] compressed = {0x00, 0x01, 0x02}; + byte[] out = new byte[1]; + + // When + long result = sut.decompress(compressed, 1, 2, out, 0); + + // Then — only code 1 ('Y') is decoded. + assertThat(result).isEqualTo(1); + assertThat(out).isEqualTo("Y".getBytes(StandardCharsets.UTF_8)); + } + } + + @Nested + class MultipleRows { + + @Test + void decompress_sequentialRows_doNotCorruptEachOther() { + // Given — two independent rows written into the same output buffer at sequential + // positions. Row 0 = "ab" (symbol), row 1 = escaped 'c'. A wrong outPos advance would + // let row 1 overwrite row 0's tail, so decoding both and checking each byte catches it. + Symbol ab = Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2); + Decompressor sut = Decompressor.of(new long[]{ab.packedBytes()}, new int[]{2}); + byte[] out = new byte[3]; + + // When + long afterRow0 = sut.decompress(new byte[]{0x00}, 0, 1, out, 0); + long afterRow1 = sut.decompress(new byte[]{(byte) Decompressor.ESCAPE, 'c'}, 0, 2, out, afterRow0); + + // Then + assertThat(afterRow0).isEqualTo(2); + assertThat(afterRow1).isEqualTo(3); + assertThat(Arrays.copyOfRange(out, 0, 2)).isEqualTo("ab".getBytes(StandardCharsets.UTF_8)); + assertThat(Arrays.copyOfRange(out, 2, 3)).isEqualTo("c".getBytes(StandardCharsets.UTF_8)); + } + } +} diff --git a/fsst/src/test/java/io/github/dfa1/vortex/fsst/SymbolTest.java b/fsst/src/test/java/io/github/dfa1/vortex/fsst/SymbolTest.java new file mode 100644 index 00000000..210903d7 --- /dev/null +++ b/fsst/src/test/java/io/github/dfa1/vortex/fsst/SymbolTest.java @@ -0,0 +1,74 @@ +package io.github.dfa1.vortex.fsst; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class SymbolTest { + + @Test + void of_packsBytesLsbFirst() { + // Given — "ab" should pack to 0x6261 (byte 'a'=0x61 in the low byte, 'b'=0x62 next). + byte[] data = "ab".getBytes(StandardCharsets.UTF_8); + + // When + Symbol result = Symbol.of(data, 0, 2); + + // Then + assertThat(result.packedBytes()).isEqualTo(0x6261L); + assertThat(result.length()).isEqualTo(2); + } + + @Test + void of_respectsOffset() { + // Given — packing 2 bytes starting at offset 2 must read "cd", not "ab". + byte[] data = "abcd".getBytes(StandardCharsets.UTF_8); + + // When + Symbol result = Symbol.of(data, 2, 2); + + // Then + assertThat(result.byteAt(0)).isEqualTo((byte) 'c'); + assertThat(result.byteAt(1)).isEqualTo((byte) 'd'); + } + + @Test + void byteAt_returnsEachByteInOrder() { + // Given + Symbol sut = Symbol.of("WXYZ".getBytes(StandardCharsets.UTF_8), 0, 4); + + // When + byte[] result = {sut.byteAt(0), sut.byteAt(1), sut.byteAt(2), sut.byteAt(3)}; + + // Then + assertThat(result).isEqualTo("WXYZ".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void of_eightByteSymbol_packsFullLong() { + // Given — a full 8-byte symbol fills every byte of the long, the length boundary. + byte[] data = "ABCDEFGH".getBytes(StandardCharsets.UTF_8); + + // When + Symbol result = Symbol.of(data, 0, 8); + + // Then + assertThat(result.length()).isEqualTo(8); + assertThat(result.byteAt(7)).isEqualTo((byte) 'H'); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1, 9}) + void constructor_lengthOutOfRange_throws(int length) { + // Given + // When + // Then + assertThatThrownBy(() -> new Symbol(0L, length)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/pom.xml b/pom.xml index dd492199..405c8ce3 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ proto-gen fbs-gen + fsst core reader writer @@ -160,6 +161,11 @@ vortex-writer ${project.version} + + io.github.dfa1.vortex + vortex-fsst + ${project.version} + io.github.dfa1.vortex vortex-csv