feat(fsst): module skeleton, Symbol, Decompressor

fsst/pom.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>io.github.dfa1.vortex</groupId>
+		<artifactId>vortex-java</artifactId>
+		<version>0.12.4-SNAPSHOT</version>
+	</parent>
+
+	<artifactId>vortex-fsst</artifactId>
+
+	<name>vortex-fsst</name>
+	<description>Standalone FSST (Fast Static Symbol Table) string compression algorithm. Zero dependency on the Vortex
+		wire format or FFM memory model beyond the JDK's own java.lang.foreign — so the compression algorithm can be
+		iterated on and benchmarked independently.
+	</description>
+
+	<dependencies>
+		<!-- testing -->
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.assertj</groupId>
+			<artifactId>assertj-core</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+</project>

fsst/src/main/java/io/github/dfa1/vortex/fsst/Decompressor.java

@@ -0,0 +1,67 @@
+package io.github.dfa1.vortex.fsst;
+
+/// Decodes an FSST-compressed byte stream against a trained symbol table (the FSST paper's
+/// Algorithm 1).
+///
+/// The table is held as parallel arrays indexed by code: `packedSymbols` gives each code's bytes
+/// packed LSB-first into a `long` (the [Symbol] convention) and `lengths` gives its byte length.
+/// Decoding walks the compressed stream one code at a time: the escape code `0xFF` copies the
+/// next literal byte verbatim, any other code appends its symbol's bytes to the output.
+public final class Decompressor {
+
+    /// Escape code: emitted as `0xFF` followed by one literal byte, which is copied to the
+    /// output unchanged. Real symbol codes are therefore `0..254`.
+    public static final int ESCAPE = 0xFF;
+
+    private final long[] packedSymbols;
+    private final int[] lengths;
+
+    private Decompressor(long[] packedSymbols, int[] lengths) {
+        this.packedSymbols = packedSymbols;
+        this.lengths = lengths;
+    }
+
+    /// Creates a decompressor over a trained symbol table.
+    ///
+    /// The two arrays are indexed by code and must be the same length; the arrays are referenced,
+    /// not copied, so the caller must not mutate them afterwards.
+    ///
+    /// @param packedSymbols each code's bytes packed LSB-first into a `long`, indexed by code
+    /// @param lengths each code's symbol length in bytes, indexed by code
+    /// @return a decompressor bound to the given table
+    public static Decompressor of(long[] packedSymbols, int[] lengths) {
+        return new Decompressor(packedSymbols, lengths);
+    }
+
+    /// Decodes the compressed byte range `[start, end)` of `compressed` into `out`, writing the
+    /// decoded bytes starting at `outPos`.
+    ///
+    /// The caller is responsible for sizing `out` so every decoded byte fits: an `end - start`
+    /// code stream expands to at most `8 * (end - start)` output bytes (a full run of length-8
+    /// symbols). This is an internal algorithm boundary, not an untrusted-input boundary, so no
+    /// defensive bounds checks are performed here.
+    ///
+    /// @param compressed the compressed code stream
+    /// @param start the index of the first code to decode, inclusive
+    /// @param end the index one past the last code to decode, exclusive
+    /// @param out the destination buffer for decoded bytes
+    /// @param outPos the index in `out` at which to start writing
+    /// @return the index in `out` one past the last byte written
+    public long decompress(byte[] compressed, int start, int end, byte[] out, long outPos) {
+        int pos = start;
+        int outIndex = (int) outPos;
+        while (pos < end) {
+            int code = compressed[pos++] & 0xFF;
+            if (code == ESCAPE) {
+                out[outIndex++] = compressed[pos++];
+            } else {
+                long packed = packedSymbols[code];
+                int length = lengths[code];
+                for (int k = 0; k < length; k++) {
+                    out[outIndex++] = (byte) (packed >>> (k * 8));
+                }
+            }
+        }
+        return outIndex;
+    }
+}

fsst/src/main/java/io/github/dfa1/vortex/fsst/Symbol.java

@@ -0,0 +1,47 @@
+package io.github.dfa1.vortex.fsst;
+
+/// A single FSST symbol: up to 8 bytes packed LSB-first into a `long` (byte 0 in the low byte,
+/// byte `k` at bit position `k * 8`) paired with an explicit length in `1..8`.
+///
+/// The LSB-first packing convention is shared everywhere a symbol crosses a boundary — the
+/// training tables, the compressor, the decompressor, and eventually the `vortex.fsst` wire
+/// format — so a symbol packed here reads back byte-for-byte identically wherever it lands.
+///
+/// @param packedBytes the symbol's bytes, LSB-first (byte `k` at bit position `k * 8`)
+/// @param length the symbol length in bytes, in `1..8`
+public record Symbol(long packedBytes, int length) {
+
+    /// Maximum symbol length in bytes, bounded by what fits in one `long`.
+    private static final int MAX_LENGTH = 8;
+
+    /// Validates that `length` is a legal FSST symbol length.
+    ///
+    /// @throws IllegalArgumentException if `length` is not in `1..8`
+    public Symbol {
+        if (length < 1 || length > MAX_LENGTH) {
+            throw new IllegalArgumentException("symbol length must be in 1..8, was " + length);
+        }
+    }
+
+    /// Packs `length` bytes of `data` starting at `offset` LSB-first into a new symbol.
+    ///
+    /// @param data the source bytes
+    /// @param offset the index of the symbol's first byte in `data`
+    /// @param length the symbol length in bytes, in `1..8`
+    /// @return a symbol holding the `length` bytes at `data[offset .. offset + length)`
+    public static Symbol of(byte[] data, int offset, int length) {
+        long value = 0;
+        for (int k = 0; k < length; k++) {
+            value |= Byte.toUnsignedLong(data[offset + k]) << (k * 8);
+        }
+        return new Symbol(value, length);
+    }
+
+    /// Returns the `i`-th byte of this symbol (0-indexed, so byte 0 is the LSB).
+    ///
+    /// @param i the byte index, in `0 .. length - 1`
+    /// @return the `i`-th byte of the symbol
+    public byte byteAt(int i) {
+        return (byte) (packedBytes >>> (i * 8));
+    }
+}

fsst/src/test/java/io/github/dfa1/vortex/fsst/DecompressorTest.java

@@ -0,0 +1,119 @@
+package io.github.dfa1.vortex.fsst;
+
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class DecompressorTest {
+
+    @Nested
+    class SingleCode {
+
+        @Test
+        void decompress_singleByteSymbol_appendsOneByte() {
+            // Given — code 0 maps to the single byte 'A'.
+            Decompressor sut = Decompressor.of(new long[]{'A'}, new int[]{1});
+            byte[] out = new byte[1];
+
+            // When
+            long result = sut.decompress(new byte[]{0x00}, 0, 1, out, 0);
+
+            // Then
+            assertThat(result).isEqualTo(1);
+            assertThat(out).isEqualTo("A".getBytes(StandardCharsets.UTF_8));
+        }
+
+        @Test
+        void decompress_multiByteSymbol_appendsAllBytesLsbFirst() {
+            // Given — code 0 maps to the 4-byte symbol "abcd", packed LSB-first via Symbol.of.
+            Symbol abcd = Symbol.of("abcd".getBytes(StandardCharsets.UTF_8), 0, 4);
+            Decompressor sut = Decompressor.of(new long[]{abcd.packedBytes()}, new int[]{4});
+            byte[] out = new byte[4];
+
+            // When
+            long result = sut.decompress(new byte[]{0x00}, 0, 1, out, 0);
+
+            // Then
+            assertThat(result).isEqualTo(4);
+            assertThat(out).isEqualTo("abcd".getBytes(StandardCharsets.UTF_8));
+        }
+
+        @Test
+        void decompress_escapeCode_copiesLiteralByte() {
+            // Given — no symbols; the escape code copies the following literal byte verbatim.
+            Decompressor sut = Decompressor.of(new long[0], new int[0]);
+            byte[] out = new byte[1];
+
+            // When
+            long result = sut.decompress(new byte[]{(byte) Decompressor.ESCAPE, 'Z'}, 0, 2, out, 0);
+
+            // Then
+            assertThat(result).isEqualTo(1);
+            assertThat(out).isEqualTo("Z".getBytes(StandardCharsets.UTF_8));
+        }
+    }
+
+    @Nested
+    class MultipleCodes {
+
+        @Test
+        void decompress_symbolsAndEscape_concatenateInOrder() {
+            // Given — code 0 = "ab" (2 bytes), code 1 = "c" (1 byte), then an escaped 'd'.
+            Symbol ab = Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2);
+            Decompressor sut = Decompressor.of(new long[]{ab.packedBytes(), 'c'}, new int[]{2, 1});
+            byte[] compressed = {0x00, 0x01, (byte) Decompressor.ESCAPE, 'd'};
+            byte[] out = new byte[4];
+
+            // When
+            long result = sut.decompress(compressed, 0, compressed.length, out, 0);
+
+            // Then
+            assertThat(result).isEqualTo(4);
+            assertThat(out).isEqualTo("abcd".getBytes(StandardCharsets.UTF_8));
+        }
+
+        @Test
+        void decompress_startEndSubrange_decodesOnlyTheGivenWindow() {
+            // Given — a stream with a leading and trailing code that must be ignored when only the
+            // middle window [1, 2) is requested.
+            Decompressor sut = Decompressor.of(new long[]{'X', 'Y', 'Z'}, new int[]{1, 1, 1});
+            byte[] compressed = {0x00, 0x01, 0x02};
+            byte[] out = new byte[1];
+
+            // When
+            long result = sut.decompress(compressed, 1, 2, out, 0);
+
+            // Then — only code 1 ('Y') is decoded.
+            assertThat(result).isEqualTo(1);
+            assertThat(out).isEqualTo("Y".getBytes(StandardCharsets.UTF_8));
+        }
+    }
+
+    @Nested
+    class MultipleRows {
+
+        @Test
+        void decompress_sequentialRows_doNotCorruptEachOther() {
+            // Given — two independent rows written into the same output buffer at sequential
+            // positions. Row 0 = "ab" (symbol), row 1 = escaped 'c'. A wrong outPos advance would
+            // let row 1 overwrite row 0's tail, so decoding both and checking each byte catches it.
+            Symbol ab = Symbol.of("ab".getBytes(StandardCharsets.UTF_8), 0, 2);
+            Decompressor sut = Decompressor.of(new long[]{ab.packedBytes()}, new int[]{2});
+            byte[] out = new byte[3];
+
+            // When
+            long afterRow0 = sut.decompress(new byte[]{0x00}, 0, 1, out, 0);
+            long afterRow1 = sut.decompress(new byte[]{(byte) Decompressor.ESCAPE, 'c'}, 0, 2, out, afterRow0);
+
+            // Then
+            assertThat(afterRow0).isEqualTo(2);
+            assertThat(afterRow1).isEqualTo(3);
+            assertThat(Arrays.copyOfRange(out, 0, 2)).isEqualTo("ab".getBytes(StandardCharsets.UTF_8));
+            assertThat(Arrays.copyOfRange(out, 2, 3)).isEqualTo("c".getBytes(StandardCharsets.UTF_8));
+        }
+    }
+}

fsst/src/test/java/io/github/dfa1/vortex/fsst/SymbolTest.java

@@ -0,0 +1,74 @@
+package io.github.dfa1.vortex.fsst;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class SymbolTest {
+
+    @Test
+    void of_packsBytesLsbFirst() {
+        // Given — "ab" should pack to 0x6261 (byte 'a'=0x61 in the low byte, 'b'=0x62 next).
+        byte[] data = "ab".getBytes(StandardCharsets.UTF_8);
+
+        // When
+        Symbol result = Symbol.of(data, 0, 2);
+
+        // Then
+        assertThat(result.packedBytes()).isEqualTo(0x6261L);
+        assertThat(result.length()).isEqualTo(2);
+    }
+
+    @Test
+    void of_respectsOffset() {
+        // Given — packing 2 bytes starting at offset 2 must read "cd", not "ab".
+        byte[] data = "abcd".getBytes(StandardCharsets.UTF_8);
+
+        // When
+        Symbol result = Symbol.of(data, 2, 2);
+
+        // Then
+        assertThat(result.byteAt(0)).isEqualTo((byte) 'c');
+        assertThat(result.byteAt(1)).isEqualTo((byte) 'd');
+    }
+
+    @Test
+    void byteAt_returnsEachByteInOrder() {
+        // Given
+        Symbol sut = Symbol.of("WXYZ".getBytes(StandardCharsets.UTF_8), 0, 4);
+
+        // When
+        byte[] result = {sut.byteAt(0), sut.byteAt(1), sut.byteAt(2), sut.byteAt(3)};
+
+        // Then
+        assertThat(result).isEqualTo("WXYZ".getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Test
+    void of_eightByteSymbol_packsFullLong() {
+        // Given — a full 8-byte symbol fills every byte of the long, the length boundary.
+        byte[] data = "ABCDEFGH".getBytes(StandardCharsets.UTF_8);
+
+        // When
+        Symbol result = Symbol.of(data, 0, 8);
+
+        // Then
+        assertThat(result.length()).isEqualTo(8);
+        assertThat(result.byteAt(7)).isEqualTo((byte) 'H');
+    }
+
+    @ParameterizedTest
+    @ValueSource(ints = {0, -1, 9})
+    void constructor_lengthOutOfRange_throws(int length) {
+        // Given
+        // When
+        // Then
+        assertThatThrownBy(() -> new Symbol(0L, length))
+                .isInstanceOf(IllegalArgumentException.class);
+    }
+}

pom.xml

@@ -40,6 +40,7 @@
 	<modules>
 		<module>proto-gen</module>
 		<module>fbs-gen</module>
+		<module>fsst</module>
 		<module>core</module>
 		<module>reader</module>
 		<module>writer</module>
@@ -160,6 +161,11 @@
 				<artifactId>vortex-writer</artifactId>
 				<version>${project.version}</version>
 			</dependency>
+			<dependency>
+				<groupId>io.github.dfa1.vortex</groupId>
+				<artifactId>vortex-fsst</artifactId>
+				<version>${project.version}</version>
+			</dependency>
 			<dependency>
 				<groupId>io.github.dfa1.vortex</groupId>
 				<artifactId>vortex-csv</artifactId>