From 4aca570de0349b353ffcda4f19dd0a02681d41ed Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 20 Jul 2026 08:09:46 +0200 Subject: [PATCH] feat(fsst): rewire writer/reader adapters onto the fsst module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 5 of #287: reduce the two FSST production files to thin wire adapters over the standalone vortex-fsst module. Only production code touched; no test files changed (that is PR 6's scope). The wire format is unchanged. Deleted (writer): the ~250-line inline SymbolTable/SymbolCandidate machinery — training loop (trainSymbolTable), stratified sampling (sampleRowIndices), the open-addressing longest-match index, candidate counting, and per-row compress. All of that now lives in CompressorBuilder/Compressor/Matcher. Deleted (reader): the byte-by-byte decompressString loop. Decoding now goes through Decompressor (the unconditional 8-byte-store Algorithm 1). Kept (both): all wire/metadata plumbing — UTF-8 conversion, ProtoFSSTMetadata with narrowestUnsigned ptype selection, the EncodeNode/EncodeResult two-child tree (uncompLens/codesOff), arena allocation of every output buffer, the readUnsigned ptype dispatch, and the VarBinArray.OffsetMode result. The TRAINING_SAMPLE_SEED reproducibility constant is retained and now feeds CompressorBuilder.seed(). Wire-order code remapping (the subtle part — over-explained on purpose): The Compressor numbers its codes in gain-descending order, but the vortex.fsst wire format lays symbols out in length order (all multi-byte length-ascending, then length-1 last; mirrors Rust FSSTData::validate_symbol_lengths). So the writer must express BOTH the symbol table AND the code stream in wire codes. wireOrder = compressor.codesSortedByLength() — wireOrder[i] is the internal (gain-descending) code that belongs at wire position i. internalToWire[wireOrder[i]] = i — the inverse permutation, mapping every internal code the compressor emits to its wire position. Symbol table buffers: at wire position i write packedSymbol(wireOrder[i]) and symbolLength(wireOrder[i]) — the symbol living at wire slot i is internal symbol wireOrder[i]. Code stream: compressor.compress emits internal codes; each non-escape code byte c is rewritten to internalToWire[c]. The escape byte 0xFF and the single literal byte following it are copied through untouched — the literal is raw data, not a code, and 0xFF never collides with a code since codes are 0..254 (numSymbols <= 255). Reader side: it reads the wire symbol table into code-indexed arrays and a code b in the stream indexes wire slot b directly, so encoder position i and decoder index i refer to the same symbol — the file is self-consistent and round-trips. Verified: (1) writer,reader unit suite green with zero test changes (1713 tests); (2) checkstyle/javadoc gate green; (3) JavaWritesRustReadsIntegration javaWriter_jniReader_fsstUtf8Column — the real Rust reference reader decodes this encoder's output; (4) RustJavaReaderComparison fsst.vortex fixture (our decoder reads a Rust-written FSST file) and FileSizeComparison highCardinalityUtf8_javaVsJni (FSST still dispatch-selected) green; (5) full reactor build green; plus a throwaway 11-row driver proving encode→remap→wire symbol table→Decompressor reproduces every input byte-for-byte with the wire table in valid length order. Also fixed the empty-table read: with 0 trained symbols the writer floors the symbol buffers to a 1-byte placeholder, so the decoder derives the symbol count from symbolsBuf.byteSize()/8 (floors to 0) rather than symbolLensBuf.byteSize(). Co-Authored-By: Claude Sonnet 5 --- reader/pom.xml | 4 + .../reader/decode/FsstEncodingDecoder.java | 53 +-- writer/pom.xml | 4 + .../writer/encode/FsstEncodingEncoder.java | 384 +++--------------- 4 files changed, 92 insertions(+), 353 deletions(-) diff --git a/reader/pom.xml b/reader/pom.xml index dc2982c5..1f4a7c3b 100644 --- a/reader/pom.xml +++ b/reader/pom.xml @@ -20,6 +20,10 @@ io.github.dfa1.vortex vortex-core + + io.github.dfa1.vortex + vortex-fsst + io.github.dfa1.zstd zstd diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java index 343f9acb..6f3c6ae8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java @@ -9,16 +9,20 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata; import io.github.dfa1.vortex.core.proto.ProtoPType; +import io.github.dfa1.vortex.fsst.Decompressor; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; /// Read-only decoder for `vortex.fsst`. +/// +/// This class is a thin wire adapter over the standalone `vortex-fsst` module (issue #287): after +/// parsing the `vortex.fsst` wire buffers (symbol table, per-row uncompressed lengths, code offsets, +/// [ProtoFSSTMetadata]), it hands the symbol table and each row's code range to a [Decompressor], +/// which runs the FSST paper's Algorithm 1 decode. public final class FsstEncodingDecoder implements EncodingDecoder { - private static final int ESCAPE = 0xFF; - @Override public EncodingId encodingId() { return EncodingId.VORTEX_FSST; @@ -63,12 +67,30 @@ public Array decode(DecodeContext ctx) { long uncompLensCap = SegmentBroadcast.capacity(uncompLensSeg, uncompLenPType.byteSize()); long codesOffCap = SegmentBroadcast.capacity(codesOffsetsSeg, codesOffPType.byteSize()); + // Read the wire symbol table into parallel code-indexed arrays once per chunk (there are at + // most 255 symbols), then hand them to the decompressor. symbolsBuf carries one LSB-first + // long per symbol, so its size divided by 8 is the symbol count: an empty table is written + // as a 1-byte placeholder buffer (allocations are floored at 1 byte), which floors to 0 + // symbols here — an all-escape column decodes without touching the symbol table. + int numSymbols = (int) (symbolsBuf.byteSize() / 8); + long[] packedSymbols = new long[numSymbols]; + int[] symbolLengths = new int[numSymbols]; + for (int code = 0; code < numSymbols; code++) { + packedSymbols[code] = symbolsBuf.getAtIndex(VortexFormat.LE_LONG, code); + symbolLengths[code] = Byte.toUnsignedInt(symbolLensBuf.get(ValueLayout.JAVA_BYTE, code)); + } + Decompressor decompressor = Decompressor.of(packedSymbols, symbolLengths); + long totalUncompressed = 0L; for (long i = 0; i < n; i++) { totalUncompressed += readUnsigned(uncompLensSeg, i % uncompLensCap, uncompLenPType); } - MemorySegment outBytes = ctx.arena().allocate(totalUncompressed); + // Allocate 7 bytes of slack past the true logical length: the decompressor's unconditional + // 8-byte-store trick writes a full 8 bytes for the final symbol even when it contributes as + // few as 1 real byte. The slack is sliced off before the buffer is exposed, so callers still + // see an exactly-sized buffer. + MemorySegment outBytes = ctx.arena().allocate(totalUncompressed + 7); MemorySegment outOffsets = ctx.arena().allocate((n + 1) * 4L, 4); outOffsets.setAtIndex(VortexFormat.LE_INT, 0, 0); @@ -76,31 +98,12 @@ public Array decode(DecodeContext ctx) { for (long i = 0; i < n; i++) { long cStart = readUnsigned(codesOffsetsSeg, i % codesOffCap, codesOffPType); long cEnd = readUnsigned(codesOffsetsSeg, (i + 1) % codesOffCap, codesOffPType); - outPos = decompressString(compressedBytes, symbolsBuf, symbolLensBuf, - cStart, cEnd, outBytes, outPos); + outPos = decompressor.decompress(compressedBytes, cStart, cEnd, outBytes, outPos); outOffsets.setAtIndex(VortexFormat.LE_INT, i + 1, (int) outPos); } - return new VarBinArray.OffsetMode(ctx.dtype(), n, outBytes.asReadOnly(), outOffsets.asReadOnly(), PType.I32); - } - - private static long decompressString( - MemorySegment compressed, MemorySegment symbols, MemorySegment symLens, - long start, long end, MemorySegment out, long outPos - ) { - for (long j = start; j < end; j++) { - int b = Byte.toUnsignedInt(compressed.get(ValueLayout.JAVA_BYTE, j)); - if (b == ESCAPE) { - out.set(ValueLayout.JAVA_BYTE, outPos++, compressed.get(ValueLayout.JAVA_BYTE, ++j)); - } else { - int symLen = Byte.toUnsignedInt(symLens.get(ValueLayout.JAVA_BYTE, b)); - long sym = symbols.getAtIndex(VortexFormat.LE_LONG, b); - for (int k = 0; k < symLen; k++) { - out.set(ValueLayout.JAVA_BYTE, outPos++, (byte) (sym >>> (k * 8))); - } - } - } - return outPos; + return new VarBinArray.OffsetMode(ctx.dtype(), n, + outBytes.asSlice(0, totalUncompressed).asReadOnly(), outOffsets.asReadOnly(), PType.I32); } private static long readUnsigned(MemorySegment seg, long idx, PType ptype) { diff --git a/writer/pom.xml b/writer/pom.xml index 1fbf553e..6e28edb1 100644 --- a/writer/pom.xml +++ b/writer/pom.xml @@ -20,6 +20,10 @@ io.github.dfa1.vortex vortex-core + + io.github.dfa1.vortex + vortex-fsst + io.github.dfa1.zstd zstd diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java index 727d1df3..fac11207 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java @@ -6,57 +6,38 @@ import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata; +import io.github.dfa1.vortex.fsst.Compressor; +import io.github.dfa1.vortex.fsst.CompressorBuilder; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Random; /// Write-only encoder for `vortex.fsst`. /// -/// Builds a variable-length (1-8 byte) symbol table with iterative refinement, following the -/// FSST paper (Boncz/Neumann/Winter, "FSST: Fast Random Access String Compression"). Each -/// training pass compresses a bounded sample of the input with the current table using -/// longest-match-first greedy parsing, counts the byte savings of extending each matched symbol -/// by one more byte, then rebuilds the table from the top candidates. The symbol table converges -/// after a handful of passes, after which the whole input is compressed with the final table. +/// This class is a thin wire adapter over the standalone `vortex-fsst` module (issue #287): the +/// FSST compression algorithm — symbol-table training and greedy longest-match compression — lives +/// entirely in [CompressorBuilder]/[Compressor]. This adapter converts the input strings to UTF-8 +/// bytes, drives training, compresses each row, and lays the result out in the `vortex.fsst` wire +/// format (symbol table buffers, remapped code stream, per-row uncompressed lengths and code +/// offsets, plus the [ProtoFSSTMetadata] describing the two offset ptypes). /// /// The wire format packs each symbol's bytes LSB-first into a `long` (first byte in the low byte) /// alongside a per-symbol length byte, and reserves code `0xFF` as the single-literal-byte escape. /// Real symbol codes are therefore `0..254` (max 255 symbols), each 1-8 bytes long. +/// +/// Symbols are laid out on the wire in length order — all multi-byte symbols (length 2-8) first in +/// non-decreasing length order, then all length-1 symbols — as the `vortex.fsst` wire contract +/// requires (mirroring Rust `FSSTData::validate_symbol_lengths`). The [Compressor] instead numbers +/// its codes in gain-descending order, so this adapter remaps every code the compressor emits into +/// that length-sorted wire order (see [#encode]). public final class FsstEncodingEncoder implements EncodingEncoder { /// Escape opcode: emitted as `0xFF` followed by one literal byte. private static final int ESCAPE = 0xFF; - /// Maximum number of real symbols (codes `0..254`; `0xFF` is the escape). - private static final int MAX_SYMBOLS = 255; - - /// Maximum symbol length in bytes, bounded by what fits in one `long`. - private static final int MAX_SYMBOL_LENGTH = 8; - - /// Number of training passes. The FSST paper reports the table stabilizing after roughly - /// five iterations; each pass can only lengthen symbols by one byte, so five passes suffice - /// to grow symbols up to length ~6 from an empty start, which captures nearly all of the gain - /// on the string workloads seen here. More passes cost linear time for negligible benefit. - private static final int TRAINING_ITERATIONS = 5; - - /// Cap on the number of input strings sampled per training pass. Training is O(sample bytes) - /// per pass, so very large inputs are sampled rather than fully scanned; the final compression - /// pass still runs over every string. 25k strings is a large-enough sample to learn a stable - /// table while keeping training cost bounded. - private static final int TRAINING_SAMPLE_STRINGS = 25_000; - - /// Number of contiguous strides the training sample is drawn from, mirroring - /// [CascadingCompressor]'s stratified sampling. One random contiguous chunk of rows is - /// taken from each stride so the sample covers the whole input rather than only its prefix. - private static final int TRAINING_STRIDE_COUNT = 32; - /// Fixed seed for the training-sample PRNG. Encoding must be reproducible: the same input /// always trains the same symbol table and produces byte-identical output. private static final long TRAINING_SAMPLE_SEED = 0x5EEDL; @@ -81,32 +62,47 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { byteArrays[i] = strings[i].getBytes(StandardCharsets.UTF_8); } - SymbolTable table = trainSymbolTable(byteArrays); + Compressor compressor = new CompressorBuilder().seed(TRAINING_SAMPLE_SEED).train(byteArrays); + int numSymbols = compressor.symbolCount(); - byte[][] compressed = new byte[n][]; - for (int i = 0; i < n; i++) { - compressed[i] = table.compress(byteArrays[i]); + // Wire order: the wire lists symbols in length order (multi-byte length-ascending, then + // length-1 last), whereas the compressor numbers its codes gain-descending. wireOrder[i] is + // the compressor's (internal) code that belongs at wire position i; internalToWire is its + // inverse, mapping every internal code emitted by compress() to its wire code. Both the + // symbol-table buffers and the code stream must be expressed in wire codes so the file is + // self-consistent. + int[] wireOrder = compressor.codesSortedByLength(); + int[] internalToWire = new int[numSymbols]; + for (int i = 0; i < numSymbols; i++) { + internalToWire[wireOrder[i]] = i; } Arena arena = ctx.arena(); - int numSymbols = table.size(); MemorySegment symBuf = arena.allocate(Math.max(numSymbols * 8L, 1), 8); - for (int i = 0; i < numSymbols; i++) { - symBuf.setAtIndex(VortexFormat.LE_LONG, i, table.packedBytes(i)); - } - MemorySegment symLenBuf = arena.allocate(Math.max(numSymbols, 1)); for (int i = 0; i < numSymbols; i++) { - symLenBuf.set(ValueLayout.JAVA_BYTE, i, (byte) table.length(i)); + int internalCode = wireOrder[i]; + symBuf.setAtIndex(VortexFormat.LE_LONG, i, compressor.packedSymbol(internalCode)); + symLenBuf.set(ValueLayout.JAVA_BYTE, i, (byte) compressor.symbolLength(internalCode)); } + // Compress each row into a scratch buffer holding the compressor's internal codes, then + // remap only the code bytes (never the literal byte following an escape) to wire codes. + byte[][] compressed = new byte[n][]; int totalCompressed = 0; int maxUncompLen = 0; for (int i = 0; i < n; i++) { - totalCompressed += compressed[i].length; - maxUncompLen = Math.max(maxUncompLen, byteArrays[i].length); + byte[] row = byteArrays[i]; + byte[] scratch = new byte[row.length * 2]; + int len = (int) compressor.compress(row, 0, row.length, scratch, 0); + remapCodesToWire(scratch, len, internalToWire); + compressed[i] = new byte[len]; + System.arraycopy(scratch, 0, compressed[i], 0, len); + totalCompressed += len; + maxUncompLen = Math.max(maxUncompLen, row.length); } + MemorySegment compBuf = arena.allocate(Math.max(totalCompressed, 1)); long pos = 0; for (byte[] c : compressed) { @@ -154,291 +150,23 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { null, null); } - /// Trains a symbol table by iteratively refining candidate symbols against a bounded sample. - /// - /// @param byteArrays the raw byte content of every input string - /// @return the final symbol table (0-255 symbols, each 1-8 bytes) - private static SymbolTable trainSymbolTable(byte[][] byteArrays) { - int sampleSize = Math.min(byteArrays.length, TRAINING_SAMPLE_STRINGS); - int[] sampleIndices = sampleRowIndices(byteArrays.length, sampleSize); - SymbolTable table = SymbolTable.empty(); - for (int iteration = 0; iteration < TRAINING_ITERATIONS; iteration++) { - Map counts = new HashMap<>(); - for (int idx : sampleIndices) { - table.countCandidates(byteArrays[idx], counts); - } - if (counts.isEmpty()) { - break; - } - table = SymbolTable.fromRankedCandidates(counts); - } - return table; - } - - /// Stratified sample of row indices: partitions `[0, n)` into `TRAINING_STRIDE_COUNT` - /// contiguous strides and draws one contiguous sub-range from each, the same scheme as - /// `CascadingCompressor`'s stratified sampling. Sampling only the first `sampleSize` rows - /// would silently bias the learned table on inputs sorted or clustered by value; this covers - /// the whole input while keeping each drawn chunk contiguous. + /// Remaps every code byte in `stream[0..length)` from the compressor's internal (gain-descending) + /// code to its wire (length-sorted) code, in place. The escape byte `0xFF` and the single literal + /// byte that follows it are copied through unchanged — the literal is raw data, not a code. /// - /// @param n total row count - /// @param sampleSize number of rows to draw, `<= n` - /// @return `sampleSize` row indices into `[0, n)` - @SuppressWarnings("java:S2245") // Deterministic PRNG is the contract: training samples must - // be reproducible across builds. No security boundary. - private static int[] sampleRowIndices(int n, int sampleSize) { - int[] indices = new int[sampleSize]; - if (sampleSize == 0) { - return indices; - } - int strideCount = Math.min(TRAINING_STRIDE_COUNT, sampleSize); - Random rng = new Random(TRAINING_SAMPLE_SEED); - int dstOff = 0; - int partRemainder = n % strideCount; - int partShortStep = n / strideCount; - int partLongStep = partShortStep + 1; - int sampleRemainder = sampleSize % strideCount; - int sampleShortStep = sampleSize / strideCount; - int sampleLongStep = sampleShortStep + 1; - for (int s = 0; s < strideCount; s++) { - int partStart = s * partShortStep + Math.min(s, partRemainder); - int partLen = s < partRemainder ? partLongStep : partShortStep; - int sampleLen = s < sampleRemainder ? sampleLongStep : sampleShortStep; - int maxStart = Math.max(0, partLen - sampleLen); - int offsetInPart = maxStart == 0 ? 0 : rng.nextInt(maxStart + 1); - int srcOff = partStart + offsetInPart; - for (int k = 0; k < sampleLen; k++) { - indices[dstOff++] = srcOff + k; - } - } - return indices; - } - - /// A candidate symbol: up to 8 bytes packed LSB-first into a `long` (first byte in the low - /// byte, matching the wire format) paired with an explicit length in `1..8`. - /// - /// @param packedBytes the symbol's bytes, LSB-first (byte `k` at bit position `k * 8`) - /// @param length the symbol length in bytes, `1..8` - private record SymbolCandidate(long packedBytes, int length) { - } - - /// An immutable FSST symbol table plus the machinery to compress against it and to count - /// extension candidates during training. - private static final class SymbolTable { - - /// Smallest match-index capacity; also the capacity used for the empty table. - private static final int MIN_INDEX_CAPACITY = 4; - - private final long[] packed; - private final int[] lengths; - private final int count; - - /// Open-addressing index for exact-match lookup during greedy parsing and compression, - /// keyed by `(packedBytes, length)`, mapping to the symbol code. Plain primitive arrays - /// rather than `Map`: `longestMatch` below runs up to - /// `MAX_SYMBOL_LENGTH` probes per byte position, for every position of every row in the - /// whole dataset, so a boxed record key per probe is real allocation pressure in the - /// hottest loop this encoder has. - private final long[] indexKey; - private final byte[] indexLen; // 0 = empty slot; valid symbol lengths are 1..8 - private final int[] indexCode; - private final int indexMask; - - private SymbolTable(long[] packed, int[] lengths, int count) { - this.packed = packed; - this.lengths = lengths; - this.count = count; - - int capacity = MIN_INDEX_CAPACITY; - while (capacity < count * 4) { - capacity <<= 1; - } - this.indexMask = capacity - 1; - this.indexKey = new long[capacity]; - this.indexLen = new byte[capacity]; - this.indexCode = new int[capacity]; - for (int code = 0; code < count; code++) { - indexInsert(packed[code], lengths[code], code); - } - } - - private void indexInsert(long key, int len, int code) { - int idx = indexHash(key, len) & indexMask; - while (indexLen[idx] != 0) { - idx = (idx + 1) & indexMask; - } - indexKey[idx] = key; - indexLen[idx] = (byte) len; - indexCode[idx] = code; - } - - private int indexLookup(long key, int len) { - int idx = indexHash(key, len) & indexMask; - while (indexLen[idx] != 0) { - if (indexLen[idx] == len && indexKey[idx] == key) { - return indexCode[idx]; - } - idx = (idx + 1) & indexMask; - } - return -1; - } - - private static int indexHash(long key, int len) { - long h = (key ^ ((long) len * 0x9E3779B97F4A7C15L)) * 0xC2B2AE3D27D4EB4FL; - return (int) (h ^ (h >>> 32)); - } - - static SymbolTable empty() { - return new SymbolTable(new long[0], new int[0], 0); - } - - /// Builds a table from ranked candidates, keeping the top `MAX_SYMBOLS` by estimated gain. - /// - /// Gain is `count * length`: the number of input bytes this symbol covers, since every - /// symbol code — regardless of length — emits exactly one output byte. This is the FSST - /// paper's "apparent gain" and it is what makes length-1 symbols competitive: a frequent - /// single byte covers many input bytes at one output byte each and, crucially, replaces a - /// 2-byte escape. Dropping a frequent single byte from the table forces escapes (2 output - /// bytes per occurrence), which is the failure mode of a naive bigram-only table. Ties - /// break toward longer symbols (strictly more valuable per code). - static SymbolTable fromRankedCandidates(Map counts) { - record Ranked(SymbolCandidate candidate, long gain) { - } - Ranked[] ranked = new Ranked[counts.size()]; - int idx = 0; - for (Map.Entry e : counts.entrySet()) { - long gain = e.getValue() * (long) e.getKey().length(); - ranked[idx++] = new Ranked(e.getKey(), gain); - } - Arrays.sort(ranked, (a, b) -> { - int byGain = Long.compare(b.gain(), a.gain()); - if (byGain != 0) { - return byGain; - } - return Integer.compare(b.candidate().length(), a.candidate().length()); - }); - - int keep = Math.min(ranked.length, MAX_SYMBOLS); - SymbolCandidate[] kept = new SymbolCandidate[keep]; - for (int i = 0; i < keep; i++) { - kept[i] = ranked[i].candidate(); - } - - // Wire order (Rust `FSSTData::validate_symbol_lengths`): multi-byte symbols - // (length 2-8) first in non-decreasing length order, then all length-1 symbols. - // Selection above picks candidates by gain; this final sort only reorders codes - // to satisfy that wire contract, so ties within a length bucket are arbitrary. - Arrays.sort(kept, (a, b) -> { - boolean aSingle = a.length() == 1; - boolean bSingle = b.length() == 1; - if (aSingle != bSingle) { - return aSingle ? 1 : -1; - } - return Integer.compare(a.length(), b.length()); - }); - - long[] packed = new long[keep]; - int[] lengths = new int[keep]; - for (int i = 0; i < keep; i++) { - packed[i] = kept[i].packedBytes(); - lengths[i] = kept[i].length(); - } - return new SymbolTable(packed, lengths, keep); - } - - int size() { - return count; - } - - long packedBytes(int code) { - return packed[code]; - } - - int length(int code) { - return lengths[code]; - } - - /// Returns the code of the longest symbol matching `input` at `pos`, or -1 if - /// none matches. Checks lengths `MAX_SYMBOL_LENGTH` down to 1. - private int longestMatch(byte[] input, int pos) { - int maxLen = Math.min(MAX_SYMBOL_LENGTH, input.length - pos); - for (int len = maxLen; len >= 1; len--) { - int code = indexLookup(pack(input, pos, len), len); - if (code >= 0) { - return code; - } - } - return -1; - } - - /// Walks `input` with longest-match-first greedy parsing using the current table, - /// accumulating training candidates into `counts`: - /// - each single byte that starts a match (or an unmatched byte) as a length-1 candidate, - /// so the table can bootstrap from empty and keep its seed symbols; - /// - the matched multi-byte symbol itself, so it survives the next rebuild; - /// - the concatenation of the current match with the next match, capped at 8 bytes. This is - /// the FSST paper's core heuristic: pairing adjacent symbols lets symbols roughly double - /// in length each pass (not just grow by one byte), so length-8 symbols emerge in a few - /// passes rather than seven. - void countCandidates(byte[] input, Map counts) { - int i = 0; - while (i < input.length) { - int code = longestMatch(input, i); - int matchLen = code >= 0 ? lengths[code] : 1; - - // Always offer the length-1 symbol at this position (seeds/keeps single bytes). - bump(counts, new SymbolCandidate(pack(input, i, 1), 1)); - - if (matchLen > 1) { - // Reinforce the matched multi-byte symbol so it survives the next rebuild. - bump(counts, new SymbolCandidate(pack(input, i, matchLen), matchLen)); - } - - // Pair the current match with the following match to form a longer candidate. - int next = i + matchLen; - if (next < input.length && matchLen < MAX_SYMBOL_LENGTH) { - int nextCode = longestMatch(input, next); - int nextLen = nextCode >= 0 ? lengths[nextCode] : 1; - int pairLen = Math.min(matchLen + nextLen, MAX_SYMBOL_LENGTH); - bump(counts, new SymbolCandidate(pack(input, i, pairLen), pairLen)); - } - - i += matchLen; - } - } - - /// Compresses `input` with longest-match-first greedy parsing over the final table. - /// Positions matching no symbol are emitted as an escape (`0xFF` + literal byte). - byte[] compress(byte[] input) { - byte[] out = new byte[input.length * 2]; - int outLen = 0; - int i = 0; - while (i < input.length) { - int code = longestMatch(input, i); - if (code >= 0) { - out[outLen++] = (byte) code; - i += lengths[code]; - } else { - out[outLen++] = (byte) ESCAPE; - out[outLen++] = input[i]; - i++; - } - } - return Arrays.copyOf(out, outLen); - } - - private static void bump(Map counts, SymbolCandidate candidate) { - counts.merge(candidate, 1L, Long::sum); - } - - /// Packs `len` bytes of `input` starting at `pos` LSB-first into a - /// `long`: byte 0 in the low byte, matching the wire and decoder conventions. - private static long pack(byte[] input, int pos, int len) { - long value = 0; - for (int k = 0; k < len; k++) { - value |= Byte.toUnsignedLong(input[pos + k]) << (k * 8); + /// @param stream the freshly compressed code stream, mutated in place + /// @param length the number of valid bytes in `stream` + /// @param internalToWire maps an internal code to its wire code, indexed by internal code + private static void remapCodesToWire(byte[] stream, int length, int[] internalToWire) { + int j = 0; + while (j < length) { + int code = stream[j] & 0xFF; + if (code == ESCAPE) { + j += 2; + } else { + stream[j] = (byte) internalToWire[code]; + j++; } - return value; } } }