diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 7ad74d9a9..6e79f32b9 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -1017,6 +1017,7 @@ For background log or metrics sinks, use explicit writer-entry/release signals a - For bounded NDJSON streams, include plain and recipe/audit search paths, parse the final `terminal_record`, verify authoritative-versus-lower-bound totals (including per-query and total-limit omissions) and selection reasons/counts before result-limit trimming, count UTF-8 bytes including newlines, and assert partial exit `11` plus the explicit `--allow-partial` exit-`0` opt-in. Also assert that capped profile/verbose/envelope combinations fail before stdout. - Keep `symbols` and `files` `--results-only` coverage in one seeded discovery fixture: assert each command emits only parseable NDJSON result rows without a `terminal_record`, and assert array output is rejected instead of silently ignoring the row-stream control. - For `find --all`, cover row-terminal, count-object, and human scan metadata together: assert indexed literal candidate selection can reach a late-sorted match before the line cap, every indexed candidate is reverified, `search_strategy` / fallback reasons distinguish regex, exact-normalized, short, non-ASCII, legacy-index, active-bulk-rebuild, and missing-sync-trigger scans, and trigram synchronization survives incremental insert/update/delete plus bulk rebuild. Simulate an older writer by leaving the trigram table while removing its synchronization triggers; prove query-only fallback remains complete and writable initialization rebuilds the stale index. Also assert active file/line caps, `scan_complete` / authority fields, stable continuation/recovery guidance, the same partial-exit opt-in, option-order handling, and rejection of every row format that cannot carry terminal metadata. Envelope tests must keep row terminals in `metadata.stream_terminal` and preserve count objects as results. +- Long-token search fallback coverage must pair a greater-than-1000-rune minified HTML/JavaScript token with a short ASCII middle literal and an ordinary shorter-token substring control. Assert search, total count, and per-file count stay aligned; the fallback must require current `fts_token_too_long` evidence and a synchronized trigram index. - For `find` context controls, cover separated and inline `--context`, both relative orders with `--before` / `--after`, and a JSON-lines batch invocation; assert that explicit asymmetric sides win independently of order. - For rejected checkpoint names, assert the usage exit/error code and syntax hint together, and verify that no checkpoint directory was created. @@ -2078,6 +2079,7 @@ background の log / metrics sink は、sleep や狭い stopwatch 閾値では - 上限付き NDJSON stream は plain search と recipe / audit search の両経路を含め、最後の `terminal_record` を解析し、query ごとおよび total-limit の省略を含む authoritative / lower-bound の総件数、result-limit 適用前の selection 理由 / 件数、改行を含む UTF-8 byte 数、partial 終了コード `11`、明示的な `--allow-partial` による終了コード `0` の opt-in を検証する。上限付き profile / verbose / envelope の組み合わせが stdout 前に失敗することも確認する。 - `symbols` と `files` の `--results-only` coverage は 1 つの seed 済み discovery fixture で共有し、各 command が `terminal_record` を含まない parse 可能な NDJSON result row だけを出力すること、および array 出力では row-stream control を黙って無視せず拒否することを検証する。 - `find --all` は row 終端、count object、human scan metadata を併せて検証する。index を使う literal の候補選択が line cap より後ろに sort された一致へ到達できること、index の全候補を再検証すること、`search_strategy` / fallback reason が regex、exact normalization、短い query、非 ASCII、旧 index、bulk rebuild 中、同期 trigger 欠落の走査を区別すること、trigram 同期が incremental insert / update / delete と bulk rebuild 後も維持されることを確認する。旧 writer を模擬して trigram table を残したまま同期 trigger を削除し、query-only fallback が完全性を保つことと writable initialization が古い index を再構築することを証明する。さらに、有効な file / line cap、`scan_complete` / authority field、安定した continuation / recovery 案内、同じ partial-exit opt-in、option 順序の扱い、終端 metadata を持てない全 row format の拒否を確認する。envelope test では row 終端を `metadata.stream_terminal` に入れ、count object を result として保持することを検証する。 +- long-token search fallback coverage では、1000 rune 超の minified HTML / JavaScript token、中央にある短い ASCII literal、通常の短い token 内の substring control を組み合わせる。search、総 count、file ごとの count が一致すること、および fallback が現在有効な `fts_token_too_long` の evidence と同期済み trigram index を必須とすることを検証する。 - `find` の context control は、separated / inline の `--context`、`--before` / `--after` との両方の相対順序、JSON Lines batch invocation を検証し、明示した asymmetric side が順序に依存せず優先されることを確認する。 - 拒否される checkpoint 名では usage の終了コード / error code と構文 hint を併せて検証し、checkpoint directory が作成されていないことも確認する。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 4d88f8073..96e19e664 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1415,6 +1415,13 @@ Search normalizes literal FTS queries to Unicode NFC before matching. If every literal token exceeds SQLite FTS5 unicode61's 1000-character token cap, zero-result JSON includes `query_degraded_reason` and `tokens_dropped`. Index validation reports long unbroken FTS tokens as `fts_token_too_long`. +When an ordinary literal search has no matches, an ASCII unicode61 token +substring of at least three characters can be recovered from a reported +overlong token through the synchronized trigram index. The candidate is +rechecked against the greater-than-1000-rune unicode61 token before it is +returned, so substrings in ordinary tokens remain non-matches. This bounded +fallback is unavailable while the trigram index is missing, rebuilding, or +lacks its synchronization triggers. Literal-safe `search` queries are capped at 1000 characters and 128 whitespace terms. Oversized generated input is rejected before FTS5 sanitization; split it into smaller searches or use narrower text. @@ -4956,6 +4963,11 @@ literal FTS クエリは照合前に Unicode NFC へ正規化されます。す token が SQLite FTS5 unicode61 の 1000 文字 token 上限を超える場合、0 件 JSON には `query_degraded_reason` と `tokens_dropped` が含まれます。index validation は長い連続 FTS token を `fts_token_too_long` として報告します。 +通常の literal search が 0 件の場合、3 文字以上の ASCII unicode61 token の +部分文字列は、同期済み trigram index を介して報告済みの長すぎる token 内から復旧できます。 +候補を返す前に 1000 rune 超の unicode61 token 内の一致を再検証するため、通常の +token 内の部分文字列は一致になりません。この限定 fallback は trigram index が +存在しない場合、再構築中の場合、または同期 trigger が欠落している場合は使われません。 literal-safe な `search` query は 1000 文字、128 whitespace term までです。 生成された大きすぎる入力は FTS5 sanitization 前に拒否されるため、小さな検索へ分割するか、 より狭い text にしてください。 diff --git a/changelog.d/unreleased/5080.fixed.md b/changelog.d/unreleased/5080.fixed.md new file mode 100644 index 000000000..d213c791f --- /dev/null +++ b/changelog.d/unreleased/5080.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 5080 +affected: + - src/CodeIndex/Database/DbSearchReader.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.OversizeContentIssues.cs + - tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs + - USER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Literal search now recovers ASCII substrings inside reported overlong unicode61 tokens (#5080)** — when ordinary FTS returns no match, `search` uses the synchronized trigram index only for chunks carrying `fts_token_too_long` and rechecks the greater-than-1000-rune token before returning a result, without widening ordinary-token matches. + +## 日本語 + +- **literal search が報告済みの長すぎる unicode61 token 内の ASCII 部分文字列を検索できるようになりました (#5080)** — 通常の FTS が一致を返さない場合、`search` は `fts_token_too_long` のある chunk に限って同期済み trigram index を使い、1000 rune 超の token 内の一致を再検証して返すため、通常 token の照合範囲は広がりません。 diff --git a/src/CodeIndex/Database/DbSearchReader.cs b/src/CodeIndex/Database/DbSearchReader.cs index 6441da232..b5f88b977 100644 --- a/src/CodeIndex/Database/DbSearchReader.cs +++ b/src/CodeIndex/Database/DbSearchReader.cs @@ -1,4 +1,5 @@ using System.Text; +using CodeIndex.Indexer; using Microsoft.Data.Sqlite; namespace CodeIndex.Database; @@ -299,6 +300,41 @@ FROM fts_chunks throw CreateRawFtsSqliteSyntaxException(ex); } + var fallbackQueryEligible = IsLongFtsTokenFallbackQueryEligible( + normalizedQuery, + rawQuery, + exactSearch, + prefix); + var primaryResultsExistBeforeCursor = raw.Count == 0 && + cursor != null && + !hasCandidatePostProcessing && + fallbackQueryEligible && + HasPrimarySearchResultBeforeCursor(cmd); + if (raw.Count == 0 && + !hasCandidatePostProcessing && + !primaryResultsExistBeforeCursor && + resultRanking == SearchResultRanking.Default) + { + raw = SearchLongFtsTokenFallback( + normalizedQuery, + limit, + cursor?.Offset ?? 0, + lang, + rawQuery, + exactSearch, + prefix, + pathPatterns, + excludePathPatterns, + excludeTests, + since, + visibilityRank, + deduplicate, + deduplicate + ? SearchPrimaryMatchContext.Create(query, normalizedQuery, rawQuery, exactSearch, lang) + : null, + requiredPathPatterns); + } + var results = deduplicate ? DeduplicateOverlappingResults(raw, SearchPrimaryMatchContext.Create(query, normalizedQuery, rawQuery, exactSearch, lang)) : raw; if (hasContextRanking) results = RankCredentialContextSearchResults(results, query, normalizedQuery, rawQuery, exactSearch, lang); @@ -332,6 +368,265 @@ FROM fts_chunks return pagedResults; } + private List SearchLongFtsTokenFallback( + string normalizedQuery, + int limit, + int offset, + string? lang, + bool rawQuery, + bool exactSearch, + bool prefix, + IReadOnlyList? pathPatterns, + IReadOnlyList? excludePathPatterns, + bool excludeTests, + DateTime? since, + bool visibilityRank, + bool deduplicate, + SearchPrimaryMatchContext? matchContext, + IReadOnlyList? requiredPathPatterns = null) + { + if (limit <= 0) + return []; + + var results = new List(); + var matchedCount = 0; + var keptMatchLines = deduplicate + ? new Dictionary>(StringComparer.Ordinal) + : null; + var keptIntervals = deduplicate + ? new Dictionary(StringComparer.Ordinal) + : null; + foreach (var result in EnumerateLongFtsTokenFallback( + normalizedQuery, + lang, + rawQuery, + exactSearch, + prefix, + pathPatterns, + excludePathPatterns, + excludeTests, + since, + visibilityRank, + requiredPathPatterns)) + { + if (deduplicate && + !AddSearchResultDedupCoverage( + result, + matchContext!, + keptMatchLines!, + keptIntervals!)) + { + continue; + } + + matchedCount++; + if (matchedCount <= Math.Max(0, offset)) + continue; + + result.NextOffset = matchedCount; + results.Add(result); + if (results.Count >= limit) + break; + } + + return results; + } + + private IEnumerable EnumerateLongFtsTokenFallback( + string normalizedQuery, + string? lang, + bool rawQuery, + bool exactSearch, + bool prefix, + IReadOnlyList? pathPatterns, + IReadOnlyList? excludePathPatterns, + bool excludeTests, + DateTime? since, + bool visibilityRank, + IReadOnlyList? requiredPathPatterns = null) + { + var literalToken = normalizedQuery.Trim(); + if (!IsLongFtsTokenFallbackQueryEligible(normalizedQuery, rawQuery, exactSearch, prefix) || + !_hasIssuesTable || + !HasTable(DbContext.FtsChunksTrigramTableName) || + DbWriter.IsFtsBulkLoadMarkerSet(GetMetaString(DbWriter.FtsBulkLoadInProgressMetaKey)) || + !HasAllFtsChunksTrigramSyncTriggers()) + { + yield break; + } + + using var cmd = _conn.CreateCommand(); + var coverageTokens = GetSearchCoverageTokens(literalToken, rawQuery: false); + var sql = $@" + SELECT f.path, f.lang, c.start_line, c.end_line, c.content, + 0.0 AS rank, + {GetSearchVisibilitySql()} AS visibility, + c.id AS chunk_id, + long_token_issue.line AS long_token_line + FROM {DbContext.FtsChunksTrigramTableName} + JOIN chunks c ON {DbContext.FtsChunksTrigramTableName}.rowid = c.id + JOIN files f ON c.file_id = f.id{SearchSymbolMatchJoinsSql} + JOIN ( + SELECT file_id, MIN(line) AS line + FROM file_issues + WHERE kind = @longTokenIssueKind + GROUP BY file_id + ) long_token_issue ON long_token_issue.file_id = f.id + WHERE {DbContext.FtsChunksTrigramTableName} MATCH @trigramQuery + AND long_token_issue.line BETWEEN c.start_line AND c.end_line"; + if (lang != null) + sql += " AND f.lang = @lang"; + if (since != null && _fileColumns.Contains("modified")) + sql += " AND f.modified >= @since"; + AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); + AppendAdditionalPathIncludeFilters(ref sql, requiredPathPatterns, "requiredPathPattern"); + sql += $" ORDER BY {GetSearchOrderSql(coverageTokens.Count, exactLiteralBoost: false)}"; + + cmd.CommandText = sql; + SqliteCommandPolicy.AddText( + cmd, + "@trigramQuery", + "\"" + literalToken.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""); + SqliteCommandPolicy.AddText(cmd, "@longTokenIssueKind", "fts_token_too_long"); + SqliteCommandPolicy.Add(cmd, "@rankingQuery", literalToken); + SqliteCommandPolicy.Add(cmd, "@rankingQueryPrefix", $"{EscapeLikeQuery(literalToken)}%"); + SqliteCommandPolicy.Add(cmd, "@visibilityRank", visibilityRank ? 1 : 0); + AddSearchCoverageParameters(cmd, coverageTokens); + if (lang != null) + SqliteCommandPolicy.Add(cmd, "@lang", lang); + if (since != null && _fileColumns.Contains("modified")) + SqliteCommandPolicy.Add(cmd, "@since", since.Value); + AddPathFilterParameters(cmd, pathPatterns, excludePathPatterns); + AddPathIncludeFilterParameters(cmd, requiredPathPatterns, "requiredPathPattern"); + + using var reader = cmd.ExecuteTrackedReader(); + while (reader.TrackedRead()) + { + var startLine = reader.GetInt32(2); + var endLine = reader.GetInt32(3); + var content = reader.GetString(4); + var longTokenLine = reader.GetInt32(8); + if (!ContainsOversizeFtsTokenMatch(content, startLine, endLine, longTokenLine, literalToken)) + continue; + + yield return new SearchResult + { + Path = reader.GetString(0), + Lang = GetNullableString(reader, 1), + StartLine = startLine, + EndLine = endLine, + Content = content, + Score = reader.GetDouble(5), + Visibility = GetNullableString(reader, 6), + ChunkId = reader.GetInt64(7), + }; + } + } + + private static bool IsLongFtsTokenFallbackQueryEligible( + string normalizedQuery, + bool rawQuery, + bool exactSearch, + bool prefix) + { + var literalToken = normalizedQuery.Trim(); + return !rawQuery && + !exactSearch && + !prefix && + literalToken.Length >= 3 && + literalToken.All(character => character <= 0x7f && FileIndexer.IsLikelyUnicode61AsciiTokenChar(character)); + } + + private static bool HasPrimarySearchResultBeforeCursor(SqliteCommand command) + { + var limitParameter = command.Parameters["@limit"]; + var cursorParameter = command.Parameters["@cursorOffset"]; + var originalLimit = limitParameter.Value; + var originalOffset = cursorParameter.Value; + try + { + limitParameter.Value = 1; + cursorParameter.Value = 0; + using var reader = command.ExecuteTrackedReader(); + return reader.TrackedRead(); + } + finally + { + limitParameter.Value = originalLimit; + cursorParameter.Value = originalOffset; + } + } + + private bool HasAllFtsChunksTrigramSyncTriggers() + { + using var command = _conn.CreateCommand(); + command.CommandText = DbContext.CountFtsChunksTrigramSyncTriggersSql; + return SqliteCommandPolicy.ReadInt32Scalar( + command, + "search trigram FTS synchronization trigger count") == 3; + } + + private static bool ContainsOversizeFtsTokenMatch( + string content, + int chunkStartLine, + int chunkEndLine, + int issueLine, + string query) + { + if (issueLine < chunkStartLine || issueLine > chunkEndLine) + return false; + + var lineStart = 0; + for (var line = chunkStartLine; line < issueLine; line++) + { + var newline = content.IndexOf('\n', lineStart); + if (newline < 0) + return false; + lineStart = newline + 1; + } + + var newlineIndex = content.IndexOf('\n', lineStart); + var lineEnd = newlineIndex < 0 ? content.Length : newlineIndex; + return ContainsOversizeFtsTokenMatch(content.AsSpan(lineStart, lineEnd - lineStart), query); + } + + private static bool ContainsOversizeFtsTokenMatch(ReadOnlySpan line, string query) + { + var tokenStart = -1; + var tokenRuneCount = 0; + var offset = 0; + foreach (var rune in line.EnumerateRunes()) + { + if (FileIndexer.IsLikelyUnicode61TokenRune(rune)) + { + if (tokenStart < 0) + tokenStart = offset; + tokenRuneCount++; + } + else + { + if (OversizeTokenContainsQuery(line, query, tokenStart, offset, tokenRuneCount)) + return true; + tokenStart = -1; + tokenRuneCount = 0; + } + + offset += rune.Utf16SequenceLength; + } + + return OversizeTokenContainsQuery(line, query, tokenStart, offset, tokenRuneCount); + } + + private static bool OversizeTokenContainsQuery( + ReadOnlySpan line, + string query, + int tokenStart, + int tokenEnd, + int tokenRuneCount) + => tokenStart >= 0 && + tokenRuneCount > FtsUnicode61MaxTokenLength && + line[tokenStart..tokenEnd].IndexOf(query.AsSpan(), StringComparison.OrdinalIgnoreCase) >= 0; + private static List FilterSearchResultByTokenBoundary(SearchResult result, SearchMatchLineTerms terms) { if (string.IsNullOrWhiteSpace(terms.NormalizedQuery)) @@ -1043,6 +1338,35 @@ FROM fts_chunks throw CreateRawFtsSqliteSyntaxException(ex); } + if (count == 0) + { + foreach (var result in EnumerateLongFtsTokenFallback( + normalizedQuery, + lang, + rawQuery, + exact, + prefix, + pathPatterns, + excludePathPatterns, + excludeTests, + since, + visibilityRank)) + { + if (deduplicate && + !AddSearchResultDedupCoverage( + result, + matchContext!, + keptMatchLines, + keptIntervals)) + { + continue; + } + + count++; + keptFiles.Add(result.Path); + } + } + return new QueryCountResult(count, keptFiles.Count); } @@ -1152,6 +1476,36 @@ FROM fts_chunks throw CreateRawFtsSqliteSyntaxException(ex); } + if (countsByPath.Count == 0) + { + foreach (var result in EnumerateLongFtsTokenFallback( + normalizedQuery, + lang, + rawQuery, + exact, + prefix, + pathPatterns, + excludePathPatterns, + excludeTests, + since, + visibilityRank)) + { + if (deduplicate && + !AddSearchResultDedupCoverage( + result, + matchContext!, + keptMatchLines, + keptIntervals)) + { + continue; + } + + countsByPath[result.Path] = countsByPath.TryGetValue(result.Path, out var fallbackCount) + ? fallbackCount + 1 + : 1; + } + } + return countsByPath .Select(kv => new SearchFileCountResult(kv.Key, kv.Value)) .OrderByDescending(group => group.Count) diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.OversizeContentIssues.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.OversizeContentIssues.cs index ddbc59c2b..695fdfb09 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.OversizeContentIssues.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.OversizeContentIssues.cs @@ -50,7 +50,7 @@ private static void AddOversizeContentIssues( Path = relativePath, Kind = "fts_token_too_long", Line = longFtsTokenLine, - Message = $"Line {longFtsTokenLine} contains an FTS5 unicode61 token longer than {CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength} characters; that token is not searchable through FTS", + Message = $"Line {longFtsTokenLine} contains an FTS5 unicode61 token longer than {CodeIndex.Database.DbReader.FtsUnicode61MaxTokenLength} characters; unicode61 cannot index that token, but literal search can recover eligible ASCII substrings through the bounded long-token fallback", }); } } diff --git a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs index a13961d4c..44a9bfa7d 100644 --- a/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs +++ b/tests/CodeIndex.Tests/DbSearchReaderIssueTests.cs @@ -1,4 +1,5 @@ using CodeIndex.Database; +using CodeIndex.Indexer; using CodeIndex.Models; namespace CodeIndex.Tests; @@ -185,6 +186,183 @@ public void Search_LiteralQueryAtLengthLimitRuns_Issue3081() Assert.Equal("src/literal-length-limit.cs", result.Path); } + [Fact] + public void Search_RecoversAsciiSubstringInsideReportedOversizeFtsToken_Issue5080() + { + const string recoveredNeedle = "MiddleRecoveryNeedle"; + const string ordinarySubstring = "OrdinarySubstring"; + var longToken = new string('a', 520) + recoveredNeedle + new string('b', 520); + var content = $""; + var fileId = InsertIndexedFile("reports/minified.html", "html", content); + var issues = FileIndexer.ValidateContent( + "reports/minified.html", + System.Text.Encoding.UTF8.GetBytes(content), + content); + + Assert.Empty(_reader.Search( + recoveredNeedle, + pathPatterns: ["reports/minified.html"], + limit: 1)); + + _writer.InsertIssues(fileId, issues); + var secondLongToken = new string('c', 520) + recoveredNeedle + new string('d', 520); + var secondContent = $"const generated={secondLongToken};"; + var secondFileId = InsertIndexedFile("reports/minified-second.js", "javascript", secondContent); + _writer.InsertIssues( + secondFileId, + FileIndexer.ValidateContent( + "reports/minified-second.js", + System.Text.Encoding.UTF8.GetBytes(secondContent), + secondContent)); + + var results = _reader.Search( + recoveredNeedle, + pathPatterns: ["reports/minified.html"], + limit: 1); + var scopedCount = _reader.CountSearchResults( + recoveredNeedle, + pathPatterns: ["reports/minified.html"]); + var scopedCountsByFile = _reader.CountSearchResultsByFile( + recoveredNeedle, + pathPatterns: ["reports/minified.html"]); + var ordinarySubstringResults = _reader.Search( + ordinarySubstring, + pathPatterns: ["reports/minified.html"], + limit: 1); + var firstPage = _reader.Search(recoveredNeedle, limit: 1); + var secondPage = _reader.Search( + recoveredNeedle, + limit: 1, + cursor: new SearchCursor(0, 0, firstPage[0].NextOffset)); + var totalCount = _reader.CountSearchResults(recoveredNeedle); + var countsByFile = _reader.CountSearchResultsByFile(recoveredNeedle); + + var result = Assert.Single(results); + Assert.Equal("reports/minified.html", result.Path); + Assert.Contains(recoveredNeedle, result.Content, StringComparison.Ordinal); + Assert.Equal(new QueryCountResult(1, 1), scopedCount); + var fileCount = Assert.Single(scopedCountsByFile); + Assert.Equal("reports/minified.html", fileCount.Path); + Assert.Equal(1, fileCount.Count); + Assert.Empty(ordinarySubstringResults); + Assert.Single(firstPage); + Assert.Single(secondPage); + Assert.NotEqual(firstPage[0].Path, secondPage[0].Path); + Assert.Equal(new QueryCountResult(2, 2), totalCount); + Assert.Equal(2, countsByFile.Count); + Assert.All(countsByFile, countByFile => Assert.Equal(1, countByFile.Count)); + + using var dropTrigger = _db.Connection.CreateCommand(); + dropTrigger.CommandText = DbContext.DropFtsChunksTrigramInsertTriggerSql; + dropTrigger.ExecuteNonQuery(); + Assert.Empty(_reader.Search( + recoveredNeedle, + pathPatterns: ["reports/minified.html"], + limit: 1)); + } + + [Fact] + public void Search_CursorPastOrdinaryMatchDoesNotSwitchToLongTokenFallback_Issue5080() + { + const string needle = "PrimaryMatchNeedle"; + var longContent = new string('a', 520) + needle + new string('b', 520); + var longFileId = InsertIndexedFile("reports/long-primary-control.js", "javascript", longContent); + _writer.InsertIssues( + longFileId, + FileIndexer.ValidateContent( + "reports/long-primary-control.js", + System.Text.Encoding.UTF8.GetBytes(longContent), + longContent)); + InsertIndexedFile("src/primary-control.js", "javascript", $"const value = {needle};"); + + var firstPage = _reader.Search(needle, limit: 1); + var firstResult = Assert.Single(firstPage); + Assert.Equal("src/primary-control.js", firstResult.Path); + + var beyondPrimaryResults = _reader.Search( + needle, + limit: 1, + cursor: new SearchCursor(0, 0, firstResult.NextOffset)); + + Assert.Empty(beyondPrimaryResults); + } + + [Fact] + public void Search_PostProcessedCursorMissDoesNotProbeOffsetParameters_Issue5080() + { + var cursor = new SearchCursor(0, 0, 1); + + var guardedResults = _reader.Search( + "NoGuardedCursorMatch", + limit: 1, + cursor: cursor, + guardFilters: + [ + new SearchGuardFilter( + SearchGuardRole.Require, + SearchGuardDirection.Before, + "guard"), + ]); + var contextRankedResults = _reader.Search( + "NoContextCursorMatch", + limit: 1, + cursor: cursor, + resultRanking: SearchResultRanking.CredentialContext); + + Assert.Empty(guardedResults); + Assert.Empty(contextRankedResults); + } + + [Fact] + public void Search_DeduplicatesOverlappingLongTokenChunksBeforeCursorPaging_Issue5080() + { + const string needle = "OverlapRecoveryNeedle"; + var content = new string('a', 520) + needle + new string('b', 520); + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "reports/overlap-minified.js", + Lang = "javascript", + Size = content.Length, + Lines = 1, + Modified = new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc), + }); + _writer.InsertChunks( + [ + new ChunkRecord + { + FileId = fileId, + ChunkIndex = 0, + StartLine = 1, + EndLine = 1, + Content = content, + }, + new ChunkRecord + { + FileId = fileId, + ChunkIndex = 1, + StartLine = 1, + EndLine = 1, + Content = content, + }, + ]); + _writer.InsertIssues( + fileId, + FileIndexer.ValidateContent( + "reports/overlap-minified.js", + System.Text.Encoding.UTF8.GetBytes(content), + content)); + + var firstPage = _reader.Search(needle, limit: 1); + var firstResult = Assert.Single(firstPage); + var secondPage = _reader.Search( + needle, + limit: 1, + cursor: new SearchCursor(0, 0, firstResult.NextOffset)); + + Assert.Empty(secondPage); + Assert.Equal(new QueryCountResult(1, 1), _reader.CountSearchResults(needle)); + } + [Fact] public void Search_LiteralQueryOverLengthLimitThrows_Issue3081() { @@ -275,7 +453,7 @@ public void BuildPathLikePattern_RejectsWildcardHeavyPathPattern_Issue3794() Assert.Contains(DbReader.MaxPathLikePatternWildcards.ToString(), ex.Message, StringComparison.Ordinal); } - private void InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) + private long InsertIndexedFile(string path, string lang, string content, DateTime? modified = null) { var normalized = content.Replace("\r\n", "\n"); var lines = normalized.Split('\n'); @@ -296,6 +474,7 @@ private void InsertIndexedFile(string path, string lang, string content, DateTim EndLine = lines.Length, Content = normalized, }]); + return fileId; } private static string BuildLiteralTermQuery(int termCount) diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index f9b93dcb1..9c2070086 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -8011,7 +8011,7 @@ public void ValidateContent_OversizeFtsToken_EmitsFtsTokenTooLongIssue() var issue = Assert.Single(issues, i => i.Kind == "fts_token_too_long"); Assert.Equal(2, issue.Line); - Assert.Contains("not searchable through FTS", issue.Message); + Assert.Contains("bounded long-token fallback", issue.Message); } [Fact] @@ -8025,7 +8025,7 @@ public void ValidateContent_OversizeUnicodeFtsToken_EmitsFtsTokenTooLongIssue() var issue = Assert.Single(issues, i => i.Kind == "fts_token_too_long"); Assert.Equal(2, issue.Line); - Assert.Contains("not searchable through FTS", issue.Message); + Assert.Contains("bounded long-token fallback", issue.Message); } [Fact] diff --git a/tests/CodeIndex.Tests/NormalizedContentFactsTests.cs b/tests/CodeIndex.Tests/NormalizedContentFactsTests.cs index 7f9828a0a..d483d77cf 100644 --- a/tests/CodeIndex.Tests/NormalizedContentFactsTests.cs +++ b/tests/CodeIndex.Tests/NormalizedContentFactsTests.cs @@ -583,7 +583,7 @@ private static IEnumerable BuildExpectedValidationIssues(Normal yield return new IssueProjection( "fts_token_too_long", facts.FirstOversizeFtsTokenLine, - $"Line {facts.FirstOversizeFtsTokenLine} contains an FTS5 unicode61 token longer than {DbReader.FtsUnicode61MaxTokenLength} characters; that token is not searchable through FTS", + $"Line {facts.FirstOversizeFtsTokenLine} contains an FTS5 unicode61 token longer than {DbReader.FtsUnicode61MaxTokenLength} characters; unicode61 cannot index that token, but literal search can recover eligible ASCII substrings through the bounded long-token fallback", null, null); }