From 3b1862805f5bb9d2d5f07b28f343ab5a520be9e6 Mon Sep 17 00:00:00 2001 From: Ian Oberst Date: Thu, 13 Aug 2026 15:27:53 -0700 Subject: [PATCH 1/2] test(mysql): cover spaced qualified identifiers Co-authored-by: Codex Ai-assisted: true --- normalizer_test.go | 14 +++++ sqllexer_test.go | 124 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/normalizer_test.go b/normalizer_test.go index d6d842a..ecb1cfb 100644 --- a/normalizer_test.go +++ b/normalizer_test.go @@ -1160,6 +1160,20 @@ func TestNormalizeDeobfuscatedSQL(t *testing.T) { } } +func TestNormalizeMySQLPerformanceSchemaQualifiedIdentifiers(t *testing.T) { + input := "SELECT `performance_schema` . `events_statements_summary_by_digest` . `DIGEST_TEXT` " + + "FROM `performance_schema` . `events_statements_summary_by_digest` " + + "WHERE `SCHEMA_NAME` = ?" + expected := "SELECT performance_schema.events_statements_summary_by_digest.DIGEST_TEXT " + + "FROM performance_schema.events_statements_summary_by_digest WHERE SCHEMA_NAME = ?" + + normalizer := NewNormalizer(WithCollectTables(true)) + got, metadata, err := normalizer.Normalize(input, WithDBMS(DBMSMySQL)) + assert.NoError(t, err) + assert.Equal(t, expected, got) + assert.Equal(t, []string{"performance_schema.events_statements_summary_by_digest"}, metadata.Tables) +} + func TestGroupObfuscatedValues(t *testing.T) { tests := []struct { input string diff --git a/sqllexer_test.go b/sqllexer_test.go index b950a01..be43e29 100644 --- a/sqllexer_test.go +++ b/sqllexer_test.go @@ -1170,6 +1170,130 @@ here */`, } } +func TestLexerMySQLQualifiedIdentifiersWithSpaces(t *testing.T) { + tests := []struct { + name string + input string + expectedType TokenType + expectedValue string + expectedNext *TokenSpec + expectedDigits bool + }{ + { + name: "unquoted", + input: "mydb . table", + expectedType: IDENT, + expectedValue: "mydb.table", + }, + { + name: "quoted and partially quoted", + input: "`mydb` . table . `column`", + expectedType: QUOTED_IDENT, + expectedValue: "`mydb`.table.`column`", + }, + { + name: "keyword component", + input: "mydb . select", + expectedType: IDENT, + expectedValue: "mydb.select", + }, + { + name: "unicode components", + input: "édb . 表 . column", + expectedType: IDENT, + expectedValue: "édb.表.column", + }, + { + name: "space after consumed dot", + input: "mydb. table", + expectedType: IDENT, + expectedValue: "mydb.table", + }, + { + name: "digit metadata", + input: "mydb . tenant42", + expectedType: IDENT, + expectedValue: "mydb.tenant42", + expectedDigits: true, + }, + { + name: "table before column list", + input: "mydb . table(id)", + expectedType: IDENT, + expectedValue: "mydb.table", + expectedNext: &TokenSpec{Type: PUNCTUATION, Value: "("}, + }, + { + name: "qualified wildcard", + input: "mydb . table . *", + expectedType: IDENT, + expectedValue: "mydb.table.", + expectedNext: &TokenSpec{Type: WILDCARD, Value: "*"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lexer := New(tt.input, WithDBMS(DBMSMySQL)) + token := lexer.Scan() + if token.Type != tt.expectedType { + t.Fatalf("got token type %v, want %v", token.Type, tt.expectedType) + } + if token.Value != tt.expectedValue { + t.Fatalf("got token value %q, want %q", token.Value, tt.expectedValue) + } + if token.hasDigits != tt.expectedDigits { + t.Fatalf("got hasDigits %v, want %v", token.hasDigits, tt.expectedDigits) + } + + if tt.expectedNext == nil { + if next := lexer.Scan(); next.Type != EOF { + t.Fatalf("got trailing token %#v, want EOF", next) + } + return + } + + next := lexer.Scan() + if next.Type != tt.expectedNext.Type || next.Value != tt.expectedNext.Value { + t.Fatalf("got next token %#v, want %#v", next, *tt.expectedNext) + } + }) + } +} + +func TestLexerQualifiedIdentifierSpacingIsMySQLSpecific(t *testing.T) { + lexer := New("mydb . table") + expected := []TokenSpec{ + {Type: IDENT, Value: "mydb"}, + {Type: SPACE, Value: " "}, + {Type: PUNCTUATION, Value: "."}, + {Type: SPACE, Value: " "}, + {Type: KEYWORD, Value: "table"}, + } + + for i, want := range expected { + got := lexer.Scan() + if got.Type != want.Type || got.Value != want.Value { + t.Fatalf("token[%d] got %#v, want %#v", i, got, want) + } + } +} + +func TestLexerMySQLAdjacentQuotedRoutineKeepsTokenType(t *testing.T) { + lexer := New("`database`.`func42`()", WithDBMS(DBMSMySQL)) + token := lexer.Scan() + + if token.Type != QUOTED_IDENT { + t.Fatalf("got token type %v, want %v", token.Type, QUOTED_IDENT) + } + if token.Value != "`database`.`func42`" { + t.Fatalf("got token value %q, want %q", token.Value, "`database`.`func42`") + } + if !token.hasDigits { + t.Fatal("expected quoted routine token to retain its digit metadata") + } +} + func TestLexerIdentifierWithDigits(t *testing.T) { tests := []struct { input string From a04d5b3894587d0bb62d560237c50e68540fb144 Mon Sep 17 00:00:00 2001 From: Ian Oberst Date: Thu, 13 Aug 2026 15:33:07 -0700 Subject: [PATCH 2/2] fix(mysql): join spaced qualified identifiers Co-authored-by: Codex Ai-assisted: true --- normalizer_test.go | 17 +++++ sqllexer.go | 161 +++++++++++++++++++++++++++++++++++++++++ sqllexer_bench_test.go | 32 ++++++++ sqllexer_test.go | 21 +++++- 4 files changed, 229 insertions(+), 2 deletions(-) diff --git a/normalizer_test.go b/normalizer_test.go index ecb1cfb..955ae8a 100644 --- a/normalizer_test.go +++ b/normalizer_test.go @@ -1174,6 +1174,23 @@ func TestNormalizeMySQLPerformanceSchemaQualifiedIdentifiers(t *testing.T) { assert.Equal(t, []string{"performance_schema.events_statements_summary_by_digest"}, metadata.Tables) } +func TestNormalizeMySQLSpacedQualifiedNameBeforeParentheses(t *testing.T) { + normalizer := NewNormalizer(WithRemoveSpaceBetweenParentheses(true)) + tests := []struct { + spaced string + adjacent string + }{ + {spaced: "SELECT mydb . func(1)", adjacent: "SELECT mydb.func(1)"}, + {spaced: "INSERT INTO mydb . table(id) VALUES (1)", adjacent: "INSERT INTO mydb.table(id) VALUES (1)"}, + } + + for _, tt := range tests { + got, _, err := normalizer.Normalize(tt.spaced, WithDBMS(DBMSMySQL)) + assert.NoError(t, err) + assert.Equal(t, tt.adjacent, got) + } +} + func TestGroupObfuscatedValues(t *testing.T) { tests := []struct { input string diff --git a/sqllexer.go b/sqllexer.go index f4bd298..c34c3fa 100644 --- a/sqllexer.go +++ b/sqllexer.go @@ -1,6 +1,7 @@ package sqllexer import ( + "strings" "unicode/utf8" ) @@ -363,6 +364,9 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { if s.start == s.cursor { return s.scanUnknown() } + if s.config.DBMS == DBMSMySQL { + return s.checkForSpacesInIdentifier(IDENT) + } return s.emit(IDENT) } @@ -418,10 +422,17 @@ func (s *Lexer) scanIdentifier(ch rune) *Token { if ch == '(' { return s.emit(FUNCTION) } + if s.config.DBMS == DBMSMySQL { + return s.checkForSpacesInIdentifier(IDENT) + } return s.emit(IDENT) } func (s *Lexer) scanDoubleQuotedIdentifier(delimiter rune) *Token { + return s.scanDoubleQuotedIdentifierComponent(delimiter, true) +} + +func (s *Lexer) scanDoubleQuotedIdentifierComponent(delimiter rune, joinQualified bool) *Token { closingDelimiter := delimiter if delimiter == '[' { closingDelimiter = ']' @@ -471,9 +482,159 @@ func (s *Lexer) scanDoubleQuotedIdentifier(delimiter rune) *Token { ch = s.nextBy(size) } s.next() // consume the closing quote (ASCII) + if joinQualified && s.config.DBMS == DBMSMySQL { + return s.checkForSpacesInIdentifier(QUOTED_IDENT) + } return s.emit(QUOTED_IDENT) } +// scanMySQLQualifiedIdentifierComponent scans an unquoted identifier after a +// qualifier dot. Keywords are valid identifier components in this position, +// so this deliberately skips the keyword trie used by scanIdentifier. +func (s *Lexer) scanMySQLQualifiedIdentifierComponent(ch rune) *Token { + s.start = s.cursor + for isIdentifier(ch) { + s.hasDigits = s.hasDigits || isDigit(ch) + ch = s.nextBy(utf8.RuneLen(ch)) + } + if ch == '(' { + return s.emit(FUNCTION) + } + return s.emit(IDENT) +} + +// checkForSpacesInIdentifier joins MySQL qualified identifiers whose +// components have been separated by whitespace around a dot. MySQL emits this +// form in normalized performance_schema statements, for example: +// `schema` . `table` . `column`. +func (s *Lexer) checkForSpacesInIdentifier(tokenType TokenType) *Token { + token := s.emit(tokenType) + identifierStart := s.cursor - len(token.Value) + identifierEnd := s.cursor + resultType := token.Type + hasDigits := token.hasDigits + hasQuotes := token.hasQuotes + hasSeparatorSpaces := false + joined := false + + for { + ch := s.peek() + componentEnd := s.cursor + separatorHasSpaces := false + + // Unquoted identifiers already consume an adjacent dot. Continue from + // that dot as well as from a dot separated by whitespace. + dotConsumed := identifierEnd > identifierStart && s.src[identifierEnd-1] == '.' + if dotConsumed { + for isSpace(ch) { + separatorHasSpaces = true + ch = s.next() + } + } else { + for isSpace(ch) { + separatorHasSpaces = true + ch = s.next() + } + if ch != '.' { + s.cursor = componentEnd + break + } + ch = s.next() + for isSpace(ch) { + separatorHasSpaces = true + ch = s.next() + } + } + + // Match the characters scanMySQLQualifiedIdentifierComponent can + // actually consume. isAlphaNumeric also accepts Unicode numbers, while + // isIdentifier does not; accepting one here would make no progress. + if !isLetter(ch) && !isDigit(ch) && ch != '$' && ch != '`' { + if ch == '*' { + // Preserve the qualifier dot on the identifier and let the next + // scan emit the wildcard, matching the adjacent-token behavior. + joined = true + hasSeparatorSpaces = hasSeparatorSpaces || separatorHasSpaces + identifierEnd = s.cursor + break + } + s.cursor = componentEnd + break + } + + if ch == '`' { + token = s.scanDoubleQuotedIdentifierComponent('`', false) // sadscan:disable np.hashicorp.1 -- lexer token, not a Vault token + } else { + token = s.scanMySQLQualifiedIdentifierComponent(ch) // sadscan:disable np.hashicorp.1 -- lexer token, not a Vault token + } + + joined = true + hasSeparatorSpaces = hasSeparatorSpaces || separatorHasSpaces + identifierEnd = s.cursor + hasDigits = hasDigits || token.hasDigits + hasQuotes = hasQuotes || token.hasQuotes + + if token.Type == ERROR { + resultType = ERROR + break + } + if token.Type == FUNCTION { + // Match the existing token type for an adjacent unquoted name such + // as db.func(. Quoted or partially quoted names retain their quoted + // identifier type; recognizing those as routines needs SQL context. + if resultType != QUOTED_IDENT { + resultType = FUNCTION + } + break + } + if resultType == QUOTED_IDENT || token.Type == QUOTED_IDENT { + resultType = QUOTED_IDENT + } + } + + if !joined { + return token + } + + token.Type = resultType + token.hasDigits = hasDigits + token.hasQuotes = hasQuotes + token.isSimpleIdentifier = false + if !hasSeparatorSpaces { + token.Value = s.src[identifierStart:identifierEnd] + return token + } + token.Value = compactMySQLQualifiedIdentifier(s.src[identifierStart:identifierEnd]) + return token +} + +func compactMySQLQualifiedIdentifier(value string) string { + compactLength := len(value) + inQuotes := false + for i := 0; i < len(value); i++ { + ch := value[i] + if ch == '`' { + inQuotes = !inQuotes + } else if !inQuotes && isSpace(rune(ch)) { + compactLength-- + } + } + + var compact strings.Builder + compact.Grow(compactLength) + inQuotes = false + for i := 0; i < len(value); i++ { + ch := value[i] + if ch == '`' { + inQuotes = !inQuotes + } else if !inQuotes && isSpace(rune(ch)) { + continue + } + compact.WriteByte(ch) + } + return compact.String() +} + func (s *Lexer) scanWhitespace() *Token { // scan whitespace, tab, newline, carriage return s.start = s.cursor diff --git a/sqllexer_bench_test.go b/sqllexer_bench_test.go index 0132a3e..6bc72b2 100644 --- a/sqllexer_bench_test.go +++ b/sqllexer_bench_test.go @@ -3,6 +3,7 @@ package sqllexer import ( "fmt" "strconv" + "strings" "testing" ) @@ -105,3 +106,34 @@ func BenchmarkLexer(b *testing.B) { }) } } + +func BenchmarkLexerMySQLQualifiedIdentifier(b *testing.B) { + for _, components := range []int{2, 32, 256} { + benchmarks := []struct { + name string + input string + }{ + { + name: "spaced", + input: "t" + strings.Repeat(" . t", components-1), + }, + { + name: "adjacent", + input: "t" + strings.Repeat(".t", components-1), + }, + } + + for _, benchmark := range benchmarks { + b.Run(benchmark.name+"/"+strconv.Itoa(components), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + lexer := New(benchmark.input, WithDBMS(DBMSMySQL)) + token := lexer.Scan() + if token.Type != IDENT || len(token.Value) != components*2-1 { + b.Fatalf("unexpected token: %#v", token) + } + } + }) + } + } +} diff --git a/sqllexer_test.go b/sqllexer_test.go index be43e29..d351bc2 100644 --- a/sqllexer_test.go +++ b/sqllexer_test.go @@ -1217,9 +1217,9 @@ func TestLexerMySQLQualifiedIdentifiersWithSpaces(t *testing.T) { expectedDigits: true, }, { - name: "table before column list", + name: "unquoted name before parentheses", input: "mydb . table(id)", - expectedType: IDENT, + expectedType: FUNCTION, expectedValue: "mydb.table", expectedNext: &TokenSpec{Type: PUNCTUATION, Value: "("}, }, @@ -1294,6 +1294,23 @@ func TestLexerMySQLAdjacentQuotedRoutineKeepsTokenType(t *testing.T) { } } +func TestLexerMySQLQualifiedIdentifierRequiresScannableComponent(t *testing.T) { + lexer := New("mydb .߂", WithDBMS(DBMSMySQL)) + expected := []TokenSpec{ + {Type: IDENT, Value: "mydb"}, + {Type: SPACE, Value: " "}, + {Type: PUNCTUATION, Value: "."}, + {Type: UNKNOWN, Value: "߂"}, + } + + for i, want := range expected { + got := lexer.Scan() + if got.Type != want.Type || got.Value != want.Value { + t.Fatalf("token[%d] got %#v, want %#v", i, got, want) + } + } +} + func TestLexerIdentifierWithDigits(t *testing.T) { tests := []struct { input string