diff --git a/Cargo.lock b/Cargo.lock index f0f6bd5..5413dea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2106,6 +2106,7 @@ dependencies = [ "axum", "bytes", "futures", + "rusty-filter", "rusty-ivyml", "rusty-macros", "serde", @@ -2143,6 +2144,14 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rusty-filter" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "rusty-ivyml" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index dd8d5b1..7ee0431 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rusty", "rusty-macros", "rusty-ivyml", "rusty-server", "rusty-docs", "rusty-desktop"] +members = ["rusty", "rusty-filter", "rusty-macros", "rusty-ivyml", "rusty-server", "rusty-docs", "rusty-desktop"] resolver = "2" [workspace.package] diff --git a/rusty-docs/docs/02_concepts/07_filters.md b/rusty-docs/docs/02_concepts/07_filters.md new file mode 100644 index 0000000..867847c --- /dev/null +++ b/rusty-docs/docs/02_concepts/07_filters.md @@ -0,0 +1,179 @@ +## Filters + +`rusty_filter` parses the filter query language Ivy's filter editor uses, into an +AST you can validate against a column schema, evaluate over rows, and print back +to a canonical string. It is what makes a filter typed in one place mean the same +thing in the other. + +```rust +use rusty::prelude::*; + +let columns = vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), +]; +let result = parse_query("[age] > 30 AND [name] starts with \"A\"", &columns); +let filter = result.filters.expect("the query is valid"); + +let rows = vec![ + serde_json::json!({"age": 41, "name": "Ada"}), + serde_json::json!({"age": 41, "name": "Bob"}), + serde_json::json!({"age": 12, "name": "Ann"}), +]; +assert_eq!(retain_matching(&filter, rows, &columns).len(), 1); +``` + +`parse_query` returns a `ParseResult` carrying **either** `filters` or `errors`, +never both. Empty and whitespace-only input is valid and matches every row, so a +cleared filter box needs no special case. + +### Column references + +A column is named in square brackets: `[age]`. Whatever is between the brackets +is the name, verbatim — spaces and unicode included, so `[first name]` works and +`[ age ]` is a different column from `[age]`. Names are matched case-sensitively; +keywords are not. `[]` is a syntax error, but `[ ]` is the column named `" "`. + +### Operators + +Every operator has a symbolic and a spelled-out form, and the spelled-out forms +are case-insensitive. Both parse to one `FilterFunction`: + +| Function | Spellings | +|----------|-----------| +| `Equals` | `=`, `==`, `equals` | +| `Equals` negated | `!=`, `not equals`, `not equal` | +| `GreaterThan` | `>`, `greater than` | +| `GreaterThanOrEqual` | `>=`, `greater than or equal` | +| `LessThan` | `<`, `less than` | +| `LessThanOrEqual` | `<=`, `less than or equal` | +| `Contains` | `contains`, `not contains` | +| `StartsWith` | `starts with`, `not starts with` | +| `EndsWith` | `ends with`, `not ends with` | +| `IsBlank` | `is blank` | +| `IsNotBlank` | `is not blank` | + +There is no `NotEquals` function: `!=` is `Equals` with `negate: true`. The same +goes for the negated text operators. + +Two spellings that look like they should work and do not: bare `equal` is a +syntax error where `equals` is fine, though `not equal` is accepted; and the +`or equal` tail of `>=` and `<=` is singular, so `greater than or equals` is +rejected. + +### Which operators a column type allows + +`ColumnType` has five variants, and validation rejects a mismatch with +`Operator 'x' is not compatible with type 'y'`: + +| Column type | Allowed | +|-------------|---------| +| `String` | `equals`, `contains`, `starts with`, `ends with`, `is blank`, `is not blank` | +| `Number` | `equals`, `>`, `>=`, `<`, `<=` | +| `Date` | `equals`, `>`, `>=`, `<`, `<=`, `is blank`, `is not blank` | +| `Boolean` | `equals` | +| `Enum` | `equals`, `is blank`, `is not blank` | + +`ColumnType::normalize` maps a backend type name onto one of the five: +`INT32`, `INT64`, `DOUBLE`, `DECIMAL` and `NUMBER` become `Number`; `TEXT`, +`STRING` and `ICON` become `String`; `DATE` and `DATETIME` become `Date`; and any +name it does not recognize becomes `String`. + +### Literals + +| Kind | Examples | Notes | +|------|----------|-------| +| Number | `42`, `-5`, `1.5`, `007` | One JavaScript number type, so `007` is `7` and `1.000` is `1`. `1e5` is a lexer error. | +| String | `"Ada"`, `"a\"b"`, `"a\\b"` | Double quotes only. `\"`, `\'` and `\\` are the only escapes; anything else keeps its backslash, so `\t` stays two characters rather than becoming a tab. A raw newline inside a string is an error, but a raw tab is allowed. | +| Boolean | `true`, `false` | Case-insensitive. | +| Date | `"2024-01-01"`, `"2024-01-01T10:20:30.123Z"` | A quoted string on a `Date` column. Validation is shape-only, so `"2024-13-99"` passes it and then matches nothing. `"2024-1-1"` is rejected — pad to two digits. | + +The value type has to match the column, or validation reports +`Expected number for column 'age', got string`. + +### Combining and precedence + +`AND` binds tighter than `OR`, and both are case-insensitive. Parentheses +override precedence and are **never collapsed**: `(([age] > 1))` keeps both +group levels in the AST, because the frontend renders those groups. + +`NOT` negates the filter that follows and toggles rather than accumulates, so +`not not [age] > 1` is the same filter as `[age] > 1`. + +```rust +"[age] > 1 AND [name] = \"a\" OR [active] = true" +// parses as: ([age] > 1 AND [name] = "a") OR [active] = true +``` + +The AST reflects that: the root is an `OR` group whose first arm is a nested +`AND` group. Same-operator chains stay flat — a three-arm `AND` is one group +with three filters, not two nested ones. + +### Evaluating + +| Function | Use | +|----------|-----| +| `evaluate(&filter, &row, &columns)` | Does one row match? | +| `retain_matching(&filter, rows, &columns)` | Keep the matching rows, in order. | +| `count_matches(&filter, &rows, &columns)` | Count without allocating. | + +Evaluation is strict about types: `equals` on a `String` column does not match +the number `1` against `"1"`. `contains`, `starts with` and `ends with` are +case-sensitive. A `null` or absent column value fails every operator except +`is blank`. An empty `AND` group matches everything — and so does an empty `OR` +group, which is the reference's behaviour rather than the `false` you might +expect. + +### Canonical strings and cache keys + +`to_query_string` prints an AST back to a query, and `canonical_key` is that +same function under the name that says what it is for: two equivalent filters +produce one string, so they can share one cache entry. + +```rust +use rusty::core::QueryService; + +let filter = parse_query_unchecked("[age] greater than 30").unwrap(); +assert_eq!(QueryService::filtered_key("people", Some(&filter)), "people?[age] > 30"); +``` + +The canonical spelling is not simply the input echoed back. It is symbolic for +the orderings and spelled out for equality — `[age] greater than 30` prints as +`[age] > 30` while `[age] = 1` prints as `[age] equals 1` — because that is what +the frontend's `formatQuery` does, and the two sides have to agree. + +Printing is idempotent from the second pass onwards, but the first pass can +reshape the AST: a negated leaf prints as `NOT (...)`, and re-parsing reads those +parentheses as a group, moving the negation onto a wrapping group filter. The +filter still means the same thing, and the reference does the same, so compare +canonical strings rather than ASTs when you need to know whether two filters +match. + +### Compatibility with the frontend editor + +The AST serializes to exactly the JSON `filter-query-editor` produces — +`{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}}]}}` +— key presence included, which is finer than it sounds. A comparison omits the +`negate` key altogether; a text operation emits `negate: false`. Behaviour was +matched against version 2.2.0 over roughly a hundred inputs, and +`rusty-filter/tests/frontend_ast_compat.rs` pins the shape against captured +bundle output. + +**The browser's filter box does not reach this crate.** The rendered `DataTable` +sends filter queries to a gRPC `DataTableService` that Rusty does not implement, +so typing in the filter box goes nowhere. What works today is Rust-side +filtering you call yourself, via [`DataTable::apply_filter`](../03_widgets/11_data_table.md) +or the functions above. This crate is what a future `DataTableService` would +parse with. + +### Not implemented + +- The gRPC `DataTableService` (`Query`, `ParseFilter`, `Distinct`). +- `parseInvalidQuery` and its LLM-based query repair. +- Sorting, aggregations, pagination and column selection — the rest of Ivy's + `DataTableQuery`. +- Multiple errors for one input. ANTLR recovers from a syntax error and can + report a cascade; this parser reports the first and stops. Semantic errors are + reported in full, one per offending condition. +- Error spans are byte offsets, not UTF-16 code units. They agree for ASCII and + part company once a multi-byte character appears before the error. diff --git a/rusty-docs/docs/03_widgets/11_data_table.md b/rusty-docs/docs/03_widgets/11_data_table.md index 1ba3d1c..3cee191 100644 --- a/rusty-docs/docs/03_widgets/11_data_table.md +++ b/rusty-docs/docs/03_widgets/11_data_table.md @@ -101,9 +101,51 @@ DataTable::new(vec![ .into() ``` +### Server-side filtering + +`apply_filter` runs a query in the same grammar Ivy's filter editor uses and +returns a table holding only the matching rows. See +[Filters](../02_concepts/07_filters.md) for the grammar itself. + +```rust +let table = DataTable::new(vec![ + DataTableColumn::new("name", "Name", ColType::Text), + DataTableColumn::new("age", "Age", ColType::Number), +]) +.rows(vec![ + json!({"name": "Alice", "age": 30}), + json!({"name": "Bob", "age": 25}), +]); + +let filtered = table.apply_filter("[age] > 28").expect("valid query"); +assert_eq!(filtered.rows.len(), 1); +``` + +| Method | Returns | Description | +|--------|---------|-------------| +| `.filter_columns()` | `Vec` | The columns a query may name, as the grammar sees them | +| `.apply_filter(q)` | `Result>` | Keep the rows matching `q`; `Err` carries the parse and validation errors | + +A column is offered to `filter_columns` only when it is `filterable` and not +`hidden`, which is the same test the frontend applies before handing columns to +its editor. Naming an excluded column therefore gets +`Column 'x' does not exist` on both sides rather than working on one of them. +Each `ColType` maps onto one of the grammar's five types: `Number` to `number`, +`Boolean` to `boolean`, `Date` and `DateTime` to `date`, and `Text`, `Icon`, +`Labels` and `Link` to `string`. + +An empty or whitespace-only query is valid and keeps every row, so clearing a +filter needs no special case. + ### Limitations Rows travel inline in the widget JSON, exactly as `Table` does: the whole set is serialized on every build. Ivy's `DataTableConnection` — the server-side query pipeline that pages, sorts and filters large datasets on demand — is not ported, -so paginate or pre-filter in your own code before handing rows to the widget. +so paginate in your own code before handing rows to the widget. + +Filtering is available, but only when **your code** calls `apply_filter`. The +filter box in the rendered table sends its query to a gRPC `DataTableService` +that Rusty does not implement, so typing there still goes nowhere; the same +applies to sorting and to the search box. Treat `allow_filtering`, +`allow_sorting` and `show_search` as frontend chrome until that service exists. diff --git a/rusty-docs/docs/04_api_reference/01_prelude.md b/rusty-docs/docs/04_api_reference/01_prelude.md index ea0704d..fbca29f 100644 --- a/rusty-docs/docs/04_api_reference/01_prelude.md +++ b/rusty-docs/docs/04_api_reference/01_prelude.md @@ -93,3 +93,31 @@ Note the argument order: wherever a hook takes dependencies, they come **before* | Type | Description | |------|-------------| | `RustyServer` | WebSocket server — `new(port, factory).serve().await` | + +### Filters + +Re-exported from `rusty_filter`. See [Filters](../02_concepts/07_filters.md). + +| Item | Description | +|------|-------------| +| `parse_query(q, &columns)` | Parse and validate, returning `ParseResult` | +| `parse_query_unchecked(q)` | Parse without a column schema | +| `evaluate(&filter, &row, &columns)` | Does one row match? | +| `retain_matching(&filter, rows, &columns)` | Keep the matching rows | +| `count_matches(&filter, &rows, &columns)` | Count the matching rows | +| `validate_filter_group(&filter, &columns)` | Semantic errors only | +| `to_query_string(&filter)` | Print an AST back to a query | +| `canonical_key(&filter)` | The same string, named for cache-key use | +| `ColumnDef` | One filterable column — `new(name, ColumnType)` | +| `ColumnType` | `String`, `Number`, `Boolean`, `Date`, `Enum` | +| `FilterGroup` | `op` plus a list of `Filter` | +| `Filter` | A `Condition` or a nested `FilterGroup`, optionally negated | +| `Condition` | `column`, `function`, `args` | +| `FilterFunction` | `Equals`, `GreaterThan`, `Contains`, `IsBlank`, … | +| `LogicalOp` | `And`, `Or` | +| `ParseResult` | `filters` **or** `errors`, never both | +| `ParseError` | `message`, `start`, `end`, `severity` | +| `ErrorSeverity` | `Error`, `Warning` | + +The crate itself is re-exported too, so `rusty_filter::lexer` and the other +modules are reachable without adding a second dependency. diff --git a/rusty-filter/Cargo.toml b/rusty-filter/Cargo.toml new file mode 100644 index 0000000..4215181 --- /dev/null +++ b/rusty-filter/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rusty-filter" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Filter query grammar parser and AST for Rusty-Framework" + +[dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/rusty-filter/src/ast.rs b/rusty-filter/src/ast.rs new file mode 100644 index 0000000..a344c93 --- /dev/null +++ b/rusty-filter/src/ast.rs @@ -0,0 +1,304 @@ +//! The filter AST, mirroring `dist/types/filter.d.ts` of `filter-query-editor`. +//! +//! The shapes here are deliberately faithful to the TypeScript interfaces, +//! `Option` placement included, so that `serde_json` output is byte-compatible +//! with what the browser's editor produces. See the crate-level docs for the +//! rules governing which keys are present. + +use serde::{Deserialize, Serialize}; + +/// How the filters of a group are combined. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum LogicalOp { + /// Every filter must match. This is the default, matching the empty-query + /// result `{op: "AND", filters: []}`. + #[default] + #[serde(rename = "AND")] + And, + #[serde(rename = "OR")] + Or, +} + +/// The ten functions the reference implementation emits. +/// +/// There is deliberately no `NotEquals`: `!=`, `not equals` and `not equal` all +/// produce [`FilterFunction::Equals`] with `negate: Some(true)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FilterFunction { + Equals, + GreaterThan, + LessThan, + GreaterThanOrEqual, + LessThanOrEqual, + Contains, + StartsWith, + EndsWith, + IsBlank, + IsNotBlank, +} + +impl FilterFunction { + /// The human-readable name used in validation messages, from + /// `getOperatorDisplayName` in `dist/validator/TypeChecker.js`. + pub fn display_name(self) -> &'static str { + match self { + FilterFunction::Equals => "equals", + FilterFunction::GreaterThan => "greater than", + FilterFunction::LessThan => "less than", + FilterFunction::GreaterThanOrEqual => "greater than or equal", + FilterFunction::LessThanOrEqual => "less than or equal", + FilterFunction::Contains => "contains", + FilterFunction::StartsWith => "starts with", + FilterFunction::EndsWith => "ends with", + FilterFunction::IsBlank => "is blank", + FilterFunction::IsNotBlank => "is not blank", + } + } + + /// Whether this is one of the two argument-less existence operators. + pub fn is_blank_operator(self) -> bool { + matches!(self, FilterFunction::IsBlank | FilterFunction::IsNotBlank) + } +} + +/// A single filter expression, e.g. `[status] equals "open"`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Condition { + /// The column identifier, exactly as written between the brackets. + pub column: String, + /// The comparison or text operation. + pub function: FilterFunction, + /// Values to compare against. Empty for the two blank operators. + pub args: Vec, +} + +impl Condition { + pub fn new( + column: impl Into, + function: FilterFunction, + args: Vec, + ) -> Self { + Condition { + column: column.into(), + function, + args, + } + } +} + +/// A collection of filters combined with `AND` or `OR`. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct FilterGroup { + pub op: LogicalOp, + pub filters: Vec, +} + +impl FilterGroup { + /// A group combining `filters` with `AND`. + pub fn and(filters: Vec) -> Self { + FilterGroup { + op: LogicalOp::And, + filters, + } + } + + /// A group combining `filters` with `OR`. + pub fn or(filters: Vec) -> Self { + FilterGroup { + op: LogicalOp::Or, + filters, + } + } + + /// Whether this group holds no filters. An empty group is what the empty + /// query parses to, and it is the case [`crate::print::canonical_key`] + /// treats as "no filter at all". + pub fn is_empty(&self) -> bool { + self.filters.is_empty() + } +} + +/// One entry of a [`FilterGroup`]: a condition or a nested group, optionally +/// negated. +/// +/// `condition` and `group` are mutually exclusive but both optional, as in the +/// TypeScript interface. `negate` is `Option` on purpose: comparisons omit +/// the key entirely, text operations always emit it (`false` included). +/// Round-tripping the frontend's JSON must not change which keys are present. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct Filter { + #[serde(skip_serializing_if = "Option::is_none")] + pub condition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub group: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub negate: Option, +} + +impl Filter { + /// A condition filter with no `negate` key, as a comparison produces. + pub fn condition( + column: impl Into, + function: FilterFunction, + args: Vec, + ) -> Self { + Filter { + condition: Some(Condition::new(column, function, args)), + group: None, + negate: None, + } + } + + /// A filter wrapping a nested group, as `(...)` produces. + pub fn from_group(group: FilterGroup) -> Self { + Filter { + condition: None, + group: Some(group), + negate: None, + } + } + + /// This filter with `negate: Some(negate)`. Passing `false` emits the key + /// with a `false` value rather than dropping it, which is what a text + /// operation without `NOT` does. + pub fn negated(mut self, negate: bool) -> Self { + self.negate = Some(negate); + self + } + + /// Whether negation is in effect. A missing key and `Some(false)` both mean + /// "not negated", exactly as the JavaScript truthiness test does. + pub fn is_negated(&self) -> bool { + self.negate.unwrap_or(false) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn logical_op_defaults_to_and() { + assert_eq!(LogicalOp::default(), LogicalOp::And); + assert_eq!(FilterGroup::default(), FilterGroup::and(vec![])); + } + + #[test] + fn logical_op_serializes_uppercase() { + assert_eq!(serde_json::to_value(LogicalOp::And).unwrap(), json!("AND")); + assert_eq!(serde_json::to_value(LogicalOp::Or).unwrap(), json!("OR")); + } + + #[test] + fn filter_function_serializes_camel_case() { + let pairs = [ + (FilterFunction::Equals, "equals"), + (FilterFunction::GreaterThan, "greaterThan"), + (FilterFunction::LessThan, "lessThan"), + (FilterFunction::GreaterThanOrEqual, "greaterThanOrEqual"), + (FilterFunction::LessThanOrEqual, "lessThanOrEqual"), + (FilterFunction::Contains, "contains"), + (FilterFunction::StartsWith, "startsWith"), + (FilterFunction::EndsWith, "endsWith"), + (FilterFunction::IsBlank, "isBlank"), + (FilterFunction::IsNotBlank, "isNotBlank"), + ]; + for (func, expected) in pairs { + assert_eq!(serde_json::to_value(func).unwrap(), json!(expected)); + } + } + + #[test] + fn display_names_match_the_reference() { + assert_eq!(FilterFunction::Equals.display_name(), "equals"); + assert_eq!(FilterFunction::GreaterThan.display_name(), "greater than"); + assert_eq!(FilterFunction::LessThan.display_name(), "less than"); + assert_eq!( + FilterFunction::GreaterThanOrEqual.display_name(), + "greater than or equal" + ); + assert_eq!( + FilterFunction::LessThanOrEqual.display_name(), + "less than or equal" + ); + assert_eq!(FilterFunction::Contains.display_name(), "contains"); + assert_eq!(FilterFunction::StartsWith.display_name(), "starts with"); + assert_eq!(FilterFunction::EndsWith.display_name(), "ends with"); + assert_eq!(FilterFunction::IsBlank.display_name(), "is blank"); + assert_eq!(FilterFunction::IsNotBlank.display_name(), "is not blank"); + } + + #[test] + fn absent_negate_is_omitted_from_json() { + let f = Filter::condition("age", FilterFunction::GreaterThan, vec![json!(1)]); + let value = serde_json::to_value(&f).unwrap(); + assert_eq!( + value, + json!({"condition": {"column": "age", "function": "greaterThan", "args": [1]}}) + ); + assert!(value.get("negate").is_none()); + assert!(value.get("group").is_none()); + } + + #[test] + fn false_negate_is_present_in_json() { + let f = + Filter::condition("name", FilterFunction::Contains, vec![json!("a")]).negated(false); + let value = serde_json::to_value(&f).unwrap(); + assert_eq!(value.get("negate"), Some(&json!(false))); + } + + #[test] + fn is_negated_treats_missing_and_false_alike() { + let bare = Filter::condition("age", FilterFunction::Equals, vec![json!(1)]); + assert!(!bare.is_negated()); + assert!(!bare.clone().negated(false).is_negated()); + assert!(bare.negated(true).is_negated()); + } + + #[test] + fn group_constructors_set_the_operator() { + assert_eq!(FilterGroup::and(vec![]).op, LogicalOp::And); + assert_eq!(FilterGroup::or(vec![]).op, LogicalOp::Or); + assert!(FilterGroup::and(vec![]).is_empty()); + assert!(!FilterGroup::and(vec![Filter::default()]).is_empty()); + } + + #[test] + fn from_group_wraps_without_negation() { + let inner = FilterGroup::or(vec![Filter::condition( + "age", + FilterFunction::Equals, + vec![json!(1)], + )]); + let f = Filter::from_group(inner.clone()); + assert_eq!(f.group, Some(inner)); + assert!(f.condition.is_none()); + assert!(f.negate.is_none()); + } + + #[test] + fn blank_operators_are_recognised() { + assert!(FilterFunction::IsBlank.is_blank_operator()); + assert!(FilterFunction::IsNotBlank.is_blank_operator()); + assert!(!FilterFunction::Equals.is_blank_operator()); + assert!(!FilterFunction::Contains.is_blank_operator()); + } + + #[test] + fn round_trips_through_json() { + let group = FilterGroup::or(vec![ + Filter::from_group(FilterGroup::and(vec![ + Filter::condition("age", FilterFunction::GreaterThan, vec![json!(1)]), + Filter::condition("name", FilterFunction::Contains, vec![json!("a")]) + .negated(false), + ])), + Filter::condition("active", FilterFunction::Equals, vec![json!(true)]).negated(true), + ]); + let json = serde_json::to_string(&group).unwrap(); + let back: FilterGroup = serde_json::from_str(&json).unwrap(); + assert_eq!(back, group); + } +} diff --git a/rusty-filter/src/column.rs b/rusty-filter/src/column.rs new file mode 100644 index 0000000..a67e2b8 --- /dev/null +++ b/rusty-filter/src/column.rs @@ -0,0 +1,166 @@ +//! Column schema used for validation and evaluation. + +use serde::{Deserialize, Serialize}; + +/// The five column types the grammar's validator knows about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ColumnType { + #[default] + String, + Number, + Boolean, + Date, + Enum, +} + +impl ColumnType { + /// Map a backend type name onto a [`ColumnType`], reproducing + /// `normalizeColumnType` in `dist/validator/TypeChecker.js`. + /// + /// `INT32`, `INT64`, `DOUBLE`, `DECIMAL` and `NUMBER` become + /// [`ColumnType::Number`]; `TEXT`, `STRING` and `ICON` become + /// [`ColumnType::String`]; `BOOLEAN` becomes [`ColumnType::Boolean`]; + /// `DATE` and `DATETIME` become [`ColumnType::Date`]; `ENUM` becomes + /// [`ColumnType::Enum`]. Anything else becomes [`ColumnType::String`]. + /// The comparison is case-insensitive. + pub fn normalize(type_name: &str) -> ColumnType { + let upper = type_name.to_ascii_uppercase(); + match upper.as_str() { + "INT32" | "INT64" | "DOUBLE" | "DECIMAL" | "NUMBER" => ColumnType::Number, + "TEXT" | "STRING" | "ICON" => ColumnType::String, + "BOOLEAN" => ColumnType::Boolean, + "DATE" | "DATETIME" => ColumnType::Date, + "ENUM" => ColumnType::Enum, + _ => ColumnType::String, + } + } + + /// The name used in validation messages, matching the lowercase + /// `ColumnType` union of `dist/types/column.d.ts`. + pub fn as_str(self) -> &'static str { + match self { + ColumnType::String => "string", + ColumnType::Number => "number", + ColumnType::Boolean => "boolean", + ColumnType::Date => "date", + ColumnType::Enum => "enum", + } + } +} + +/// One filterable column. +/// +/// The TypeScript `ColumnDef` also carries a `width`; nothing in parsing, +/// validation or evaluation reads it, so it is deliberately dropped here. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ColumnDef { + pub name: String, + #[serde(rename = "type")] + pub column_type: ColumnType, +} + +impl ColumnDef { + pub fn new(name: impl Into, column_type: ColumnType) -> Self { + ColumnDef { + name: name.into(), + column_type, + } + } + + /// A column whose backend type name needs normalizing first. + pub fn normalized(name: impl Into, type_name: &str) -> Self { + ColumnDef::new(name, ColumnType::normalize(type_name)) + } +} + +/// Look a column up by exact name. Names are matched verbatim — the grammar +/// does not trim what is between the brackets, so `[ s ]` and `[s]` are +/// different columns. +pub(crate) fn find_column<'a>(columns: &'a [ColumnDef], name: &str) -> Option<&'a ColumnDef> { + columns.iter().find(|c| c.name == name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalizes_number_types() { + for name in ["INT32", "INT64", "DOUBLE", "DECIMAL", "NUMBER"] { + assert_eq!(ColumnType::normalize(name), ColumnType::Number, "{name}"); + assert_eq!( + ColumnType::normalize(&name.to_lowercase()), + ColumnType::Number, + "{name} lowercase" + ); + } + } + + #[test] + fn normalizes_string_types() { + for name in ["TEXT", "STRING", "ICON", "text", "Icon"] { + assert_eq!(ColumnType::normalize(name), ColumnType::String, "{name}"); + } + } + + #[test] + fn normalizes_boolean_date_and_enum() { + assert_eq!(ColumnType::normalize("BOOLEAN"), ColumnType::Boolean); + assert_eq!(ColumnType::normalize("boolean"), ColumnType::Boolean); + assert_eq!(ColumnType::normalize("DATE"), ColumnType::Date); + assert_eq!(ColumnType::normalize("DATETIME"), ColumnType::Date); + assert_eq!(ColumnType::normalize("datetime"), ColumnType::Date); + assert_eq!(ColumnType::normalize("ENUM"), ColumnType::Enum); + assert_eq!(ColumnType::normalize("enum"), ColumnType::Enum); + } + + #[test] + fn unknown_types_fall_back_to_string() { + for name in ["GUID", "", "Labels", "Link", "whatever"] { + assert_eq!(ColumnType::normalize(name), ColumnType::String, "{name}"); + } + } + + #[test] + fn default_is_string() { + assert_eq!(ColumnType::default(), ColumnType::String); + } + + #[test] + fn type_names_are_lowercase() { + assert_eq!(ColumnType::String.as_str(), "string"); + assert_eq!(ColumnType::Number.as_str(), "number"); + assert_eq!(ColumnType::Boolean.as_str(), "boolean"); + assert_eq!(ColumnType::Date.as_str(), "date"); + assert_eq!(ColumnType::Enum.as_str(), "enum"); + } + + #[test] + fn normalized_constructor_maps_the_type() { + let col = ColumnDef::normalized("age", "INT64"); + assert_eq!(col.name, "age"); + assert_eq!(col.column_type, ColumnType::Number); + } + + #[test] + fn lookup_is_exact() { + let cols = vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new(" s ", ColumnType::String), + ]; + assert_eq!(find_column(&cols, "age").unwrap().name, "age"); + assert_eq!(find_column(&cols, " s ").unwrap().name, " s "); + assert!(find_column(&cols, "s").is_none()); + assert!(find_column(&cols, "AGE").is_none()); + } + + #[test] + fn serializes_type_as_lowercase_string() { + let col = ColumnDef::new("age", ColumnType::Number); + assert_eq!( + serde_json::to_value(&col).unwrap(), + serde_json::json!({"name": "age", "type": "number"}) + ); + } +} diff --git a/rusty-filter/src/eval.rs b/rusty-filter/src/eval.rs new file mode 100644 index 0000000..5938395 --- /dev/null +++ b/rusty-filter/src/eval.rs @@ -0,0 +1,846 @@ +//! Evaluating a filter against row data. +//! +//! A port of `FilterEvaluator` and `Comparators`. Rows are +//! [`serde_json::Value`] objects, which is what `DataTable` already stores, so +//! nothing needs converting. +//! +//! Three behaviours are worth knowing before reading the code: +//! +//! * **An empty group is `true`, whatever its operator.** The reference returns +//! early on `filters.length === 0` before it looks at `op`, so an empty `OR` +//! group is vacuously true just like an empty `AND` group. Keep it that way: +//! the empty query parses to an empty group and must match every row. +//! * **The column type decides which operators can match.** `contains` on a +//! non-string column is `false`, not a string coercion; the orderings on +//! anything but a number or a date are `false`. Because [`ColumnDef`] stores a +//! normalized [`ColumnType`], a column declared `INT32` compares as a number +//! here, whereas the reference — which switches on the raw type name — +//! silently returns `false`. That divergence is deliberate. +//! * **A non-object row has no columns.** The reference reads `row[column]`, +//! which throws for `null` and is caught into a blanket `false`; here every +//! column of a non-object row reads as missing, so `isBlank` matches it. + +use std::collections::HashMap; + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup, LogicalOp}; +use crate::column::{ColumnDef, ColumnType}; + +/// The recursion limit from `FilterEvaluator`. Exceeding it fails the whole +/// evaluation rather than the offending branch, as the reference's throw and +/// blanket `catch` do. +const MAX_RECURSION_DEPTH: usize = 100; + +/// A name-to-column lookup. On a duplicate name the first definition wins, +/// matching [`crate::column::find_column`] and so keeping validation and +/// evaluation in agreement. +type ColumnMap<'a> = HashMap<&'a str, &'a ColumnDef>; + +fn column_map<'a>(columns: &'a [ColumnDef]) -> ColumnMap<'a> { + let mut map = ColumnMap::with_capacity(columns.len()); + for column in columns { + map.entry(column.name.as_str()).or_insert(column); + } + map +} + +/// Whether `row` matches `filter`. +/// +/// Returns `false` if the filter nests deeper than 100 groups. +pub fn evaluate(filter: &FilterGroup, row: &serde_json::Value, columns: &[ColumnDef]) -> bool { + eval_group(filter, row, &column_map(columns), 0).unwrap_or(false) +} + +/// Keep only the rows matching `filter`, preserving their order. +pub fn retain_matching( + filter: &FilterGroup, + rows: Vec, + columns: &[ColumnDef], +) -> Vec { + let map = column_map(columns); + rows.into_iter() + .filter(|row| eval_group(filter, row, &map, 0).unwrap_or(false)) + .collect() +} + +/// How many of `rows` match `filter`, without building a new collection. +pub fn count_matches( + filter: &FilterGroup, + rows: &[serde_json::Value], + columns: &[ColumnDef], +) -> usize { + let map = column_map(columns); + rows.iter() + .filter(|row| eval_group(filter, row, &map, 0).unwrap_or(false)) + .count() +} + +/// `None` stands for the reference's "maximum recursion depth exceeded" throw, +/// which its caller turns into `false` for the entire filter. +fn eval_group( + group: &FilterGroup, + row: &serde_json::Value, + columns: &ColumnMap<'_>, + depth: usize, +) -> Option { + if depth > MAX_RECURSION_DEPTH { + return None; + } + // Vacuous truth, checked before `op` is examined. + if group.filters.is_empty() { + return Some(true); + } + match group.op { + LogicalOp::And => { + for filter in &group.filters { + if !eval_filter(filter, row, columns, depth + 1)? { + return Some(false); + } + } + Some(true) + } + LogicalOp::Or => { + for filter in &group.filters { + if eval_filter(filter, row, columns, depth + 1)? { + return Some(true); + } + } + Some(false) + } + } +} + +fn eval_filter( + filter: &Filter, + row: &serde_json::Value, + columns: &ColumnMap<'_>, + depth: usize, +) -> Option { + // A condition wins over a group when both are set, and a filter with + // neither never matches — even negated, since the reference returns before + // it applies negation. + let result = if let Some(condition) = &filter.condition { + eval_condition(condition, row, columns) + } else if let Some(group) = &filter.group { + // The nested group keeps this depth rather than incrementing again. + eval_group(group, row, columns, depth)? + } else { + return Some(false); + }; + Some(result != filter.is_negated()) +} + +fn eval_condition(condition: &Condition, row: &serde_json::Value, columns: &ColumnMap<'_>) -> bool { + let Some(column) = columns.get(condition.column.as_str()) else { + // A condition on an unknown column never matches. + return false; + }; + let value = row.get(&condition.column); + apply( + condition.function, + value, + &condition.args, + column.column_type, + ) +} + +/// Whether `value` — `None` when the row has no such key — satisfies +/// `function` against `args` for a column of `column_type`. +pub fn apply( + function: FilterFunction, + value: Option<&serde_json::Value>, + args: &[serde_json::Value], + column_type: ColumnType, +) -> bool { + // The blank operators ignore their arguments and are the only ones a + // missing value can satisfy. + match function { + FilterFunction::IsBlank => return is_blank(value, column_type), + FilterFunction::IsNotBlank => return !is_blank(value, column_type), + _ => {} + } + + let (Some(value), Some(arg)) = (non_null(value), non_null(args.first())) else { + return false; + }; + + match function { + FilterFunction::Equals => strict_equals(value, arg), + FilterFunction::GreaterThan + | FilterFunction::LessThan + | FilterFunction::GreaterThanOrEqual + | FilterFunction::LessThanOrEqual => compare(function, value, arg, column_type), + FilterFunction::Contains | FilterFunction::StartsWith | FilterFunction::EndsWith => { + if column_type != ColumnType::String { + return false; + } + let (Some(haystack), Some(needle)) = (value.as_str(), arg.as_str()) else { + return false; + }; + // Case-sensitive on purpose: the reference does not fold case. + match function { + FilterFunction::Contains => haystack.contains(needle), + FilterFunction::StartsWith => haystack.starts_with(needle), + _ => haystack.ends_with(needle), + } + } + FilterFunction::IsBlank | FilterFunction::IsNotBlank => unreachable!("handled above"), + } +} + +/// A missing key and a JSON `null` are both the reference's "nullish". +fn non_null(value: Option<&serde_json::Value>) -> Option<&serde_json::Value> { + match value { + Some(serde_json::Value::Null) | None => None, + other => other, + } +} + +fn is_blank(value: Option<&serde_json::Value>, column_type: ColumnType) -> bool { + match non_null(value) { + None => true, + // The empty string counts as blank for string columns only. + Some(value) => column_type == ColumnType::String && value.as_str() == Some(""), + } +} + +/// JavaScript `===` on two JSON values. +/// +/// Strict, so no coercion: the number `1` never equals the string `"1"`. Numbers +/// are compared by value rather than by JSON representation, because `1` and +/// `1.0` are the same number in JavaScript and a row loaded from a database may +/// carry either. +fn strict_equals(a: &serde_json::Value, b: &serde_json::Value) -> bool { + match (a, b) { + (serde_json::Value::Number(x), serde_json::Value::Number(y)) => x.as_f64() == y.as_f64(), + // Arrays and objects compare by identity in JavaScript, so two distinct + // ones are never equal however alike they look. + (serde_json::Value::Array(_), _) + | (_, serde_json::Value::Array(_)) + | (serde_json::Value::Object(_), _) + | (_, serde_json::Value::Object(_)) => false, + _ => a == b, + } +} + +fn compare( + function: FilterFunction, + value: &serde_json::Value, + arg: &serde_json::Value, + column_type: ColumnType, +) -> bool { + let ordering = match column_type { + ColumnType::Number => { + let (Some(a), Some(b)) = (value.as_f64(), arg.as_f64()) else { + return false; + }; + a.partial_cmp(&b) + } + ColumnType::Date => { + let (Some(a), Some(b)) = (value.as_str(), arg.as_str()) else { + return false; + }; + let (Some(a), Some(b)) = (parse_iso_millis(a), parse_iso_millis(b)) else { + // An unparseable date is the reference's `NaN`, which fails + // every comparison. + return false; + }; + Some(a.cmp(&b)) + } + // The orderings are not supported for other types. + ColumnType::String | ColumnType::Boolean | ColumnType::Enum => return false, + }; + let Some(ordering) = ordering else { + return false; + }; + match function { + FilterFunction::GreaterThan => ordering.is_gt(), + FilterFunction::LessThan => ordering.is_lt(), + FilterFunction::GreaterThanOrEqual => ordering.is_ge(), + FilterFunction::LessThanOrEqual => ordering.is_le(), + _ => false, + } +} + +/// Parse an ISO 8601 date or datetime into milliseconds since the Unix epoch, +/// returning `None` where `Date.parse` would return `NaN`. +/// +/// This follows ECMAScript's Date Time String Format, which is what the +/// reference's `new Date(...)` reaches for: the field ranges are checked +/// (month 1–12, day 1–31, hour 0–24, minute and second 0–59, and hour 24 only at +/// exactly `24:00:00`), then the arithmetic is allowed to roll over, so +/// `2024-02-30` is 1 March and `2024-01-01T24:00:00` is 2 January — while +/// `2024-13-01` and `2024-01-32` are unparseable. +/// +/// **One deliberate divergence.** A datetime with no trailing `Z` is read as +/// UTC, whereas the browser reads it in the machine's local zone. Following the +/// browser would make server-side filtering depend on the server's timezone and +/// disagree with the same filter run in the client, so a date-only value and a +/// naked datetime compare equal here — `2024-01-01T00:00:00 > 2024-01-01` is +/// false in both implementations, but for different reasons. +fn parse_iso_millis(text: &str) -> Option { + let bytes = text.as_bytes(); + let digits = |start: usize, len: usize| -> Option { + let slice = bytes.get(start..start + len)?; + if !slice.iter().all(u8::is_ascii_digit) { + return None; + } + std::str::from_utf8(slice).ok()?.parse().ok() + }; + + let year = digits(0, 4)?; + if bytes.get(4) != Some(&b'-') { + return None; + } + let month = digits(5, 2)?; + if bytes.get(7) != Some(&b'-') { + return None; + } + let day = digits(8, 2)?; + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + return None; + } + + let mut hour = 0; + let mut minute = 0; + let mut second = 0; + let mut milli = 0; + let mut end = 10; + if bytes.len() > 10 { + if bytes[10] != b'T' { + return None; + } + hour = digits(11, 2)?; + if bytes.get(13) != Some(&b':') { + return None; + } + minute = digits(14, 2)?; + if bytes.get(16) != Some(&b':') { + return None; + } + second = digits(17, 2)?; + end = 19; + if bytes.get(end) == Some(&b'.') { + milli = digits(end + 1, 3)?; + end += 4; + } + if !(0..=24).contains(&hour) || minute > 59 || second > 59 { + return None; + } + // Hour 24 names midnight ending the day, so nothing may follow it. + if hour == 24 && (minute, second, milli) != (0, 0, 0) { + return None; + } + if bytes.get(end) == Some(&b'Z') { + end += 1; + } + } + if end != bytes.len() { + return None; + } + + let days = days_from_civil(year, month, day); + Some(((days * 24 + hour) * 60 + minute) * 60_000 + second * 1_000 + milli) +} + +/// Days from 1970-01-01 to `year-month-day` in the proleptic Gregorian +/// calendar. A day past the end of its month rolls into the next one, as +/// ECMAScript's `MakeDay` does. +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + // Shift so that the leap day falls at the end of the 400-year cycle. + let y = if month <= 2 { year - 1 } else { year }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let year_of_era = y - era * 400; + let month_shifted = if month > 2 { month - 3 } else { month + 9 }; + let day_of_year = (153 * month_shifted + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse_query_unchecked; + use serde_json::json; + + fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("created", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ] + } + + fn matches(query: &str, row: serde_json::Value) -> bool { + let group = parse_query_unchecked(query).expect(query); + evaluate(&group, &row, &columns()) + } + + #[test] + fn the_probed_rows_behave_as_measured() { + let query = "[age] > 1 AND [name] contains \"a\""; + assert!(matches(query, json!({"age": 2, "name": "abc"}))); + assert!(!matches(query, json!({"age": 0, "name": "abc"}))); + assert!(!matches(query, json!({"age": 2, "name": "xyz"}))); + assert!(!matches(query, json!({"age": null, "name": "abc"}))); + } + + #[test] + fn or_needs_only_one_arm() { + let query = "[age] > 10 OR [name] = \"a\""; + assert!(matches(query, json!({"age": 11, "name": "z"}))); + assert!(matches(query, json!({"age": 0, "name": "a"}))); + assert!(!matches(query, json!({"age": 0, "name": "z"}))); + } + + #[test] + fn an_empty_group_is_true_whatever_its_operator() { + let row = json!({"age": 1}); + // This is the divergence from the plan's table: an empty OR group is + // true in the bundle, not false. + assert!(evaluate(&FilterGroup::and(vec![]), &row, &columns())); + assert!(evaluate(&FilterGroup::or(vec![]), &row, &columns())); + // And the empty query parses to exactly that group. + assert!(matches("", row.clone())); + assert!(matches(" ", row)); + } + + #[test] + fn an_empty_group_matches_even_a_non_object_row() { + assert!(evaluate(&FilterGroup::default(), &json!(null), &columns())); + assert!(evaluate(&FilterGroup::default(), &json!(7), &columns())); + } + + #[test] + fn negation_inverts_the_filter() { + assert!(matches("NOT [age] > 1", json!({"age": 0}))); + assert!(!matches("NOT [age] > 1", json!({"age": 2}))); + assert!(matches("[name] not contains \"a\"", json!({"name": "xyz"}))); + assert!(!matches( + "[name] not contains \"a\"", + json!({"name": "abc"}) + )); + // A false `negate` key must not invert anything. + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![json!(1)], + ) + .negated(false)]); + assert!(evaluate(&group, &json!({"age": 2}), &columns())); + } + + #[test] + fn negation_applies_to_groups_too() { + assert!(matches( + "NOT ([age] > 1 AND [name] = \"a\")", + json!({"age": 2, "name": "b"}) + )); + assert!(!matches( + "NOT ([age] > 1 AND [name] = \"a\")", + json!({"age": 2, "name": "a"}) + )); + } + + #[test] + fn a_null_value_fails_everything_but_is_blank() { + for query in [ + "[name] = \"a\"", + "[name] contains \"a\"", + "[name] starts with \"a\"", + "[name] ends with \"a\"", + "[name] is not blank", + ] { + assert!(!matches(query, json!({"name": null})), "{query}"); + assert!(!matches(query, json!({})), "{query} missing"); + } + assert!(matches("[name] is blank", json!({"name": null}))); + assert!(matches("[name] is blank", json!({}))); + } + + #[test] + fn a_missing_column_definition_never_matches() { + let group = parse_query_unchecked("[nope] is blank").unwrap(); + assert!(!evaluate(&group, &json!({"nope": null}), &columns())); + } + + #[test] + fn is_blank_covers_the_empty_string_for_strings_only() { + assert!(matches("[name] is blank", json!({"name": ""}))); + assert!(!matches("[name] is blank", json!({"name": "a"}))); + assert!(!matches("[name] is not blank", json!({"name": ""}))); + // An empty string in a date or enum column is not blank. + assert!(!matches("[created] is blank", json!({"created": ""}))); + assert!(!matches("[kind] is blank", json!({"kind": ""}))); + assert!(matches("[kind] is not blank", json!({"kind": ""}))); + } + + #[test] + fn equals_is_strict() { + assert!(!matches("[name] = \"1\"", json!({"name": 1}))); + assert!(matches("[name] = \"1\"", json!({"name": "1"}))); + assert!(!matches("[age] = 1", json!({"age": "1"}))); + assert!(!matches("[age] = 1", json!({"age": true}))); + assert!(!matches("[active] = true", json!({"active": 1}))); + assert!(matches("[active] = true", json!({"active": true}))); + assert!(matches("[active] = false", json!({"active": false}))); + assert!(!matches("[active] = false", json!({"active": true}))); + } + + #[test] + fn equals_compares_numbers_by_value_not_representation() { + // A row loaded from a database may hold an integer where the query + // holds a float, and JavaScript would call those equal. + assert!(matches("[age] = 1", json!({"age": 1}))); + assert!(matches("[age] = 1", json!({"age": 1.0}))); + assert!(matches("[age] = 1.0", json!({"age": 1}))); + assert!(matches("[age] = 1.5", json!({"age": 1.5}))); + assert!(!matches("[age] = 1.5", json!({"age": 1.6}))); + } + + #[test] + fn equals_never_matches_a_composite_value() { + let group = FilterGroup::and(vec![Filter::condition( + "name", + FilterFunction::Equals, + vec![json!([1])], + )]); + assert!(!evaluate(&group, &json!({"name": [1]}), &columns())); + let group = FilterGroup::and(vec![Filter::condition( + "name", + FilterFunction::Equals, + vec![json!({"a": 1})], + )]); + assert!(!evaluate(&group, &json!({"name": {"a": 1}}), &columns())); + } + + #[test] + fn number_orderings() { + assert!(matches("[age] > 1", json!({"age": 1.5}))); + assert!(!matches("[age] > 1", json!({"age": 1}))); + assert!(matches("[age] >= 1", json!({"age": 1}))); + assert!(matches("[age] < 1", json!({"age": 0.5}))); + assert!(!matches("[age] < 1", json!({"age": 1}))); + assert!(matches("[age] <= 1", json!({"age": 1}))); + assert!(matches("[age] > -1", json!({"age": 0}))); + // A non-numeric value in a numeric column cannot be ordered. + assert!(!matches("[age] > 1", json!({"age": "5"}))); + assert!(!matches("[age] > 1", json!({"age": true}))); + } + + #[test] + fn orderings_are_unsupported_for_string_boolean_and_enum() { + // The grammar and validator both reject these, but a hand-built AST + // can carry them, and then they must not match. + for column in ["name", "active", "kind"] { + let group = FilterGroup::and(vec![Filter::condition( + column, + FilterFunction::GreaterThan, + vec![json!("a")], + )]); + let row = json!({"name": "b", "active": true, "kind": "b"}); + assert!(!evaluate(&group, &row, &columns()), "{column}"); + } + } + + #[test] + fn contains_and_friends_are_case_sensitive() { + assert!(matches("[name] contains \"bc\"", json!({"name": "abcd"}))); + assert!(!matches("[name] contains \"BC\"", json!({"name": "abcd"}))); + assert!(matches( + "[name] starts with \"ab\"", + json!({"name": "abcd"}) + )); + assert!(!matches( + "[name] starts with \"AB\"", + json!({"name": "abcd"}) + )); + assert!(matches("[name] ends with \"cd\"", json!({"name": "abcd"}))); + assert!(!matches("[name] ends with \"CD\"", json!({"name": "abcd"}))); + // The empty needle is contained in everything. + assert!(matches("[name] contains \"\"", json!({"name": "abcd"}))); + } + + #[test] + fn text_operations_need_a_string_column() { + // `[kind]` is an enum, so `contains` cannot match however alike the + // values look. + let group = FilterGroup::and(vec![Filter::condition( + "kind", + FilterFunction::Contains, + vec![json!("b")], + )]); + assert!(!evaluate(&group, &json!({"kind": "abc"}), &columns())); + } + + #[test] + fn date_orderings() { + assert!(matches( + "[created] > \"2024-01-01\"", + json!({"created": "2024-06-01"}) + )); + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": "2023-06-01"}) + )); + assert!(matches( + "[created] >= \"2024-01-01\"", + json!({"created": "2024-01-01"}) + )); + assert!(matches( + "[created] < \"2024-01-01T12:00:00Z\"", + json!({"created": "2024-01-01T11:59:59Z"}) + )); + // Different shapes still compare on the instant they name. + assert!(matches( + "[created] <= \"2024-01-01\"", + json!({"created": "2024-01-01T00:00:00Z"}) + )); + assert!(matches( + "[created] >= \"2024-01-01\"", + json!({"created": "2024-01-01T00:00:00Z"}) + )); + } + + #[test] + fn a_naked_datetime_does_not_exceed_the_bare_date() { + // The measured browser answer, reached here without depending on the + // machine's timezone. + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": "2024-01-01T00:00:00"}) + )); + } + + #[test] + fn an_unparseable_date_fails_every_ordering() { + for query in [ + "[created] > \"2024-13-99\"", + "[created] < \"2024-13-99\"", + "[created] >= \"2024-13-99\"", + "[created] <= \"2024-13-99\"", + ] { + assert!(!matches(query, json!({"created": "2024-01-01"})), "{query}"); + } + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": "not a date"}) + )); + // A non-string value in a date column cannot be ordered either. + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": 1}) + )); + } + + #[test] + fn iso_parsing_matches_the_javascript_engine() { + // Values measured with `Date.parse` in Node on 2026-08-01. + assert_eq!(parse_iso_millis("2024-01-01"), Some(1_704_067_200_000)); + assert_eq!( + parse_iso_millis("2024-01-01T00:00:00Z"), + Some(1_704_067_200_000) + ); + assert_eq!( + parse_iso_millis("2024-12-31T23:59:59.999Z"), + Some(1_735_689_599_999) + ); + assert_eq!(parse_iso_millis("0000-01-01"), Some(-62_167_219_200_000)); + assert_eq!(parse_iso_millis("9999-12-31"), Some(253_402_214_400_000)); + assert_eq!(parse_iso_millis("2024-02-29"), Some(1_709_164_800_000)); + // Out-of-range fields are unparseable, exactly as in the engine. + for text in [ + "2024-13-01", + "2024-00-01", + "2024-01-32", + "2024-01-00", + "2024-01-01T25:00:00", + "2024-01-01T23:60:00", + "2024-01-01T23:59:60", + "2024-01-01T24:00:01", + "2024-01-01T24:59:59", + "2024-1-1", + "", + "x", + "2024-01-01 00:00:00", + "2024-01-01T00:00", + "2024-01-01T00:00:00.12", + "2024-01-01T00:00:00ZZ", + ] { + assert_eq!(parse_iso_millis(text), None, "{text}"); + } + } + + #[test] + fn iso_parsing_rolls_over_like_make_day_and_make_time() { + // Both measured in Node: an over-long month rolls into the next one, + // and hour 24 names the following midnight. + assert_eq!( + parse_iso_millis("2024-02-30"), + parse_iso_millis("2024-03-01") + ); + assert_eq!( + parse_iso_millis("2023-02-29"), + parse_iso_millis("2023-03-01") + ); + assert_eq!( + parse_iso_millis("2024-04-31"), + parse_iso_millis("2024-05-01") + ); + assert_eq!( + parse_iso_millis("2024-01-01T24:00:00Z"), + parse_iso_millis("2024-01-02") + ); + } + + #[test] + fn days_from_civil_anchors_on_the_epoch() { + assert_eq!(days_from_civil(1970, 1, 1), 0); + assert_eq!(days_from_civil(1970, 1, 2), 1); + assert_eq!(days_from_civil(1969, 12, 31), -1); + assert_eq!(days_from_civil(2000, 3, 1), 11_017); + assert_eq!(days_from_civil(1900, 3, 1), -25_508); + } + + #[test] + fn normalized_types_widen_what_can_match() { + // The reference switches on the raw type name, so an `INT32` column + // fails every ordering there. Here the type is normalized first. + let cols = vec![ + ColumnDef::normalized("n", "INT32"), + ColumnDef::normalized("t", "TEXT"), + ]; + let group = parse_query_unchecked("[n] > 1 AND [t] contains \"a\"").unwrap(); + assert!(evaluate(&group, &json!({"n": 2, "t": "abc"}), &cols)); + assert!(!evaluate(&group, &json!({"n": 1, "t": "abc"}), &cols)); + } + + #[test] + fn a_non_object_row_has_no_columns() { + let group = parse_query_unchecked("[name] is blank").unwrap(); + assert!(evaluate(&group, &json!(null), &columns())); + assert!(evaluate(&group, &json!([1, 2]), &columns())); + let group = parse_query_unchecked("[name] = \"a\"").unwrap(); + assert!(!evaluate(&group, &json!(null), &columns())); + } + + #[test] + fn a_filter_with_neither_side_never_matches() { + let group = FilterGroup::and(vec![Filter::default()]); + assert!(!evaluate(&group, &json!({}), &columns())); + // Negation does not rescue it: the reference returns before negating. + let group = FilterGroup::and(vec![Filter::default().negated(true)]); + assert!(!evaluate(&group, &json!({}), &columns())); + } + + #[test] + fn a_condition_wins_over_a_group_on_the_same_filter() { + let filter = Filter { + condition: Some(crate::ast::Condition::new( + "age", + FilterFunction::GreaterThan, + vec![json!(1)], + )), + group: Some(FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::LessThan, + vec![json!(0)], + )])), + negate: None, + }; + let group = FilterGroup::and(vec![filter]); + assert!(evaluate(&group, &json!({"age": 2}), &columns())); + } + + #[test] + fn nesting_past_the_depth_limit_fails_the_whole_filter() { + let deepest = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![json!(1)], + )]); + let nest = |depth: usize| { + let mut group = deepest.clone(); + for _ in 0..depth { + group = FilterGroup::and(vec![Filter::from_group(group)]); + } + group + }; + let row = json!({"age": 2}); + assert!(evaluate(&nest(99), &row, &columns())); + assert!(evaluate(&nest(100), &row, &columns())); + assert!(!evaluate(&nest(101), &row, &columns())); + } + + #[test] + fn retain_matching_keeps_order() { + let group = parse_query_unchecked("[age] > 1").unwrap(); + let rows = vec![ + json!({"age": 3, "name": "c"}), + json!({"age": 1, "name": "a"}), + json!({"age": 2, "name": "b"}), + ]; + let kept = retain_matching(&group, rows.clone(), &columns()); + assert_eq!(kept, vec![rows[0].clone(), rows[2].clone()]); + assert_eq!(count_matches(&group, &rows, &columns()), 2); + } + + #[test] + fn retain_matching_on_an_empty_filter_keeps_everything() { + let rows = vec![json!({"age": 1}), json!({"age": 2})]; + let kept = retain_matching(&FilterGroup::default(), rows.clone(), &columns()); + assert_eq!(kept, rows); + assert_eq!(count_matches(&FilterGroup::default(), &rows, &columns()), 2); + } + + #[test] + fn duplicate_column_names_resolve_to_the_first() { + let cols = vec![ + ColumnDef::new("x", ColumnType::Number), + ColumnDef::new("x", ColumnType::String), + ]; + let group = FilterGroup::and(vec![Filter::condition( + "x", + FilterFunction::GreaterThan, + vec![json!(1)], + )]); + assert!(evaluate(&group, &json!({"x": 2}), &cols)); + } + + #[test] + fn apply_is_usable_on_its_own() { + assert!(apply( + FilterFunction::Contains, + Some(&json!("abc")), + &[json!("b")], + ColumnType::String + )); + assert!(!apply( + FilterFunction::Contains, + None, + &[json!("b")], + ColumnType::String + )); + // No argument means nothing to compare against. + assert!(!apply( + FilterFunction::Equals, + Some(&json!("a")), + &[], + ColumnType::String + )); + assert!(apply( + FilterFunction::IsBlank, + None, + &[], + ColumnType::String + )); + // Only the first argument is read. + assert!(apply( + FilterFunction::Equals, + Some(&json!("a")), + &[json!("a"), json!("b")], + ColumnType::String + )); + } +} diff --git a/rusty-filter/src/lexer.rs b/rusty-filter/src/lexer.rs new file mode 100644 index 0000000..60b0dda --- /dev/null +++ b/rusty-filter/src/lexer.rs @@ -0,0 +1,792 @@ +//! Hand-written lexer for the `Filters.g4` token set. +//! +//! One [`Token`] variant per lexer rule of the grammar. Every rule below was +//! checked against the shipped `filter-query-editor` 2.2.0 bundle, both through +//! `parseQuery` and by driving the generated ANTLR lexer directly; see the crate +//! docs for the probe method. +//! +//! Whitespace (space, tab, CR, LF) is skipped between tokens, which is why +//! `1 . 5` lexes as three tokens and parses as `1.5`. +//! +//! # How lexer errors are shaped +//! +//! ANTLR's lexer walks its DFA as far as the input stays a *viable prefix* of +//! some rule, and its error message quotes everything it walked over plus the +//! character that broke viability. That is why `1e5` complains about `e5` rather +//! than `e`, and why `[age] ! 5` complains about `'! '` — bang and space — since +//! `!` is a viable prefix of `!=` and the space is what rules it out. +//! [`viable_len`] is that walk, and it is the reason the error text is built the +//! way it is instead of just naming the offending character. + +/// A lexed token together with its byte span in the input. +#[derive(Debug, Clone, PartialEq)] +pub struct SpannedToken { + pub token: Token, + /// The raw lexeme, exactly as it appeared in the input: brackets included + /// for a field, quotes included for a string, original case for a keyword. + /// Empty for [`Token::Eof`]. Error messages quote this rather than a + /// canonical spelling, so `[age] GREATER 5` complains about `GREATER`. + pub text: String, + /// Byte offset of the first character of the token. + pub start: usize, + /// Byte offset one past the last character of the token. + pub end: usize, +} + +impl SpannedToken { + /// The spelling an error message uses for this token. + pub fn describe(&self) -> &str { + if self.token == Token::Eof { + "" + } else { + &self.text + } + } +} + +/// The terminals of the grammar. +/// +/// [`Token::Field`] carries the text *between* the brackets, verbatim: no +/// trimming and no unescaping. [`Token::Str`] carries the *unescaped* body of +/// the string literal. Both keep their raw lexeme in [`SpannedToken::text`]. +#[derive(Debug, Clone, PartialEq)] +pub enum Token { + Field(String), + Str(String), + Contains, + Greater, + Starts, + Equals, + Equal, + Blank, + Less, + Than, + Ends, + With, + Not, + Or, + Is, + And, + True, + False, + LParen, + RParen, + Eq, + Eq2, + Neq, + Gt, + Ge, + Lt, + Le, + Dot, + Sign(char), + Digits(String), + Eof, +} + +impl Token { + /// The grammar's name for this token, used where a message names an + /// *expected* token rather than quoting the offending one — as in + /// `missing BLANK at ''`. + pub fn name(&self) -> &'static str { + match self { + Token::Field(_) => "FIELD", + Token::Str(_) => "STRING", + Token::Contains => "CONTAINS", + Token::Greater => "GREATER", + Token::Starts => "STARTS", + Token::Equals => "EQUALS", + Token::Equal => "EQUAL", + Token::Blank => "BLANK", + Token::Less => "LESS", + Token::Than => "THAN", + Token::Ends => "ENDS", + Token::With => "WITH", + Token::Not => "NOT", + Token::Or => "OR", + Token::Is => "IS", + Token::And => "AND", + Token::True => "TRUE", + Token::False => "FALSE", + Token::LParen => "'('", + Token::RParen => "')'", + Token::Eq => "EQUAL_SIGN", + Token::Eq2 => "EQUAL_SIGN2", + Token::Neq => "NOT_EQUAL_SIGN", + Token::Gt => "GT", + Token::Ge => "GTE", + Token::Lt => "LT", + Token::Le => "LTE", + Token::Dot => "DOT", + Token::Sign(_) => "SIGN", + Token::Digits(_) => "DIGITS", + Token::Eof => "", + } + } +} + +/// A lexing failure, with the byte span of the offending text. +#[derive(Debug, Clone, PartialEq)] +pub struct LexError { + pub message: String, + pub start: usize, + pub end: usize, +} + +/// The keyword table. Matching is ASCII-case-insensitive, and the table is also +/// what [`viable_len`] consults to decide whether a partial word could still +/// grow into a keyword. +const KEYWORDS: &[(&str, Token)] = &[ + ("contains", Token::Contains), + ("greater", Token::Greater), + ("starts", Token::Starts), + ("equals", Token::Equals), + ("equal", Token::Equal), + ("blank", Token::Blank), + ("false", Token::False), + ("less", Token::Less), + ("than", Token::Than), + ("ends", Token::Ends), + ("with", Token::With), + ("true", Token::True), + ("not", Token::Not), + ("and", Token::And), + ("or", Token::Or), + ("is", Token::Is), +]; + +/// The multi-character symbolic tokens, longest first. +const SYMBOLS2: &[(&str, Token)] = &[ + ("==", Token::Eq2), + ("!=", Token::Neq), + (">=", Token::Ge), + ("<=", Token::Le), +]; + +/// Lex `input` into tokens, always terminated by [`Token::Eof`]. +/// +/// Returns the first error encountered rather than recovering; see the crate +/// docs on the single-error divergence from the ANTLR reference. +pub fn tokenize(input: &str) -> Result, LexError> { + let bytes = input.as_bytes(); + let mut tokens = Vec::new(); + let mut i = 0usize; + + while i < bytes.len() { + let c = bytes[i]; + // WS: space, tab, CR, LF — skipped, not emitted. + if matches!(c, b' ' | b'\t' | b'\r' | b'\n') { + i += 1; + continue; + } + match longest_match(input, i) { + Some((token, len)) => { + tokens.push(SpannedToken { + token, + text: input[i..i + len].to_string(), + start: i, + end: i + len, + }); + i += len; + } + None => return Err(recognition_error(input, i)), + } + } + + tokens.push(SpannedToken { + token: Token::Eof, + text: String::new(), + start: input.len(), + end: input.len(), + }); + Ok(tokens) +} + +/// The longest complete token starting at byte offset `at`, with its length. +/// +/// Maximal munch: `==` beats `=`, `equals` beats `equal`, and a digit run is +/// taken whole. A word that is only a *prefix* of a keyword matches nothing, +/// which is what makes `end` an error while `ends` is a token. +fn longest_match(input: &str, at: usize) -> Option<(Token, usize)> { + let bytes = input.as_bytes(); + let rest = &input[at..]; + let c = bytes[at]; + + // FIELD: '[' ~[\r\n\]]+ ']' + if c == b'[' { + let mut j = at + 1; + while j < bytes.len() { + match bytes[j] { + b'\r' | b'\n' => return None, + // The `+` needs at least one inner character, so `[]` is not a + // field. + b']' if j > at + 1 => { + return Some((Token::Field(input[at + 1..j].to_string()), j + 1 - at)); + } + b']' => return None, + _ => j += 1, + } + } + return None; + } + + // STRING: '"' ( '\\' . | ~[\\"\r\n] )* '"' + if c == b'"' { + let mut j = at + 1; + let mut body = String::new(); + while j < bytes.len() { + match bytes[j] { + // An escape consumes the next character whatever it is, so a + // trailing backslash swallows the closing quote. + b'\\' => match char_at(input, j + 1) { + Some((ch, width)) => { + body.push('\\'); + body.push(ch); + j += 1 + width; + } + None => return None, + }, + b'"' => return Some((Token::Str(unescape(&body)), j + 1 - at)), + b'\r' | b'\n' => return None, + _ => { + let (ch, width) = char_at(input, j).expect("index inside input"); + body.push(ch); + j += width; + } + } + } + return None; + } + + // Two-character operators before their one-character prefixes. + for (text, token) in SYMBOLS2 { + if rest.starts_with(text) { + return Some((token.clone(), text.len())); + } + } + let single = match c { + b'(' => Some(Token::LParen), + b')' => Some(Token::RParen), + b'=' => Some(Token::Eq), + b'>' => Some(Token::Gt), + b'<' => Some(Token::Lt), + b'.' => Some(Token::Dot), + b'+' => Some(Token::Sign('+')), + b'-' => Some(Token::Sign('-')), + _ => None, + }; + if let Some(token) = single { + return Some((token, 1)); + } + + // DIGITS: [0-9]+ + if c.is_ascii_digit() { + let len = bytes[at..] + .iter() + .take_while(|b| b.is_ascii_digit()) + .count(); + return Some((Token::Digits(input[at..at + len].to_string()), len)); + } + + // Keywords, longest complete spelling wins. + let mut best: Option<(Token, usize)> = None; + for (word, token) in KEYWORDS { + if rest.len() >= word.len() && rest[..word.len()].eq_ignore_ascii_case(word) { + let better = best.as_ref().is_none_or(|(_, len)| word.len() > *len); + if better { + best = Some((token.clone(), word.len())); + } + } + } + best +} + +/// How many bytes from `at` stay a viable prefix of some lexer rule. +/// +/// Zero for a character that starts no rule at all. This is the DFA walk ANTLR +/// performs before it gives up, and the length its error message is built from. +fn viable_len(input: &str, at: usize) -> usize { + let bytes = input.as_bytes(); + let rest = &input[at..]; + let c = bytes[at]; + + // A field or string stays viable until a forbidden character appears; the + // whole remainder is viable if it simply runs out unterminated. + if c == b'[' { + let mut j = at + 1; + while j < bytes.len() && !matches!(bytes[j], b'\r' | b'\n' | b']') { + j += 1; + } + return j - at; + } + if c == b'"' { + let mut j = at + 1; + while j < bytes.len() { + match bytes[j] { + b'\\' => match char_at(input, j + 1) { + Some((_, width)) => j += 1 + width, + None => { + j += 1; + break; + } + }, + b'\r' | b'\n' => break, + _ => j += char_at(input, j).expect("index inside input").1, + } + } + return j - at; + } + + // `!` alone is viable only because `!=` exists. + if c == b'!' { + return 1; + } + // Every other symbolic token and a digit run are complete as soon as they + // start, so nothing beyond `longest_match` is viable. + if let Some((_, len)) = longest_match(input, at) { + if !c.is_ascii_alphabetic() { + return len; + } + } + + // A word is viable while it is a case-insensitive prefix of some keyword. + let mut best = 0; + for (word, _) in KEYWORDS { + let mut shared = 0; + while shared < word.len() + && shared < rest.len() + && rest.as_bytes()[shared].eq_ignore_ascii_case(&word.as_bytes()[shared]) + { + shared += 1; + } + best = best.max(shared); + } + best +} + +/// Build the `token recognition error` the reference reports at `at`. +/// +/// The quoted text runs from `at` over every viable byte plus the character that +/// broke viability, or to the end of the input if viability ran out there. The +/// *span*, by contrast, covers only the first character: the reference's error +/// listener gets no token for a lexer error and falls back to a one-unit span at +/// the reported position. +fn recognition_error(input: &str, at: usize) -> LexError { + let viable = viable_len(input, at); + let end = match char_at(input, at + viable) { + Some((_, width)) => at + viable + width, + None => input.len(), + }; + let (_, first_width) = char_at(input, at).expect("index inside input"); + LexError { + message: format!("token recognition error at: '{}'", &input[at..end]), + start: at, + end: at + first_width, + } +} + +/// The character starting at byte offset `at`, with its UTF-8 width. +fn char_at(input: &str, at: usize) -> Option<(char, usize)> { + input + .get(at..) + .and_then(|s| s.chars().next()) + .map(|c| (c, c.len_utf8())) +} + +/// Unescape a string literal body exactly as `extractStringValue` does. +/// +/// The reference applies three ordered replacements to the quoted body: +/// `\"` then `\'` then `\\`. Only those three escapes are recognised; every +/// other escape **keeps its backslash**, so `\t` stays as backslash-t and +/// `\z` stays as backslash-z. Because the replacements run left to right over +/// the whole string rather than as a single scan, a `\\` pair is only collapsed +/// after the quote escapes have been rewritten — which is why `\\'` yields +/// backslash-quote rather than backslash-backslash-quote. +fn unescape(body: &str) -> String { + let stage1 = body.replace("\\\"", "\""); + let stage2 = stage1.replace("\\'", "'"); + stage2.replace("\\\\", "\\") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn toks(input: &str) -> Vec { + tokenize(input) + .expect("expected the input to lex") + .into_iter() + .map(|t| t.token) + .collect() + } + + fn err(input: &str) -> LexError { + tokenize(input).expect_err("expected the input to fail lexing") + } + + #[test] + fn lexes_every_token_variant() { + let tokens = toks( + "[f] \"s\" contains greater starts equals equal blank less than ends with \ + not or is and true false ( ) = == != > >= < <= . + - 12", + ); + assert_eq!( + tokens, + vec![ + Token::Field("f".to_string()), + Token::Str("s".to_string()), + Token::Contains, + Token::Greater, + Token::Starts, + Token::Equals, + Token::Equal, + Token::Blank, + Token::Less, + Token::Than, + Token::Ends, + Token::With, + Token::Not, + Token::Or, + Token::Is, + Token::And, + Token::True, + Token::False, + Token::LParen, + Token::RParen, + Token::Eq, + Token::Eq2, + Token::Neq, + Token::Gt, + Token::Ge, + Token::Lt, + Token::Le, + Token::Dot, + Token::Sign('+'), + Token::Sign('-'), + Token::Digits("12".to_string()), + Token::Eof, + ] + ); + } + + #[test] + fn field_keeps_inner_text_verbatim() { + assert_eq!(toks("[a b]")[0], Token::Field("a b".to_string())); + assert_eq!(toks("[a[b]")[0], Token::Field("a[b".to_string())); + assert_eq!(toks("[café]")[0], Token::Field("café".to_string())); + assert_eq!(toks("[a\tb]")[0], Token::Field("a\tb".to_string())); + // No trimming: a single space is a legal one-character column name. + assert_eq!(toks("[ ]")[0], Token::Field(" ".to_string())); + assert_eq!(toks("[ s ]")[0], Token::Field(" s ".to_string())); + } + + #[test] + fn raw_text_is_kept_alongside_the_token() { + let tokens = tokenize("[age] GREATER \"x\"").unwrap(); + assert_eq!(tokens[0].text, "[age]"); + // Original case, so an error message can quote what was typed. + assert_eq!(tokens[1].text, "GREATER"); + assert_eq!(tokens[2].text, "\"x\""); + assert_eq!(tokens[3].describe(), ""); + assert_eq!(tokens[3].text, ""); + } + + #[test] + fn empty_field_is_rejected() { + // Viability stops after `[`, so the `]` that broke it is quoted too. + let e = err("[] = \"x\""); + assert_eq!(e.message, "token recognition error at: '[]'"); + assert_eq!((e.start, e.end), (0, 1)); + } + + #[test] + fn field_with_escaped_bracket_is_rejected() { + // `FIELD` has no escape rule, so the backslash-bracket closes the field + // and the remainder fails, exactly as in the bundle. + let tokens = tokenize("[a\\]").unwrap(); + assert_eq!(tokens[0].token, Token::Field("a\\".to_string())); + let e = err("[a\\]b] = \"x\""); + assert_eq!(e.message, "token recognition error at: 'b]'"); + assert_eq!((e.start, e.end), (4, 5)); + } + + #[test] + fn field_with_raw_newline_is_rejected() { + let e = err("[a\nb] = \"x\""); + assert_eq!(e.message, "token recognition error at: '[a\n'"); + assert_eq!((e.start, e.end), (0, 1)); + let e = err("[a\rb] = \"x\""); + assert_eq!(e.message, "token recognition error at: '[a\r'"); + } + + #[test] + fn unterminated_field_quotes_the_whole_remainder() { + let e = err("[age"); + assert_eq!(e.message, "token recognition error at: '[age'"); + assert_eq!((e.start, e.end), (0, 1)); + } + + #[test] + fn string_escapes_follow_the_reference() { + // Only the three recognised escapes are rewritten. + assert_eq!(toks("\"a\\\"b\"")[0], Token::Str("a\"b".to_string())); + assert_eq!(toks("\"a\\'b\"")[0], Token::Str("a'b".to_string())); + assert_eq!(toks("\"a\\\\b\"")[0], Token::Str("a\\b".to_string())); + // Everything else keeps its backslash. + assert_eq!(toks("\"a\\tb\"")[0], Token::Str("a\\tb".to_string())); + assert_eq!(toks("\"a\\nb\"")[0], Token::Str("a\\nb".to_string())); + assert_eq!(toks("\"a\\zb\"")[0], Token::Str("a\\zb".to_string())); + } + + #[test] + fn string_escape_replacement_order_is_sequential() { + // `\\'` collapses to backslash-quote because the `\'` rewrite runs + // before the `\\` one; a single scan would give backslash-backslash-quote. + assert_eq!(toks("\"\\\\'\"")[0], Token::Str("\\'".to_string())); + assert_eq!(toks("\"\\\\\\\\'\"")[0], Token::Str("\\\\'".to_string())); + assert_eq!(toks("\"\\\\t\"")[0], Token::Str("\\t".to_string())); + } + + #[test] + fn raw_newline_in_string_is_rejected() { + let e = err("[s] = \"a\nb\""); + assert_eq!(e.message, "token recognition error at: '\"a\n'"); + assert_eq!((e.start, e.end), (6, 7)); + } + + #[test] + fn raw_tab_in_string_is_accepted() { + assert_eq!(toks("\"a\tb\"")[0], Token::Str("a\tb".to_string())); + } + + #[test] + fn unterminated_string_is_rejected() { + let e = err("\"abc"); + assert_eq!(e.message, "token recognition error at: '\"abc'"); + // A trailing backslash consumes the closing quote. + let e = err("[s] = \"a\\\""); + assert_eq!(e.message, "token recognition error at: '\"a\\\"'"); + assert_eq!((e.start, e.end), (6, 7)); + } + + #[test] + fn single_quoted_string_is_rejected() { + let e = err("[name] = 'x'"); + assert_eq!(e.message, "token recognition error at: '''"); + assert_eq!((e.start, e.end), (9, 10)); + } + + #[test] + fn two_character_operators_win_maximal_munch() { + assert_eq!(toks("==")[0], Token::Eq2); + assert_eq!(toks("!=")[0], Token::Neq); + assert_eq!(toks(">=")[0], Token::Ge); + assert_eq!(toks("<=")[0], Token::Le); + } + + #[test] + fn spaced_two_character_operator_lexes_as_two_tokens() { + assert_eq!(toks("> ="), vec![Token::Gt, Token::Eq, Token::Eof]); + assert_eq!(toks("< ="), vec![Token::Lt, Token::Eq, Token::Eof]); + assert_eq!(toks("= ="), vec![Token::Eq, Token::Eq, Token::Eof]); + } + + #[test] + fn lone_bang_is_rejected_and_quotes_the_next_character() { + // `!` is a viable prefix of `!=`, so the space that rules it out is part + // of the quoted text. + let e = err("[age] ! 5"); + assert_eq!(e.message, "token recognition error at: '! '"); + assert_eq!((e.start, e.end), (6, 7)); + assert_eq!(err("[age] !x").message, "token recognition error at: '!x'"); + assert_eq!(err("!").message, "token recognition error at: '!'"); + } + + #[test] + fn exponent_syntax_is_rejected() { + // `e` could still become `equal`, `equals` or `ends`, so the `5` that + // rules all three out is quoted with it. + let e = err("[age] = 1e5"); + assert_eq!(e.message, "token recognition error at: 'e5'"); + assert_eq!((e.start, e.end), (9, 10)); + assert_eq!( + err("[age] = 1E5").message, + "token recognition error at: 'E5'" + ); + assert_eq!( + err("[age] = 1ee5").message, + "token recognition error at: 'ee'" + ); + assert_eq!(err("[age] = 1e").message, "token recognition error at: 'e'"); + } + + #[test] + fn a_keyword_followed_by_letters_lexes_then_fails() { + // Maximal munch takes the keyword, then the leftover letters fail — so + // the input as a whole does not lex and the error names only the tail. + // `tokenize` reports that first failure instead of the tokens preceding + // it, which is why this asserts through `err` and not `toks`. + let e = err("andx"); + assert_eq!(e.message, "token recognition error at: 'x'"); + assert_eq!((e.start, e.end), (3, 4)); + assert_eq!(err("notx").message, "token recognition error at: 'x'"); + assert_eq!(err("equalx").message, "token recognition error at: 'x'"); + assert_eq!(err("equalsx").message, "token recognition error at: 'x'"); + assert_eq!(err("blanks").message, "token recognition error at: 's'"); + // `a` is viable as a prefix of `and`, so the space is quoted with it. + assert_eq!(err("nota ").message, "token recognition error at: 'a '"); + } + + #[test] + fn adjacent_keywords_lex_without_separators() { + assert_eq!(toks("andor"), vec![Token::And, Token::Or, Token::Eof]); + assert_eq!(toks("isnot"), vec![Token::Is, Token::Not, Token::Eof]); + assert_eq!( + toks("isnotblank"), + vec![Token::Is, Token::Not, Token::Blank, Token::Eof] + ); + assert_eq!( + toks("lessthanorequal"), + vec![ + Token::Less, + Token::Than, + Token::Or, + Token::Equal, + Token::Eof + ] + ); + assert_eq!( + toks("startswith"), + vec![Token::Starts, Token::With, Token::Eof] + ); + } + + #[test] + fn a_keyword_prefix_alone_is_an_error() { + // Measured: `ends` lexes, `end`, `en` and `e` do not. + assert_eq!(toks("ends"), vec![Token::Ends, Token::Eof]); + assert_eq!(err("end").message, "token recognition error at: 'end'"); + assert_eq!(err("en").message, "token recognition error at: 'en'"); + assert_eq!(err("e").message, "token recognition error at: 'e'"); + } + + #[test] + fn an_unviable_character_quotes_only_itself() { + for (input, text) in [ + ("z", "z"), + ("q", "q"), + ("@", "@"), + ("#", "#"), + ("~", "~"), + ("zz", "z"), + ] { + assert_eq!( + err(input).message, + format!("token recognition error at: '{text}'"), + "{input}" + ); + } + // But a viable prefix drags in the character that broke it. + assert_eq!(err("ab").message, "token recognition error at: 'ab'"); + assert_eq!(err("abc").message, "token recognition error at: 'ab'"); + assert_eq!(err("an ").message, "token recognition error at: 'an '"); + assert_eq!(err("sta ").message, "token recognition error at: 'sta '"); + assert_eq!(err("i ").message, "token recognition error at: 'i '"); + } + + #[test] + fn keywords_are_case_insensitive() { + for spelling in ["AND", "and", "AnD"] { + assert_eq!(toks(spelling)[0], Token::And); + } + for spelling in ["CONTAINS", "contains", "CoNtAiNs"] { + assert_eq!(toks(spelling)[0], Token::Contains); + } + assert_eq!(toks("TRUE")[0], Token::True); + assert_eq!(toks("False")[0], Token::False); + } + + #[test] + fn equals_wins_over_equal() { + assert_eq!(toks("equals"), vec![Token::Equals, Token::Eof]); + assert_eq!(toks("equal"), vec![Token::Equal, Token::Eof]); + } + + #[test] + fn digits_are_grouped_and_dot_is_separate() { + assert_eq!( + toks("1 . 5"), + vec![ + Token::Digits("1".to_string()), + Token::Dot, + Token::Digits("5".to_string()), + Token::Eof, + ] + ); + assert_eq!(toks("007")[0], Token::Digits("007".to_string())); + } + + #[test] + fn keywords_need_no_surrounding_space() { + assert_eq!( + toks("[age]>1AND[age]<5"), + vec![ + Token::Field("age".to_string()), + Token::Gt, + Token::Digits("1".to_string()), + Token::And, + Token::Field("age".to_string()), + Token::Lt, + Token::Digits("5".to_string()), + Token::Eof, + ] + ); + } + + #[test] + fn whitespace_is_skipped() { + assert_eq!(toks(" \t\r\n "), vec![Token::Eof]); + assert_eq!(toks(""), vec![Token::Eof]); + } + + #[test] + fn spans_are_byte_offsets() { + let tokens = tokenize("[café] = \"x\"").unwrap(); + assert_eq!((tokens[0].start, tokens[0].end), (0, 7)); + assert_eq!((tokens[1].start, tokens[1].end), (8, 9)); + assert_eq!((tokens[2].start, tokens[2].end), (10, 13)); + assert_eq!(tokens[3].token, Token::Eof); + assert_eq!(tokens[3].start, "[café] = \"x\"".len()); + } + + #[test] + fn a_multibyte_offending_character_gets_a_whole_char_span() { + // The reference would report a one-code-unit span here; a byte-offset + // port reports the character's full width so the span stays sliceable. + let e = err("é"); + assert_eq!(e.message, "token recognition error at: 'é'"); + assert_eq!((e.start, e.end), (0, 2)); + assert_eq!(&"é"[e.start..e.end], "é"); + } + + #[test] + fn token_names_are_the_grammars() { + assert_eq!(Token::Blank.name(), "BLANK"); + assert_eq!(Token::With.name(), "WITH"); + assert_eq!(Token::RParen.name(), "')'"); + assert_eq!(Token::Digits("1".to_string()).name(), "DIGITS"); + assert_eq!(Token::Eof.name(), ""); + } + + #[test] + fn viability_is_zero_for_a_character_that_starts_nothing() { + assert_eq!(viable_len("z", 0), 0); + assert_eq!(viable_len("!", 0), 1); + assert_eq!(viable_len("e", 0), 1); + assert_eq!(viable_len("end", 0), 3); + assert_eq!(viable_len("ends", 0), 4); + assert_eq!(viable_len("[ab", 0), 3); + assert_eq!(viable_len("=", 0), 1); + assert_eq!(viable_len("12", 0), 2); + } +} diff --git a/rusty-filter/src/lib.rs b/rusty-filter/src/lib.rs new file mode 100644 index 0000000..232df4b --- /dev/null +++ b/rusty-filter/src/lib.rs @@ -0,0 +1,92 @@ +//! Filter query grammar, AST, validator, evaluator and printer. +//! +//! Rusty's `DataTable` renders the same filter query editor Ivy uses, so a query +//! typed in the browser has to mean the same thing on the server. This crate is +//! the Rust half: it parses a query string into the same AST, validates it +//! against a column schema, evaluates it against rows, and prints it back to a +//! canonical string suitable for a cache key. +//! +//! # Where the grammar comes from +//! +//! The grammar is not invented here. It is fixed by `docs/grammar/Filters.g4` in +//! `Ivy-Interactive/Ivy-Query-Editor` and by the shipped `filter-query-editor` +//! bundle under `src/frontend/node_modules/`. Behaviour was matched against +//! version **2.2.0** by calling `parseQuery`, `validateFilters`, `formatQuery` +//! and `evaluateFilter` over roughly a hundred inputs. Nothing in this crate +//! links against an ANTLR runtime: the lexer and the recursive-descent parser +//! are hand-written, and the only dependencies are `serde` and `serde_json`. +//! +//! # Getting started +//! +//! ``` +//! use rusty_filter::{parse_query, retain_matching, ColumnDef, ColumnType}; +//! +//! let columns = vec![ +//! ColumnDef::new("age", ColumnType::Number), +//! ColumnDef::new("name", ColumnType::String), +//! ]; +//! let result = parse_query("[age] > 30 AND [name] starts with \"A\"", &columns); +//! let filter = result.filters.expect("the query is valid"); +//! +//! let rows = vec![ +//! serde_json::json!({"age": 41, "name": "Ada"}), +//! serde_json::json!({"age": 41, "name": "Bob"}), +//! serde_json::json!({"age": 12, "name": "Ann"}), +//! ]; +//! let kept = retain_matching(&filter, rows, &columns); +//! assert_eq!(kept.len(), 1); +//! ``` +//! +//! An invalid query returns its errors instead: +//! +//! ``` +//! use rusty_filter::{parse_query, ColumnDef, ColumnType}; +//! +//! let columns = vec![ColumnDef::new("age", ColumnType::Number)]; +//! let result = parse_query("[age] contains \"3\"", &columns); +//! assert!(result.has_errors()); +//! assert_eq!( +//! result.errors()[0].message, +//! "Operator 'contains' is not compatible with type 'number'" +//! ); +//! ``` +//! +//! # Known divergences from the reference +//! +//! Four, all deliberate, each restated where it applies: +//! +//! * **Error spans are byte offsets, not UTF-16 code units.** The two agree for +//! ASCII queries and part company after a non-BMP character. Anything +//! highlighting a span from a Rust-side error has to account for that. +//! * **A syntax error stops the parse.** ANTLR recovers and can report a +//! cascade; here the first error is the only one. Semantic errors are still +//! reported in full, one per offending condition. +//! * **Evaluation uses the normalized column type.** The reference validates +//! against a normalized type but evaluates against the raw one, so a column +//! declared `INT32` passes validation for `>` and then silently matches +//! nothing. [`ColumnDef`] stores a normalized [`ColumnType`], so the same +//! filter matches here. See [`eval`]. +//! * **A datetime with no `Z` is read as UTC.** The browser reads it in the +//! machine's local zone, which would make server-side filtering depend on the +//! server's timezone. See [`eval`]. +//! +//! # Module layout +//! +//! [`lexer`] turns a query into tokens, [`parser`] turns tokens into the +//! [`ast`], [`validate`] checks that AST against a [`column`] schema, +//! [`eval`] runs it over rows, and [`print`] turns it back into text. + +pub mod ast; +pub mod column; +pub mod eval; +pub mod lexer; +pub mod parser; +pub mod print; +pub mod validate; + +pub use ast::{Condition, Filter, FilterFunction, FilterGroup, LogicalOp}; +pub use column::{ColumnDef, ColumnType}; +pub use eval::{count_matches, evaluate, retain_matching}; +pub use parser::{parse_query, parse_query_unchecked, ErrorSeverity, ParseError, ParseResult}; +pub use print::{canonical_key, to_query_string}; +pub use validate::validate_filter_group; diff --git a/rusty-filter/src/parser.rs b/rusty-filter/src/parser.rs new file mode 100644 index 0000000..1b78a61 --- /dev/null +++ b/rusty-filter/src/parser.rs @@ -0,0 +1,1617 @@ +//! Recursive-descent parser for the filter grammar, one function per parser rule. +//! +//! Precedence, from loosest to tightest: `OR`, `AND`, `NOT`, primary. The +//! AST-shaping rules are those of `dist/parser/ASTBuilder.js` and are reproduced +//! rather than simplified — in particular parentheses are never collapsed and +//! `NOT` *toggles* negation instead of forcing it true. +//! +//! # How error messages are worded +//! +//! ANTLR has four error shapes, and which one appears is decided mechanically. +//! Reproducing that decision is the only way a Rust-side message can read like +//! the browser's, so each rule below is implemented rather than approximated: +//! +//! * `extraneous input 'X' expecting T` — deleting the current token would let +//! the match succeed. +//! * `missing T at 'X'` — the current token could legally *follow* the token +//! that is missing, so one is assumed to have been left out. +//! * `mismatched input 'X' expecting T` — neither repair works. +//! * `no viable alternative at input 'X'` — a decision needing more than one +//! token of lookahead ran out of alternatives. `X` is every token's raw text +//! from the start of that decision onwards, run together. +//! +//! Deletion is tried before insertion, which is why `[age] = --5` blames the +//! extra sign instead of reporting a missing digit. Messages quote the raw +//! lexeme, so `[age] GREATER 5` complains about `GREATER`, not `greater`. +//! +//! # Divergences from the reference +//! +//! * `start` and `end` in [`ParseError`] are **byte** offsets into the input; +//! the TypeScript version reports UTF-16 code units. They agree for ASCII +//! input and differ once a multi-byte character appears before the error. +//! * On a syntax error this parser stops and reports **one** error. The ANTLR +//! reference recovers and can emit a cascade of several errors for one input; +//! matching that cascade is out of scope. The first error agrees. + +use serde::{Deserialize, Serialize}; + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup}; +use crate::column::ColumnDef; +use crate::lexer::{tokenize, SpannedToken, Token}; +use crate::validate::validate_filter_group; + +/// How serious a [`ParseError`] is. The reference emits `error` for everything +/// this crate produces; `Warning` exists to mirror the TypeScript union. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ErrorSeverity { + #[default] + Error, + Warning, +} + +/// A syntax or semantic error, with the byte span it applies to. +/// +/// Semantic errors carry `start: 0` and `end: 0` because the reference +/// validator has no position information to report — that is faithful, not a +/// defect to be "fixed". +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ParseError { + pub message: String, + pub start: usize, + pub end: usize, + pub severity: ErrorSeverity, +} + +impl ParseError { + pub fn new(message: impl Into, start: usize, end: usize) -> Self { + ParseError { + message: message.into(), + start, + end, + severity: ErrorSeverity::Error, + } + } +} + +/// The result of parsing: filters *or* errors, never both, mirroring +/// `dist/types/parser.d.ts` and the early returns of `parseQuery`. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct ParseResult { + #[serde(skip_serializing_if = "Option::is_none")] + pub filters: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub errors: Option>, +} + +impl ParseResult { + fn ok(filters: FilterGroup) -> Self { + ParseResult { + filters: Some(filters), + errors: None, + } + } + + fn failed(errors: Vec) -> Self { + ParseResult { + filters: None, + errors: Some(errors), + } + } + + /// Whether the parse produced errors. + pub fn has_errors(&self) -> bool { + self.errors.as_ref().is_some_and(|e| !e.is_empty()) + } + + /// The errors, or an empty slice on success. + pub fn errors(&self) -> &[ParseError] { + self.errors.as_deref().unwrap_or(&[]) + } +} + +/// Parse `query` and validate it against `columns`. +/// +/// Whitespace-only and empty input yield an empty `AND` group with no errors. +/// Syntax errors are reported before semantic ones, and semantic validation only +/// runs on a syntactically valid query — the same order `parseQuery` uses. +pub fn parse_query(query: &str, columns: &[ColumnDef]) -> ParseResult { + let group = match parse_query_unchecked(query) { + Ok(group) => group, + Err(errors) => return ParseResult::failed(errors), + }; + let semantic = validate_filter_group(&group, columns); + if semantic.is_empty() { + ParseResult::ok(group) + } else { + ParseResult::failed(semantic) + } +} + +/// Parse `query` without semantic validation, so no column schema is needed. +pub fn parse_query_unchecked(query: &str) -> Result> { + if query.trim().is_empty() { + return Ok(FilterGroup::default()); + } + let tokens = tokenize(query).map_err(|e| vec![ParseError::new(e.message, e.start, e.end)])?; + let mut parser = Parser::new(&tokens); + let group = parser.formula().map_err(|e| vec![e])?; + Ok(group) +} + +/// The result of visiting one rule: either a bare filter or a whole group, +/// exactly the untagged union `ASTBuilder`'s visitors return. +#[derive(Debug, Clone)] +enum Node { + Filter(Filter), + Group(FilterGroup), +} + +impl Node { + /// Wrap into a single `Filter`, as `visitGroup` does when the inner + /// expression came back as a group. + fn into_filter(self) -> Filter { + match self { + Node::Filter(f) => f, + Node::Group(g) => Filter::from_group(g), + } + } + + /// Lift into a `FilterGroup`, as `visitAndExpr` does for a single arm. + fn into_group(self) -> FilterGroup { + match self { + Node::Filter(f) => FilterGroup::and(vec![f]), + Node::Group(g) => g, + } + } + + /// Toggle negation on whichever variant this is. `visitUnaryExpr` sets + /// `negate = !negate` on the visited node, and a group node carries its own + /// `negate` only once it has been wrapped in a filter. + fn toggle_negate(self) -> Node { + match self { + Node::Filter(mut f) => { + f.negate = Some(!f.is_negated()); + Node::Filter(f) + } + Node::Group(g) => { + // A raw group has no `negate` field of its own, so the reference + // sets one on the object it is holding. Wrapping it in a filter + // is the faithful Rust equivalent: `NOT (a AND b)` becomes a + // group filter with `negate: true`. + let mut f = Filter::from_group(g); + f.negate = Some(true); + Node::Filter(f) + } + } + } +} + +struct Parser<'a> { + tokens: &'a [SpannedToken], + pos: usize, + /// How many `(` are currently open. Only the error wording reads this: a + /// token's legal followers depend on the enclosing groups, so `)` follows a + /// completed primary only inside one and `` only outside them all. + depth: usize, +} + +impl<'a> Parser<'a> { + fn new(tokens: &'a [SpannedToken]) -> Self { + Parser { + tokens, + pos: 0, + depth: 0, + } + } + + fn peek(&self) -> &'a Token { + &self.tokens[self.pos.min(self.tokens.len() - 1)].token + } + + fn peek_at(&self, offset: usize) -> &'a Token { + let idx = (self.pos + offset).min(self.tokens.len() - 1); + &self.tokens[idx].token + } + + fn current(&self) -> &'a SpannedToken { + &self.tokens[self.pos.min(self.tokens.len() - 1)] + } + + fn advance(&mut self) -> &'a SpannedToken { + let tok = self.current(); + if self.pos < self.tokens.len() - 1 { + self.pos += 1; + } + tok + } + + fn eat(&mut self, expected: &Token) -> bool { + if self.peek() == expected { + self.advance(); + true + } else { + false + } + } + + fn error(&self, message: impl Into) -> ParseError { + let tok = self.current(); + ParseError::new(message, tok.start, tok.end) + } + + /// Match a single expected token, or word the failure as ANTLR would. + /// + /// `expecting` names the token, `matches` recognises it (needed because + /// `STRING` and `DIGITS` carry payloads), and `follows` says whether a token + /// may legally appear *after* the expected one — which is what licenses the + /// `missing` wording. Deletion is tried before insertion. + fn expect( + &mut self, + expecting: &str, + matches: fn(&Token) -> bool, + follows: Follows, + ) -> Result<&'a SpannedToken, ParseError> { + if matches(self.peek()) { + return Ok(self.advance()); + } + let quoted = self.current().describe(); + if matches(self.peek_at(1)) { + return Err(self.error(format!("extraneous input '{quoted}' expecting {expecting}"))); + } + if follows(self.peek(), self.depth) { + return Err(self.error(format!("missing {expecting} at '{quoted}'"))); + } + Err(self.error(format!("mismatched input '{quoted}' expecting {expecting}"))) + } + + /// The failure for a position that expects one of a *set* of tokens. + /// + /// No `missing` wording here: with several candidates the reference never + /// picks one to insert, so only deletion is offered. + fn expected_set(&self, expecting: &str, matches: fn(&Token) -> bool) -> ParseError { + let quoted = self.current().describe(); + if matches(self.peek_at(1)) { + self.error(format!("extraneous input '{quoted}' expecting {expecting}")) + } else { + self.error(format!("mismatched input '{quoted}' expecting {expecting}")) + } + } + + /// The failure for a decision that ran out of alternatives. + /// + /// The quoted text is every token's raw spelling from `from` through the + /// token where prediction gave up, run together with no separators — so + /// `[age] not blank` reports `'[age]notblank'`. The span is that last + /// token's. + fn no_viable(&self, from: usize, failed_at: usize) -> ParseError { + let last = failed_at.min(self.tokens.len() - 1); + let text: String = self.tokens[from..=last] + .iter() + .map(|t| t.text.as_str()) + .collect(); + let tok = &self.tokens[last]; + ParseError::new( + format!("no viable alternative at input '{text}'"), + tok.start, + tok.end, + ) + } + + /// `formula : expr EOF` + fn formula(&mut self) -> Result { + let node = self.expr()?; + self.expect("", is_eof, never)?; + Ok(node.into_group()) + } + + /// `expr : orExpr` + fn expr(&mut self) -> Result { + self.or_expr() + } + + /// `orExpr : andExpr (OR andExpr)*` + /// + /// A single arm is returned unchanged. Several arms produce an `OR` group in + /// which each arm that came back as a group is spliced in bare when it holds + /// exactly one filter, and wrapped as `{group: ...}` otherwise. + fn or_expr(&mut self) -> Result { + let first = self.and_expr()?; + if self.peek() != &Token::Or { + return Ok(first); + } + let mut arms = vec![first]; + while self.eat(&Token::Or) { + arms.push(self.and_expr()?); + } + let mut filters = Vec::with_capacity(arms.len()); + for arm in arms { + match arm { + Node::Group(group) => { + if group.filters.len() == 1 { + filters.push(group.filters.into_iter().next().expect("length checked")); + } else { + filters.push(Filter::from_group(group)); + } + } + Node::Filter(f) => filters.push(f), + } + } + Ok(Node::Group(FilterGroup::or(filters))) + } + + /// `andExpr : unaryExpr (AND unaryExpr)*` + /// + /// A single arm that is already a group is returned as-is; a single filter is + /// wrapped in an `AND` group. Several arms are pushed as-is with no splicing. + fn and_expr(&mut self) -> Result { + let first = self.unary_expr()?; + if self.peek() != &Token::And { + return Ok(Node::Group(first.into_group())); + } + let mut filters = vec![first.into_filter()]; + while self.eat(&Token::And) { + filters.push(self.unary_expr()?.into_filter()); + } + Ok(Node::Group(FilterGroup::and(filters))) + } + + /// `unaryExpr : NOT unaryExpr | primary` + fn unary_expr(&mut self) -> Result { + if self.eat(&Token::Not) { + let inner = self.unary_expr()?; + return Ok(inner.toggle_negate()); + } + self.primary() + } + + /// `primary : group | comparison | textOperation | existenceOperation` + /// + /// The three field-led alternatives are chosen by looking past the field + /// reference and an optional `NOT`. When no alternative survives that + /// lookahead the failure is a `no viable alternative` quoting from the field, + /// because the field is where all three alternatives begin. + fn primary(&mut self) -> Result { + if self.peek() == &Token::LParen { + return self.group(); + } + let field = self.pos; + let Token::Field(column) = self.peek().clone() else { + return Err(self.expected_set("{FIELD, NOT, '('}", starts_primary)); + }; + match self.peek_at(1) { + Token::Contains | Token::Starts | Token::Ends => { + self.advance(); + self.text_operation(column) + } + Token::Is => { + self.advance(); + self.existence_operation(column, field) + } + // After `NOT` only negated equality and the text operators remain. + Token::Not => match self.peek_at(2) { + Token::Equals | Token::Equal => { + self.advance(); + self.comparison(column) + } + Token::Contains | Token::Starts | Token::Ends => { + self.advance(); + self.text_operation(column) + } + _ => Err(self.no_viable(field, self.pos + 2)), + }, + next if starts_comp_op(next) => { + self.advance(); + self.comparison(column) + } + _ => Err(self.no_viable(field, self.pos + 1)), + } + } + + /// `group : LPAREN expr RPAREN` + /// + /// Parentheses are never collapsed: the inner group is wrapped as a group + /// filter, so `(([age] > 1))` keeps both levels. + fn group(&mut self) -> Result { + self.advance(); // LPAREN + self.depth += 1; + let inner = self.expr()?; + // The `)` this rule is about is the one being closed, so its own follow + // set is that of the *enclosing* group: `([age] > 1` reports a missing + // `)` at ``, while `(([age] > 1)` reports the same at depth 1. + self.depth -= 1; + self.expect("')'", is_rparen, ends_primary)?; + Ok(match inner { + Node::Group(g) => Node::Filter(Filter::from_group(g)), + Node::Filter(f) => Node::Filter(f), + }) + } + + /// `comparison : fieldRef compOp operand` + /// + /// `negate` is `Some(true)` for the three not-equal spellings and `None` + /// otherwise — comparisons never emit `negate: false`. + fn comparison(&mut self, column: String) -> Result { + let (function, negate) = self.comp_op(self.pos)?; + let value = self.operand()?; + let mut filter = Filter { + condition: Some(Condition::new(column, function, vec![value])), + group: None, + negate: None, + }; + if negate { + filter.negate = Some(true); + } + Ok(Node::Filter(filter)) + } + + /// `compOp` — returns the mapped function and whether it negates. + /// + /// `op_start` is where a `no viable alternative` message should quote from: + /// the word operators are their own decision, so `[age] greater 5` quotes + /// `'greater5'` and not the field, while a failure that only becomes visible + /// here is still attributed to the whole primary. + fn comp_op(&mut self, op_start: usize) -> Result<(FilterFunction, bool), ParseError> { + let tok = self.peek().clone(); + match tok { + Token::Eq | Token::Eq2 => { + self.advance(); + Ok((FilterFunction::Equals, false)) + } + Token::Neq => { + self.advance(); + Ok((FilterFunction::Equals, true)) + } + Token::Gt => { + self.advance(); + Ok((FilterFunction::GreaterThan, false)) + } + Token::Ge => { + self.advance(); + Ok((FilterFunction::GreaterThanOrEqual, false)) + } + Token::Lt => { + self.advance(); + Ok((FilterFunction::LessThan, false)) + } + Token::Le => { + self.advance(); + Ok((FilterFunction::LessThanOrEqual, false)) + } + Token::Equals => { + self.advance(); + Ok((FilterFunction::Equals, false)) + } + // `NOT EQUALS` and `NOT EQUAL` both mean negated equality. A bare + // `EQUAL` without `NOT` is not a `compOp` alternative, which is why + // `primary` never routes one here. + Token::Not => { + self.advance(); + self.advance(); // EQUALS or EQUAL, checked by `primary` + Ok((FilterFunction::Equals, true)) + } + Token::Greater | Token::Less => { + let is_greater = tok == Token::Greater; + let word_start = self.pos; + self.advance(); + if self.peek() != &Token::Than { + // `greater`/`less` without `than` exhausts the alternatives + // for this decision, so the message quotes from the word. + return Err(self.no_viable(word_start, self.pos)); + } + self.advance(); + // The optional `OR EQUAL` tail. `OR EQUALS` (plural) is rejected + // by the grammar, which spells this alternative `OR EQUAL`. + if self.eat(&Token::Or) { + self.expect("EQUAL", is_equal, operand_follows)?; + Ok(( + if is_greater { + FilterFunction::GreaterThanOrEqual + } else { + FilterFunction::LessThanOrEqual + }, + false, + )) + } else { + Ok(( + if is_greater { + FilterFunction::GreaterThan + } else { + FilterFunction::LessThan + }, + false, + )) + } + } + // Unreachable: `primary` only routes a `compOp` start here. + _ => Err(self.no_viable(op_start, self.pos)), + } + } + + /// `textOperation : fieldRef NOT? textOp stringLiteral` + /// + /// `negate` is **always** `Some`, `false` included. + fn text_operation(&mut self, column: String) -> Result { + let has_not = self.eat(&Token::Not); + let function = self.text_op()?; + let token = self.expect("STRING", is_string, ends_primary)?; + let Token::Str(value) = &token.token else { + unreachable!("`expect` matched a STRING"); + }; + let filter = Filter { + condition: Some(Condition::new( + column, + function, + vec![serde_json::Value::String(value.clone())], + )), + group: None, + negate: Some(has_not), + }; + Ok(Node::Filter(filter)) + } + + /// `textOp : CONTAINS | STARTS WITH | ENDS WITH` + fn text_op(&mut self) -> Result { + match self.peek().clone() { + Token::Contains => { + self.advance(); + Ok(FilterFunction::Contains) + } + Token::Starts => { + self.advance(); + self.expect("WITH", is_with, string_follows)?; + Ok(FilterFunction::StartsWith) + } + // Unreachable for anything but `ENDS`: `primary` chose this rule. + _ => { + self.advance(); + self.expect("WITH", is_with, string_follows)?; + Ok(FilterFunction::EndsWith) + } + } + } + + /// `existenceOperation : fieldRef IS BLANK | fieldRef IS NOT BLANK` + /// + /// Produces `IsBlank` or `IsNotBlank` with empty `args` and no `negate`. + /// `field` is where a `no viable alternative` quotes from, since `[age] is 5` + /// abandons the whole primary rather than just this rule. + fn existence_operation(&mut self, column: String, field: usize) -> Result { + self.advance(); // IS + let negated = self.eat(&Token::Not); + if self.peek() != &Token::Blank { + // `IS NOT` has committed to this alternative, so a missing `BLANK` + // is reported as such. A bare `IS` followed by junk has not, and + // fails the primary decision instead. + if negated { + self.expect("BLANK", is_blank, ends_primary)?; + } else { + return Err(self.no_viable(field, self.pos)); + } + } + self.advance(); + let function = if negated { + FilterFunction::IsNotBlank + } else { + FilterFunction::IsBlank + }; + Ok(Node::Filter(Filter::condition(column, function, vec![]))) + } + + /// `operand : number | stringLiteral | booleanLiteral` + fn operand(&mut self) -> Result { + match self.peek().clone() { + Token::Str(s) => { + self.advance(); + Ok(serde_json::Value::String(s)) + } + Token::True => { + self.advance(); + Ok(serde_json::Value::Bool(true)) + } + Token::False => { + self.advance(); + Ok(serde_json::Value::Bool(false)) + } + Token::Sign(_) | Token::Digits(_) => self.number(), + _ => Err(self.expected_set(OPERAND_SET, starts_operand)), + } + } + + /// `number : SIGN? DIGITS (DOT DIGITS)?` + /// + /// Both sides of the dot are required, so `.5` and `5.` are errors. Since + /// whitespace is skipped between tokens, `1 . 5` is `1.5`. + fn number(&mut self) -> Result { + let negative = match self.peek() { + Token::Sign(c) => { + let neg = *c == '-'; + self.advance(); + Some(neg) + } + _ => None, + }; + // A sign with no digits ends the primary, so the follow set is the + // primary's: `[age] = -` reports a missing DIGITS at ``, while + // `[age] = - "x"` is a mismatch because a string cannot follow either. + let int_token = self.expect("DIGITS", is_digits, ends_primary)?; + let Token::Digits(int_part) = &int_token.token else { + unreachable!("`expect` matched DIGITS"); + }; + let mut text = int_part.clone(); + // The fraction is committed to on sight of the dot: the reference's + // adaptive prediction would look ahead, but every input where the two + // could differ is a syntax error either way. + if self.peek() == &Token::Dot { + self.advance(); + let frac_token = self.expect("DIGITS", is_digits, ends_primary)?; + let Token::Digits(frac) = &frac_token.token else { + unreachable!("`expect` matched DIGITS"); + }; + text.push('.'); + text.push_str(frac); + } + Ok(js_number(&text, negative == Some(true))) + } +} + +/// The token set an `operand` may start with, spelled as the message spells it. +const OPERAND_SET: &str = "{STRING, TRUE, FALSE, SIGN, DIGITS}"; + +/// Build the JSON number `parseFloat` would produce for `text`. +/// +/// The distinction matters for more than tidiness: `serde_json` remembers +/// whether a number was written as an integer, and `1.0` is not equal to `1`. +/// JavaScript has one number type, so an integral value must come back as an +/// integer or every AST comparison against the frontend's JSON fails. +fn js_number(text: &str, negative: bool) -> serde_json::Value { + // `parseFloat` on an over-long literal yields `Infinity`, which JSON cannot + // represent; the reference emits `null` for that arg. + let Ok(magnitude) = text.parse::() else { + return serde_json::Value::Null; + }; + let value = if negative { -magnitude } else { magnitude }; + if !value.is_finite() { + return serde_json::Value::Null; + } + // `-0` is `0` in JSON, and an integral f64 must serialize without a `.0`. + if value == value.trunc() && value.abs() < 9.007_199_254_740_992e15 { + return serde_json::Value::Number((value as i64).into()); + } + serde_json::Number::from_f64(value) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null) +} + +// The token predicates `expect` and `expected_set` are parameterised over. Each +// is a plain fn so the call sites stay allocation-free. + +/// Whether a token may legally follow the one `expect` is looking for, given how +/// many groups are currently open. The depth is what makes `missing STRING at +/// ')'` correct inside a group and `mismatched input ')' expecting STRING` +/// correct outside one. +type Follows = fn(&Token, usize) -> bool; + +fn never(_: &Token, _: usize) -> bool { + false +} + +fn is_eof(token: &Token) -> bool { + token == &Token::Eof +} + +fn is_rparen(token: &Token) -> bool { + token == &Token::RParen +} + +fn is_equal(token: &Token) -> bool { + token == &Token::Equal +} + +fn is_with(token: &Token) -> bool { + token == &Token::With +} + +fn is_blank(token: &Token) -> bool { + token == &Token::Blank +} + +fn is_string(token: &Token) -> bool { + matches!(token, Token::Str(_)) +} + +fn is_digits(token: &Token) -> bool { + matches!(token, Token::Digits(_)) +} + +/// The tokens a `primary` may start with: `{FIELD, NOT, '('}`. +fn starts_primary(token: &Token) -> bool { + matches!(token, Token::Field(_) | Token::Not | Token::LParen) +} + +/// The tokens an `operand` may start with. +fn starts_operand(token: &Token) -> bool { + matches!( + token, + Token::Str(_) | Token::True | Token::False | Token::Sign(_) | Token::Digits(_) + ) +} + +/// The tokens a `compOp` may start with. +fn starts_comp_op(token: &Token) -> bool { + matches!( + token, + Token::Eq + | Token::Eq2 + | Token::Neq + | Token::Gt + | Token::Ge + | Token::Lt + | Token::Le + | Token::Equals + | Token::Greater + | Token::Less + ) +} + +/// What may legally follow a completed `primary`, and so what licenses a +/// `missing` rather than a `mismatched` inside one. +/// +/// The depth is load-bearing, because ANTLR computes the follow set from the +/// rule invocation stack rather than from the grammar alone. `)` can only follow +/// while a group is open and `` only once every group has closed, which is +/// why `([name] contains` reports `mismatched input '' expecting STRING` +/// while the unparenthesised `[name] contains` reports `missing STRING at +/// ''`. Measured against the 2.2.0 bundle over `STRING`, `BLANK`, `DIGITS` +/// and `')'` at both depths. +fn ends_primary(token: &Token, depth: usize) -> bool { + match token { + Token::And | Token::Or => true, + Token::RParen => depth > 0, + Token::Eof => depth == 0, + _ => false, + } +} + +/// [`starts_operand`] as a [`Follows`]: the operand set does not depend on how +/// many groups are open. +fn operand_follows(token: &Token, _depth: usize) -> bool { + starts_operand(token) +} + +/// [`is_string`] as a [`Follows`], for the `WITH` in `STARTS WITH`. +fn string_follows(token: &Token, _depth: usize) -> bool { + is_string(token) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::LogicalOp; + use serde_json::json; + + fn parse(query: &str) -> FilterGroup { + parse_query_unchecked(query).unwrap_or_else(|e| panic!("{query:?} failed to parse: {e:?}")) + } + + fn parse_err(query: &str) -> ParseError { + parse_query_unchecked(query) + .map(|g| panic!("{query:?} unexpectedly parsed to {g:?}")) + .unwrap_err() + .into_iter() + .next() + .expect("at least one error") + } + + fn cond(column: &str, function: FilterFunction, args: Vec) -> Filter { + Filter::condition(column, function, args) + } + + // --- one assertion per row of the measured table ----------------------- + + #[test] + fn measured_greater_than() { + assert_eq!( + parse("[age] > 100"), + FilterGroup::and(vec![cond( + "age", + FilterFunction::GreaterThan, + vec![json!(100)] + )]) + ); + } + + #[test] + fn measured_not_equal_is_negated_equals() { + let expected = FilterGroup::and(vec![ + cond("age", FilterFunction::Equals, vec![json!(5)]).negated(true) + ]); + // All three not-equal spellings agree, and none produces a `notEquals`. + assert_eq!(parse("[age] != 5"), expected); + assert_eq!(parse("[age] not equals 5"), expected); + assert_eq!(parse("[age] not equal 5"), expected); + } + + #[test] + fn measured_not_contains() { + assert_eq!( + parse("[name] not contains \"ab\""), + FilterGroup::and(vec![cond( + "name", + FilterFunction::Contains, + vec![json!("ab")] + ) + .negated(true)]) + ); + } + + #[test] + fn measured_contains_emits_negate_false() { + let group = parse("[name] contains \"ab\""); + assert_eq!(group.filters[0].negate, Some(false)); + let value = serde_json::to_value(&group).unwrap(); + assert_eq!(value["filters"][0]["negate"], json!(false)); + } + + #[test] + fn measured_comparison_omits_negate() { + let group = parse("[age] > 1"); + assert_eq!(group.filters[0].negate, None); + let value = serde_json::to_value(&group).unwrap(); + assert!(value["filters"][0].get("negate").is_none()); + } + + #[test] + fn measured_or_root_with_spliced_and_arm() { + assert_eq!( + parse("[age] > 1 AND [n] = \"a\" OR [x] = true"), + FilterGroup::or(vec![ + Filter::from_group(FilterGroup::and(vec![ + cond("age", FilterFunction::GreaterThan, vec![json!(1)]), + cond("n", FilterFunction::Equals, vec![json!("a")]), + ])), + cond("x", FilterFunction::Equals, vec![json!(true)]), + ]) + ); + } + + #[test] + fn measured_not_lands_on_the_condition_filter() { + assert_eq!( + parse("NOT [age] > 1"), + FilterGroup::and(vec![cond( + "age", + FilterFunction::GreaterThan, + vec![json!(1)] + ) + .negated(true)]) + ); + } + + #[test] + fn measured_not_toggles() { + let group = parse("not not [age] > 1"); + assert_eq!(group.filters[0].negate, Some(false)); + assert_eq!(parse("NOT NOT NOT [age] > 1").filters[0].negate, Some(true)); + } + + #[test] + fn measured_parens_are_not_collapsed() { + assert_eq!( + parse("(([age] > 1))"), + FilterGroup::and(vec![Filter::from_group(FilterGroup::and(vec![ + Filter::from_group(FilterGroup::and(vec![cond( + "age", + FilterFunction::GreaterThan, + vec![json!(1)] + )])) + ]))]) + ); + // A third level nests a third time. + let deep = parse("((([age] > 1)))"); + let lvl1 = deep.filters[0].group.as_ref().unwrap(); + let lvl2 = lvl1.filters[0].group.as_ref().unwrap(); + let lvl3 = lvl2.filters[0].group.as_ref().unwrap(); + assert!(lvl3.filters[0].condition.is_some()); + } + + #[test] + fn measured_empty_and_blank_input() { + assert_eq!(parse(""), FilterGroup::default()); + assert_eq!(parse(" "), FilterGroup::default()); + assert_eq!(parse("\t\r\n"), FilterGroup::default()); + // and no errors + assert!(!parse_query("", &[]).has_errors()); + assert!(!parse_query(" ", &[]).has_errors()); + } + + #[test] + fn measured_whitespace_inside_a_number() { + for query in ["[age] = 1 . 5", "[age] = 1 .5", "[age] = 1. 5"] { + assert_eq!( + parse(query).filters[0].condition.as_ref().unwrap().args[0], + json!(1.5), + "{query}" + ); + } + } + + #[test] + fn measured_leading_zeros_are_dropped() { + assert_eq!( + parse("[age] = 007").filters[0] + .condition + .as_ref() + .unwrap() + .args[0], + json!(7) + ); + assert_eq!( + parse("[age] = 000000123").filters[0] + .condition + .as_ref() + .unwrap() + .args[0], + json!(123) + ); + } + + #[test] + fn measured_single_quotes_are_not_string_delimiters() { + assert_eq!( + parse_err("[name] = 'x'").message, + "token recognition error at: '''" + ); + } + + #[test] + fn measured_exponent_is_rejected() { + assert_eq!( + parse_err("[age] = 1e5").message, + "token recognition error at: 'e5'" + ); + } + + #[test] + fn measured_both_sides_of_the_dot_are_required() { + // `.5` — the dot is not a valid operand start. + assert!(parse_err("[age] = .5").message.contains("expecting")); + // `5.` — the fraction digits are missing. + assert_eq!(parse_err("[age] = 5.").message, "missing DIGITS at ''"); + assert_eq!( + parse_err("[age] = 1 .").message, + "missing DIGITS at ''" + ); + } + + #[test] + fn measured_nested_bracket_is_a_legal_column_name() { + assert_eq!( + parse("[a[b] = \"x\"").filters[0] + .condition + .as_ref() + .unwrap() + .column, + "a[b" + ); + } + + #[test] + fn measured_single_space_is_a_legal_column_name() { + assert_eq!( + parse("[ ] = \"x\"").filters[0] + .condition + .as_ref() + .unwrap() + .column, + " " + ); + assert!(parse_query_unchecked("[] = \"x\"").is_err()); + } + + // --- operator spellings ------------------------------------------------ + + #[test] + fn symbolic_operators_map_to_functions() { + let cases = [ + ("=", FilterFunction::Equals), + ("==", FilterFunction::Equals), + (">", FilterFunction::GreaterThan), + (">=", FilterFunction::GreaterThanOrEqual), + ("<", FilterFunction::LessThan), + ("<=", FilterFunction::LessThanOrEqual), + ]; + for (op, expected) in cases { + let group = parse(&format!("[age] {op} 5")); + assert_eq!( + group.filters[0].condition.as_ref().unwrap().function, + expected, + "{op}" + ); + } + } + + #[test] + fn word_operators_map_to_functions() { + let cases = [ + ("equals", FilterFunction::Equals), + ("greater than", FilterFunction::GreaterThan), + ("greater than or equal", FilterFunction::GreaterThanOrEqual), + ("less than", FilterFunction::LessThan), + ("less than or equal", FilterFunction::LessThanOrEqual), + ("GREATER THAN OR EQUAL", FilterFunction::GreaterThanOrEqual), + ]; + for (op, expected) in cases { + let group = parse(&format!("[age] {op} 5")); + assert_eq!( + group.filters[0].condition.as_ref().unwrap().function, + expected, + "{op}" + ); + } + } + + #[test] + fn bare_equal_without_not_is_rejected() { + assert_eq!( + parse_err("[age] equal 5").message, + "no viable alternative at input '[age]equal'" + ); + } + + #[test] + fn plural_or_equals_is_rejected() { + // The grammar spells this tail `OR EQUAL`, singular. + assert_eq!( + parse_err("[age] greater than or equals 5").message, + "mismatched input 'equals' expecting EQUAL" + ); + assert_eq!( + parse_err("[age] less than or equals 5").message, + "mismatched input 'equals' expecting EQUAL" + ); + } + + #[test] + fn text_operators_and_existence_operators() { + assert_eq!( + parse("[name] starts with \"a\"").filters[0] + .condition + .as_ref() + .unwrap() + .function, + FilterFunction::StartsWith + ); + assert_eq!( + parse("[name] ends with \"a\"").filters[0] + .condition + .as_ref() + .unwrap() + .function, + FilterFunction::EndsWith + ); + assert_eq!( + parse("[name] is blank"), + FilterGroup::and(vec![cond("name", FilterFunction::IsBlank, vec![])]) + ); + assert_eq!( + parse("[name] is not blank"), + FilterGroup::and(vec![cond("name", FilterFunction::IsNotBlank, vec![])]) + ); + } + + #[test] + fn existence_operations_carry_no_negate_key() { + for query in ["[name] is blank", "[name] is not blank"] { + let group = parse(query); + assert_eq!(group.filters[0].negate, None, "{query}"); + assert!(group.filters[0].condition.as_ref().unwrap().args.is_empty()); + } + } + + #[test] + fn negated_text_operations_always_emit_negate() { + for (query, expected) in [ + ("[name] contains \"a\"", false), + ("[name] not contains \"a\"", true), + ("[name] starts with \"a\"", false), + ("[name] not starts with \"a\"", true), + ("[name] ends with \"a\"", false), + ("[name] not ends with \"a\"", true), + ] { + assert_eq!(parse(query).filters[0].negate, Some(expected), "{query}"); + } + } + + #[test] + fn text_operation_rejects_a_non_string_argument() { + assert_eq!( + parse_err("[name] contains 5").message, + "mismatched input '5' expecting STRING" + ); + } + + // --- structural shaping ------------------------------------------------ + + #[test] + fn three_arm_and_stays_flat() { + let group = parse("[age] > 1 AND [age] < 5 AND [age] != 3"); + assert_eq!(group.op, LogicalOp::And); + assert_eq!(group.filters.len(), 3); + assert!(group.filters.iter().all(|f| f.condition.is_some())); + assert_eq!(group.filters[2].negate, Some(true)); + } + + #[test] + fn three_arm_or_stays_flat() { + let group = parse("[age] > 1 OR [age] < 5 OR [age] != 3"); + assert_eq!(group.op, LogicalOp::Or); + assert_eq!(group.filters.len(), 3); + assert!(group.filters.iter().all(|f| f.condition.is_some())); + } + + #[test] + fn or_splices_single_filter_arms_and_wraps_the_rest() { + // Middle arm has two filters, so it stays wrapped; the outer arms are bare. + let group = parse("[age] > 1 OR [age] < 2 AND [age] > 3 OR [age] < 4"); + assert_eq!(group.op, LogicalOp::Or); + assert_eq!(group.filters.len(), 3); + assert!(group.filters[0].condition.is_some()); + assert_eq!( + group.filters[1].group.as_ref().map(|g| g.filters.len()), + Some(2) + ); + assert!(group.filters[2].condition.is_some()); + } + + #[test] + fn an_explicit_group_arm_of_or_keeps_its_wrapper() { + // `([age] > 1)` is a *group filter* inside a one-filter AND group, so the + // splice hands the group filter through rather than unwrapping the paren. + let group = parse("([age] > 1) OR [age] < 2"); + assert_eq!(group.op, LogicalOp::Or); + assert_eq!( + group.filters[0] + .group + .as_ref() + .map(|g| (g.op, g.filters.len())), + Some((LogicalOp::And, 1)) + ); + assert!(group.filters[1].condition.is_some()); + } + + #[test] + fn a_group_inside_and_is_pushed_as_is() { + let group = parse("([age] > 1 OR [age] < 0) AND [name] = \"a\""); + assert_eq!(group.op, LogicalOp::And); + assert_eq!(group.filters.len(), 2); + assert_eq!( + group.filters[0].group.as_ref().map(|g| g.op), + Some(LogicalOp::Or) + ); + assert!(group.filters[1].condition.is_some()); + } + + #[test] + fn not_before_a_group_negates_the_group_filter() { + let group = parse("NOT ([age] > 1 AND [age] < 5)"); + assert_eq!(group.filters.len(), 1); + assert_eq!(group.filters[0].negate, Some(true)); + assert_eq!( + group.filters[0].group.as_ref().map(|g| g.filters.len()), + Some(2) + ); + } + + #[test] + fn not_before_a_group_toggles_too() { + let group = parse("NOT NOT ([age] > 1)"); + assert_eq!(group.filters[0].negate, Some(false)); + assert!(group.filters[0].group.is_some()); + } + + #[test] + fn not_inside_an_and_chain() { + let group = parse("[age] > 1 AND NOT [age] < 5"); + assert_eq!(group.filters.len(), 2); + assert_eq!(group.filters[0].negate, None); + assert_eq!(group.filters[1].negate, Some(true)); + } + + // --- syntax errors ----------------------------------------------------- + + #[test] + fn trailing_operator_with_no_operand_is_rejected() { + assert_eq!( + parse_err("[age] >").message, + "mismatched input '' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}" + ); + assert_eq!( + parse_err("[age] equals").message, + "mismatched input '' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}" + ); + } + + #[test] + fn dangling_and_is_rejected() { + assert_eq!( + parse_err("[age] > 1 AND").message, + "mismatched input '' expecting {FIELD, NOT, '('}" + ); + assert_eq!( + parse_err("[age] > 1 OR").message, + "mismatched input '' expecting {FIELD, NOT, '('}" + ); + } + + #[test] + fn trailing_extra_literal_is_rejected() { + let e = parse_err("[age] > 1 1"); + assert_eq!(e.message, "extraneous input '1' expecting "); + assert_eq!((e.start, e.end), (10, 11)); + } + + #[test] + fn a_second_operator_is_rejected() { + // `mismatched`, not `extraneous`: nothing may follow ``, so deleting + // the offending token would not let the match succeed either. + assert_eq!( + parse_err("[age] = 5 = 5").message, + "mismatched input '=' expecting " + ); + assert_eq!( + parse_err("[age] = 1.5.5").message, + "mismatched input '.' expecting " + ); + // A second *literal* is deletable, because a literal is what `` + // would follow — hence `extraneous` for this one. + assert_eq!( + parse_err("[age] > 1 1").message, + "extraneous input '1' expecting " + ); + } + + #[test] + fn spaced_two_character_operator_is_rejected() { + for query in ["[age] > = 5", "[age] < = 5", "[age] = = 5"] { + let e = parse_err(query); + assert!( + e.message + .contains("expecting {STRING, TRUE, FALSE, SIGN, DIGITS}"), + "{query}: {}", + e.message + ); + } + } + + #[test] + fn double_sign_is_rejected() { + // Deletion is tried first and the second `-` is followed by a digit, so + // the extra sign is blamed rather than a digit called missing. + assert_eq!( + parse_err("[age] = --5").message, + "extraneous input '-' expecting DIGITS" + ); + assert_eq!( + parse_err("[age] = -.5").message, + "extraneous input '.' expecting DIGITS" + ); + } + + #[test] + fn a_bare_literal_or_field_is_rejected() { + assert_eq!( + parse_err("5").message, + "mismatched input '5' expecting {FIELD, NOT, '('}" + ); + assert_eq!( + parse_err("()").message, + "mismatched input ')' expecting {FIELD, NOT, '('}" + ); + } + + #[test] + fn a_field_with_no_operator_quotes_from_the_field() { + // The three field-led alternatives all begin at the field, so the + // message runs the tokens together from there rather than naming just + // the token that failed. + assert_eq!( + parse_err("[age]").message, + "no viable alternative at input '[age]'" + ); + assert_eq!( + parse_err("[age] not").message, + "no viable alternative at input '[age]not'" + ); + assert_eq!( + parse_err("[age] not 5").message, + "no viable alternative at input '[age]not5'" + ); + assert_eq!( + parse_err("[age] not blank").message, + "no viable alternative at input '[age]notblank'" + ); + assert_eq!( + parse_err("[age] is").message, + "no viable alternative at input '[age]is'" + ); + assert_eq!( + parse_err("[age] is 5").message, + "no viable alternative at input '[age]is5'" + ); + assert_eq!( + parse_err("[age] not >").message, + "no viable alternative at input '[age]not>'" + ); + // Nesting and position do not change where the quoting starts. + assert_eq!( + parse_err("([age] equal 5)").message, + "no viable alternative at input '[age]equal'" + ); + assert_eq!( + parse_err("[age] > 1 AND [name] equal \"a\"").message, + "no viable alternative at input '[name]equal'" + ); + } + + #[test] + fn a_word_operator_with_no_than_quotes_from_the_word() { + // `greater`/`less` open a decision of their own, so the field is not + // part of the quoted text. + assert_eq!( + parse_err("[age] greater 5").message, + "no viable alternative at input 'greater5'" + ); + assert_eq!( + parse_err("[age] less 5").message, + "no viable alternative at input 'less5'" + ); + assert_eq!( + parse_err("[age] greater").message, + "no viable alternative at input 'greater'" + ); + assert_eq!( + parse_err("[age] less").message, + "no viable alternative at input 'less'" + ); + assert_eq!( + parse_err("([age] greater 5)").message, + "no viable alternative at input 'greater5'" + ); + } + + #[test] + fn quoted_text_keeps_the_original_case() { + assert_eq!( + parse_err("[age] EQUAL 5").message, + "no viable alternative at input '[age]EQUAL'" + ); + assert_eq!( + parse_err("[age] GREATER 5").message, + "no viable alternative at input 'GREATER5'" + ); + assert_eq!( + parse_err("[age] NOT 5").message, + "no viable alternative at input '[age]NOT5'" + ); + } + + #[test] + fn unbalanced_parens_are_rejected() { + assert_eq!(parse_err("([age] > 1").message, "missing ')' at ''"); + assert_eq!( + parse_err("([age] > 1 AND [age] < 2").message, + "missing ')' at ''" + ); + // A token that cannot follow the group is a mismatch, not an omission. + assert_eq!( + parse_err("([age] > 1 5").message, + "mismatched input '5' expecting ')'" + ); + assert_eq!( + parse_err("[age] > 1)").message, + "extraneous input ')' expecting " + ); + } + + #[test] + fn a_missing_token_is_reported_as_missing_when_one_would_do() { + // Insertion fires only where the offending token could legally follow + // the one left out. + for (query, expected) in [ + ("[age] is not", "missing BLANK at ''"), + ("[age] is not AND [age] > 1", "missing BLANK at 'AND'"), + ("[name] contains", "missing STRING at ''"), + ("[name] not contains", "missing STRING at ''"), + ("[name] starts with", "missing STRING at ''"), + ("[name] contains AND [age] > 1", "missing STRING at 'AND'"), + ("[age] greater than or 5", "missing EQUAL at '5'"), + ("[age] = -", "missing DIGITS at ''"), + ("[age] = 5.", "missing DIGITS at ''"), + ("[age] = 5. AND [age] > 1", "missing DIGITS at 'AND'"), + ("([age] = 5.)", "missing DIGITS at ')'"), + ] { + assert_eq!(parse_err(query).message, expected, "{query}"); + } + } + + #[test] + fn a_deletable_token_is_reported_as_extraneous() { + // Deletion is tried before insertion, so the doubled sign is blamed + // rather than a digit being called missing. + for (query, expected) in [ + ("[age] = --5", "extraneous input '-' expecting DIGITS"), + ("[age] = -.5", "extraneous input '.' expecting DIGITS"), + ( + "[age] = .5", + "extraneous input '.' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}", + ), + ( + "[age] > = 5", + "extraneous input '=' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}", + ), + ( + "AND [age] > 1", + "extraneous input 'AND' expecting {FIELD, NOT, '('}", + ), + ("[age] > 1 1", "extraneous input '1' expecting "), + ( + "[age] is blank blank", + "extraneous input 'blank' expecting ", + ), + ] { + assert_eq!(parse_err(query).message, expected, "{query}"); + } + } + + #[test] + fn an_undeletable_unomittable_token_is_a_mismatch() { + for (query, expected) in [ + ("[name] starts 5", "mismatched input '5' expecting WITH"), + ("[name] ends 5", "mismatched input '5' expecting WITH"), + ("[name] starts", "mismatched input '' expecting WITH"), + ("[name] ends", "mismatched input '' expecting WITH"), + ( + "[name] starts AND [age] > 1", + "mismatched input 'AND' expecting WITH", + ), + ("[name] is not 5", "mismatched input '5' expecting BLANK"), + ("[name] contains 5", "mismatched input '5' expecting STRING"), + ( + "[name] contains true", + "mismatched input 'true' expecting STRING", + ), + ("[age] = 5.true", "mismatched input 'true' expecting DIGITS"), + ( + "[age] greater than or", + "mismatched input '' expecting EQUAL", + ), + ("[age] = 5 = 5", "mismatched input '=' expecting "), + ("[age] > 1 1 1", "mismatched input '1' expecting "), + ] { + assert_eq!(parse_err(query).message, expected, "{query}"); + } + } + + // --- number handling --------------------------------------------------- + + #[test] + fn signs_are_applied() { + for query in ["[age] = -5", "[age] = - 5"] { + assert_eq!( + parse(query).filters[0].condition.as_ref().unwrap().args[0], + json!(-5), + "{query}" + ); + } + for query in ["[age] = +5", "[age] = + 5"] { + assert_eq!( + parse(query).filters[0].condition.as_ref().unwrap().args[0], + json!(5), + "{query}" + ); + } + } + + #[test] + fn trailing_fraction_zeros_are_normalized() { + assert_eq!( + parse("[age] = 1.500").filters[0] + .condition + .as_ref() + .unwrap() + .args[0], + json!(1.5) + ); + // `1.000` is integral, and the reference emits `1` for it — JavaScript + // has one number type, so a written fraction that happens to be whole + // still serializes without a decimal point. + let group = parse("[age] = 1.000"); + let one = &group.filters[0].condition.as_ref().unwrap().args[0]; + assert_eq!(one, &json!(1)); + assert_eq!(one.to_string(), "1"); + } + + #[test] + fn an_unrepresentable_number_becomes_null() { + // `parseFloat` overflows to Infinity in the reference, which serializes + // as `null`; the same arg must appear here. + let query = format!("[age] = {}", "9".repeat(400)); + assert_eq!( + parse(&query).filters[0].condition.as_ref().unwrap().args[0], + serde_json::Value::Null + ); + } + + // --- parse_query and ParseResult --------------------------------------- + + #[test] + fn parse_result_is_filters_or_errors_never_both() { + let cols = vec![ColumnDef::new("age", crate::column::ColumnType::Number)]; + let ok = parse_query("[age] > 1", &cols); + assert!(ok.filters.is_some()); + assert!(ok.errors.is_none()); + assert!(!ok.has_errors()); + assert!(ok.errors().is_empty()); + + let bad = parse_query("[nope] > 1", &cols); + assert!(bad.filters.is_none()); + assert!(bad.has_errors()); + assert_eq!(bad.errors().len(), 1); + } + + #[test] + fn parse_result_json_omits_the_absent_side() { + let ok = ParseResult::ok(FilterGroup::default()); + let value = serde_json::to_value(&ok).unwrap(); + assert!(value.get("errors").is_none()); + assert!(value.get("filters").is_some()); + + let bad = ParseResult::failed(vec![ParseError::new("x", 0, 0)]); + let value = serde_json::to_value(&bad).unwrap(); + assert!(value.get("filters").is_none()); + assert_eq!(value["errors"][0]["severity"], json!("error")); + } + + #[test] + fn syntax_errors_preempt_semantic_ones() { + // `[nope]` does not exist *and* the trailing `1` is extraneous; the + // syntax error is the one reported. + let cols = vec![ColumnDef::new("age", crate::column::ColumnType::Number)]; + let result = parse_query("[nope] > 1 1", &cols); + assert_eq!( + result.errors()[0].message, + "extraneous input '1' expecting " + ); + } + + #[test] + fn error_spans_are_byte_offsets() { + // The documented divergence, pinned as a difference rather than an + // agreement: `é` is two bytes but one UTF-16 code unit, so the reference + // reports 11 for this input where the byte offset is 12. The span must + // still slice the input correctly, which is what makes it the useful one. + let query = "[café] > 1 1"; + let e = parse_err(query); + assert_eq!((e.start, e.end), (12, 13)); + assert_eq!(&query[e.start..e.end], "1"); + // The same query in pure ASCII agrees with the reference exactly, and the + // gap between the two is precisely the extra byte `é` costs. + let ascii = parse_err("[cafe] > 1 1"); + assert_eq!((ascii.start, ascii.end), (11, 12)); + assert_eq!(e.start - ascii.start, query.len() - "[cafe] > 1 1".len()); + } + + #[test] + fn severity_serializes_lowercase() { + assert_eq!( + serde_json::to_value(ErrorSeverity::Error).unwrap(), + json!("error") + ); + assert_eq!( + serde_json::to_value(ErrorSeverity::Warning).unwrap(), + json!("warning") + ); + assert_eq!(ErrorSeverity::default(), ErrorSeverity::Error); + } +} diff --git a/rusty-filter/src/print.rs b/rusty-filter/src/print.rs new file mode 100644 index 0000000..308d51e --- /dev/null +++ b/rusty-filter/src/print.rs @@ -0,0 +1,526 @@ +//! Printing an AST back to a query string. +//! +//! A port of `ASTPrinter` in canonical mode only — the non-canonical option +//! combinations have no caller in this repo, so the options struct is not +//! reproduced. +//! +//! The canonical spelling is deliberately asymmetric: `=` prints as the word +//! `equals` while the four orderings stay symbolic. That is what the bundle +//! does, and changing it would make the two sides disagree on cache keys. + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup, LogicalOp}; + +/// Print `group` as a canonical query string. +/// +/// An empty group prints as the empty string. Output is stable for a given AST +/// but is not always re-parseable to the same *text*: `[name] not contains "a"` +/// prints as `NOT ([name] contains "a")`, which parses back to the same AST. +pub fn to_query_string(group: &FilterGroup) -> String { + if group.filters.is_empty() { + return String::new(); + } + print_group(group, false) +} + +/// The canonical string of `group`, for use as a cache key. +/// +/// This is [`to_query_string`] under a name that says what it is for. Two ASTs +/// that filter identically produce the same key, and the frontend's +/// `formatQuery` produces that same key for the same AST. +pub fn canonical_key(group: &FilterGroup) -> String { + to_query_string(group) +} + +fn print_group(group: &FilterGroup, nested: bool) -> String { + if group.filters.is_empty() { + return String::new(); + } + let separator = match group.op { + LogicalOp::And => " AND ", + LogicalOp::Or => " OR ", + }; + let joined = group + .filters + .iter() + .map(print_filter) + .collect::>() + .join(separator); + if nested { + format!("({joined})") + } else { + joined + } +} + +fn print_filter(filter: &Filter) -> String { + // A filter with neither side prints as nothing, and an empty nested group + // prints as nothing too — which is how the reference ends up emitting a + // dangling `AND` for `[age] equals 1 AND `. The parser never + // builds either shape; a hand-built AST can, and reproducing the oddity + // keeps the two sides' cache keys in step. + let mut result = if let Some(condition) = &filter.condition { + print_condition(condition) + } else if let Some(group) = &filter.group { + print_group(group, true) + } else { + String::new() + }; + if filter.is_negated() { + // Parenthesize unless the text is already a group, so `NOT` always + // takes a bracketed operand — an empty operand included, as `NOT ()`. + if !result.starts_with('(') { + result = format!("({result})"); + } + result = format!("NOT {result}"); + } + result +} + +fn print_condition(condition: &Condition) -> String { + let column = print_column(&condition.column); + match condition.function { + FilterFunction::IsBlank => return format!("{column} IS BLANK"), + FilterFunction::IsNotBlank => return format!("{column} IS NOT BLANK"), + _ => {} + } + let operator = print_operator(condition.function); + // Only the first argument is printed; the AST never carries more. + match condition.args.first() { + Some(arg) => format!("{column} {operator} {}", print_value(arg)), + None => format!("{column} {operator}"), + } +} + +/// Bracket the column name, unless it already opens with a bracket — a name +/// stored as `[age]` would otherwise become `[[age]]`. +fn print_column(name: &str) -> String { + if name.starts_with('[') { + name.to_string() + } else { + format!("[{name}]") + } +} + +fn print_operator(function: FilterFunction) -> &'static str { + match function { + FilterFunction::Equals => "equals", + FilterFunction::GreaterThan => ">", + FilterFunction::LessThan => "<", + FilterFunction::GreaterThanOrEqual => ">=", + FilterFunction::LessThanOrEqual => "<=", + FilterFunction::Contains => "contains", + FilterFunction::StartsWith => "STARTS WITH", + FilterFunction::EndsWith => "ENDS WITH", + FilterFunction::IsBlank => "IS BLANK", + FilterFunction::IsNotBlank => "IS NOT BLANK", + } +} + +fn print_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(text) => { + // The backslash is escaped first, then the quote, so a literal + // backslash does not swallow the quote's escape. + let escaped = text.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") + } + serde_json::Value::Bool(true) => "true".to_string(), + serde_json::Value::Bool(false) => "false".to_string(), + serde_json::Value::Number(number) => match number.as_f64() { + Some(value) => js_number_to_string(value), + // Unreachable for finite JSON numbers, but a large integer literal + // is better printed than dropped. + None => number.to_string(), + }, + serde_json::Value::Null => "null".to_string(), + // The reference falls through to `String(value)` here. + other => js_string_of(other), + } +} + +/// Format a number the way JavaScript's `Number.prototype.toString` does, so +/// that a key built here matches one built in the browser. +/// +/// Rust's own `Display` agrees for everyday values but never switches to +/// exponential notation, where JavaScript does so above 10^21 and below 10^-6: +/// `1e21` and `1e-7` print as `1e+21` and `1e-7`, not as long digit strings. +fn js_number_to_string(value: f64) -> String { + if value == 0.0 { + // Covers -0.0, which JavaScript prints as `0`. + return "0".to_string(); + } + if value.is_nan() { + return "NaN".to_string(); + } + if value.is_infinite() { + return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_string(); + } + if value < 0.0 { + return format!("-{}", js_number_to_string(-value)); + } + + // `{:e}` gives the shortest round-tripping digits plus a decimal exponent, + // which is exactly the `s` and `n` of the spec: `s` is the digit string and + // `n` is one past the exponent. + let exp_form = format!("{value:e}"); + let (mantissa, exponent) = exp_form.split_once('e').expect("{:e} always emits an e"); + let digits: String = mantissa.chars().filter(|c| *c != '.').collect(); + let k = digits.len() as i32; + let n = exponent + .parse::() + .expect("{:e} emits a decimal exponent") + + 1; + + if k <= n && n <= 21 { + // Whole number: the digits then n - k zeros. + let mut out = digits; + out.extend(std::iter::repeat_n('0', (n - k) as usize)); + out + } else if 0 < n && n <= 21 { + // A decimal point inside the digits. + format!("{}.{}", &digits[..n as usize], &digits[n as usize..]) + } else if -6 < n && n <= 0 { + // A leading zero, then -n zeros, then the digits. + let zeros = "0".repeat((-n) as usize); + format!("0.{zeros}{digits}") + } else { + // Exponential, with an explicit `+` on a positive exponent. + let sign = if n >= 1 { "+" } else { "-" }; + let magnitude = (n - 1).abs(); + if k == 1 { + format!("{digits}e{sign}{magnitude}") + } else { + format!("{}.{}e{sign}{magnitude}", &digits[..1], &digits[1..]) + } + } +} + +/// JavaScript's `String(value)` for the JSON values `printValue` does not +/// special-case: an array joins its elements with commas, treating null and +/// undefined as empty, and any other object is `[object Object]`. +fn js_string_of(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Array(items) => items + .iter() + .map(|item| match item { + serde_json::Value::Null => String::new(), + serde_json::Value::String(text) => text.clone(), + serde_json::Value::Bool(true) => "true".to_string(), + serde_json::Value::Bool(false) => "false".to_string(), + serde_json::Value::Number(number) => match number.as_f64() { + Some(value) => js_number_to_string(value), + None => number.to_string(), + }, + nested => js_string_of(nested), + }) + .collect::>() + .join(","), + serde_json::Value::Object(_) => "[object Object]".to_string(), + serde_json::Value::Null => "null".to_string(), + serde_json::Value::String(text) => text.clone(), + serde_json::Value::Bool(true) => "true".to_string(), + serde_json::Value::Bool(false) => "false".to_string(), + serde_json::Value::Number(number) => match number.as_f64() { + Some(value) => js_number_to_string(value), + None => number.to_string(), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse_query_unchecked; + use serde_json::json; + + fn printed(query: &str) -> String { + to_query_string(&parse_query_unchecked(query).expect(query)) + } + + #[test] + fn the_measured_round_trips_are_reproduced() { + assert_eq!(printed("[age]>100"), "[age] > 100"); + assert_eq!(printed("[age] greater than 100"), "[age] > 100"); + assert_eq!( + printed("[name] not contains \"a\""), + "NOT ([name] contains \"a\")" + ); + assert_eq!(printed("NOT [age] > 1"), "NOT ([age] > 1)"); + assert_eq!( + printed("[age] > 1 AND [name] = \"a\" OR [active] = true"), + "([age] > 1 AND [name] equals \"a\") OR [active] equals true" + ); + assert_eq!(printed("[name] is not blank"), "[name] IS NOT BLANK"); + } + + #[test] + fn every_operator_has_a_canonical_spelling() { + let group = FilterGroup::and( + [ + FilterFunction::GreaterThan, + FilterFunction::LessThan, + FilterFunction::GreaterThanOrEqual, + FilterFunction::LessThanOrEqual, + FilterFunction::Equals, + FilterFunction::Contains, + FilterFunction::StartsWith, + FilterFunction::EndsWith, + ] + .into_iter() + .map(|f| Filter::condition("c", f, vec![json!("v")])) + .collect(), + ); + assert_eq!( + to_query_string(&group), + "[c] > \"v\" AND [c] < \"v\" AND [c] >= \"v\" AND [c] <= \"v\" \ + AND [c] equals \"v\" AND [c] contains \"v\" \ + AND [c] STARTS WITH \"v\" AND [c] ENDS WITH \"v\"" + ); + } + + #[test] + fn blank_operators_print_as_keywords() { + let group = FilterGroup::and(vec![ + Filter::condition("c", FilterFunction::IsBlank, vec![]), + Filter::condition("c", FilterFunction::IsNotBlank, vec![]), + ]); + assert_eq!(to_query_string(&group), "[c] IS BLANK AND [c] IS NOT BLANK"); + // Stray args on a blank operator are ignored, as in the reference. + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::IsBlank, + vec![json!("x")], + )]); + assert_eq!(to_query_string(&group), "[c] IS BLANK"); + } + + #[test] + fn an_empty_group_prints_as_nothing() { + assert_eq!(to_query_string(&FilterGroup::default()), ""); + assert_eq!(to_query_string(&FilterGroup::or(vec![])), ""); + assert_eq!(printed(""), ""); + assert_eq!(printed(" "), ""); + } + + #[test] + fn a_false_negate_key_prints_nothing_extra() { + let group = FilterGroup::and(vec![Filter::condition( + "name", + FilterFunction::Contains, + vec![json!("a")], + ) + .negated(false)]); + assert_eq!(to_query_string(&group), "[name] contains \"a\""); + // Which is exactly what the parser produces for the un-negated form. + assert_eq!(printed("[name] contains \"a\""), "[name] contains \"a\""); + } + + #[test] + fn nested_groups_are_parenthesized() { + assert_eq!( + printed("([age] > 1 AND [b] = 2) OR [c] = 3"), + "([age] > 1 AND [b] equals 2) OR [c] equals 3" + ); + assert_eq!( + printed("(([age] > 1))"), + "(([age] > 1))", + "parens are not collapsed" + ); + assert_eq!( + printed("NOT ([age] > 1 AND [b] = 2)"), + "NOT ([age] > 1 AND [b] equals 2)" + ); + } + + #[test] + fn negation_of_an_empty_group_still_brackets() { + let group = FilterGroup::and(vec![ + Filter::from_group(FilterGroup::default()).negated(true) + ]); + assert_eq!(to_query_string(&group), "NOT ()"); + let group = FilterGroup::and(vec![Filter::default().negated(true)]); + assert_eq!(to_query_string(&group), "NOT ()"); + } + + #[test] + fn a_filter_with_neither_side_prints_as_nothing() { + let group = FilterGroup::and(vec![Filter::default()]); + assert_eq!(to_query_string(&group), ""); + // And an empty arm leaves a dangling operator, faithfully. + let group = FilterGroup::and(vec![ + Filter::condition("age", FilterFunction::Equals, vec![json!(1)]), + Filter::from_group(FilterGroup::default()), + ]); + assert_eq!(to_query_string(&group), "[age] equals 1 AND "); + } + + #[test] + fn a_condition_with_no_args_prints_the_bare_operator() { + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![], + )]); + assert_eq!(to_query_string(&group), "[age] >"); + } + + #[test] + fn columns_are_bracketed_at_most_once() { + let group = FilterGroup::and(vec![ + Filter::condition("age", FilterFunction::Equals, vec![json!(1)]), + Filter::condition("[age]", FilterFunction::Equals, vec![json!(1)]), + Filter::condition("a]b", FilterFunction::Equals, vec![json!(1)]), + ]); + assert_eq!( + to_query_string(&group), + "[age] equals 1 AND [age] equals 1 AND [a]b] equals 1" + ); + } + + #[test] + fn strings_escape_the_backslash_before_the_quote() { + let cases = [ + ("a", "\"a\""), + ("", "\"\""), + ("\"", "\"\\\"\""), + ("\\", "\"\\\\\""), + ("\\\"", "\"\\\\\\\"\""), + ("\\\\", "\"\\\\\\\\\""), + // Whitespace and single quotes pass through untouched. + ("\t", "\"\t\""), + ("\n", "\"\n\""), + ("'", "\"'\""), + ]; + for (input, expected) in cases { + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!(input)], + )]); + assert_eq!( + to_query_string(&group), + format!("[c] equals {expected}"), + "{input:?}" + ); + } + } + + #[test] + fn a_quoted_string_survives_the_round_trip() { + let query = "[c] = \"a\\\"b\\\\c\""; + let group = parse_query_unchecked(query).unwrap(); + let printed = to_query_string(&group); + assert_eq!(printed, "[c] equals \"a\\\"b\\\\c\""); + assert_eq!(parse_query_unchecked(&printed).unwrap(), group); + } + + #[test] + fn booleans_and_null_print_as_literals() { + let group = FilterGroup::and(vec![ + Filter::condition("b", FilterFunction::Equals, vec![json!(true)]), + Filter::condition("b", FilterFunction::Equals, vec![json!(false)]), + Filter::condition("b", FilterFunction::Equals, vec![serde_json::Value::Null]), + ]); + assert_eq!( + to_query_string(&group), + "[b] equals true AND [b] equals false AND [b] equals null" + ); + } + + #[test] + fn numbers_print_the_way_javascript_prints_them() { + // Every expectation measured against `String(n)` in Node. + let cases = [ + (1.0, "1"), + (-1.0, "-1"), + (0.0, "0"), + (-0.0, "0"), + (1.5, "1.5"), + (0.1, "0.1"), + (100.0, "100"), + (1e21, "1e+21"), + (1e-7, "1e-7"), + (-1e-7, "-1e-7"), + (1e20, "100000000000000000000"), + (1e-6, "0.000001"), + (1.2345e2, "123.45"), + (1.5e22, "1.5e+22"), + (1.25e-8, "1.25e-8"), + (f64::INFINITY, "Infinity"), + (f64::NEG_INFINITY, "-Infinity"), + (f64::NAN, "NaN"), + ]; + for (value, expected) in cases { + assert_eq!(js_number_to_string(value), expected, "{value}"); + } + } + + #[test] + fn numbers_from_the_parser_print_back_identically() { + for query in [ + "[n] = 0", + "[n] = 1", + "[n] = -1", + "[n] = 1.5", + "[n] = 0.1", + "[n] = 100", + "[n] = 100000000000000000000", + ] { + assert_eq!(printed(query), query.replace('=', "equals"), "{query}"); + } + // A literal too large for an f64 becomes `Infinity`, which the + // reference stores as null; printing says `null` rather than inventing + // a value. + let huge = format!("[n] = 1{}", "0".repeat(400)); + assert_eq!(printed(&huge), "[n] equals null"); + } + + #[test] + fn composite_args_fall_back_to_the_javascript_string_conversion() { + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!([1, 2])], + )]); + assert_eq!(to_query_string(&group), "[c] equals 1,2"); + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!({"a": 1})], + )]); + assert_eq!(to_query_string(&group), "[c] equals [object Object]"); + // Null inside an array is the empty string, not `null`. + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!([null, "a", [2, 3]])], + )]); + assert_eq!(to_query_string(&group), "[c] equals ,a,2,3"); + } + + #[test] + fn only_the_first_arg_is_printed() { + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!("a"), json!("b")], + )]); + assert_eq!(to_query_string(&group), "[c] equals \"a\""); + } + + #[test] + fn canonical_key_is_the_query_string() { + let group = parse_query_unchecked("[age]>100").unwrap(); + assert_eq!(canonical_key(&group), to_query_string(&group)); + assert_eq!(canonical_key(&FilterGroup::default()), ""); + } + + #[test] + fn synonyms_collapse_onto_one_key() { + // The point of a canonical key: differently spelled equivalents agree. + let keys = ["[age]>100", "[age] > 100", "[age] greater than 100"] + .map(|q| canonical_key(&parse_query_unchecked(q).unwrap())); + assert_eq!(keys[0], keys[1]); + assert_eq!(keys[1], keys[2]); + } +} diff --git a/rusty-filter/src/validate.rs b/rusty-filter/src/validate.rs new file mode 100644 index 0000000..6a23927 --- /dev/null +++ b/rusty-filter/src/validate.rs @@ -0,0 +1,603 @@ +//! Semantic validation against a column schema. +//! +//! A port of `SemanticValidator` and `TypeChecker`, message strings included, so +//! that a Rust-side error reads exactly like the browser's. +//! +//! Every error carries `start: 0` and `end: 0`: the reference validator has no +//! position information to report, and the frontend's own `parseQuery` passes +//! those zeros straight through. That is faithful behaviour, not a defect. + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup}; +use crate::column::{find_column, ColumnDef, ColumnType}; +use crate::parser::ParseError; + +/// Validate `group` against `columns`, returning every error found. +/// +/// Validation walks nested groups. A condition naming an unknown column reports +/// only that error and is not checked further, matching the reference's early +/// return. +pub fn validate_filter_group(group: &FilterGroup, columns: &[ColumnDef]) -> Vec { + let mut errors = Vec::new(); + walk_group(group, columns, &mut errors); + errors +} + +fn walk_group(group: &FilterGroup, columns: &[ColumnDef], errors: &mut Vec) { + for filter in &group.filters { + walk_filter(filter, columns, errors); + } +} + +fn walk_filter(filter: &Filter, columns: &[ColumnDef], errors: &mut Vec) { + // `condition` wins when both are somehow set, as the `if / else if` does. + if let Some(condition) = &filter.condition { + validate_condition(condition, columns, errors); + } else if let Some(group) = &filter.group { + walk_group(group, columns, errors); + } +} + +fn validate_condition(condition: &Condition, columns: &[ColumnDef], errors: &mut Vec) { + let Some(column) = find_column(columns, &condition.column) else { + errors.push(error(format!( + "Column '{}' does not exist", + condition.column + ))); + return; + }; + let column_type = column.column_type; + + if condition.function.is_blank_operator() { + if !is_blank_operator_compatible(column_type) { + errors.push(incompatible(condition.function, column_type)); + } + // Blank operators take no arguments, so there is nothing left to check. + return; + } + + if !is_operator_compatible(column_type, condition.function) { + errors.push(incompatible(condition.function, column_type)); + return; + } + + if condition.args.is_empty() { + errors.push(error(format!( + "Operator '{}' requires a value", + condition.function.display_name() + ))); + return; + } + + for arg in &condition.args { + if let Err(message) = validate_value_type(arg, column) { + errors.push(error(message)); + } + } +} + +fn error(message: String) -> ParseError { + ParseError::new(message, 0, 0) +} + +fn incompatible(function: FilterFunction, column_type: ColumnType) -> ParseError { + error(format!( + "Operator '{}' is not compatible with type '{}'", + function.display_name(), + column_type.as_str() + )) +} + +/// Whether `function` may be applied to `column_type`. +/// +/// `String` allows equals, contains, startsWith and endsWith; `Number` and +/// `Date` allow equals plus the four orderings; `Boolean` and `Enum` allow +/// equals only. +pub fn is_operator_compatible(column_type: ColumnType, function: FilterFunction) -> bool { + use FilterFunction::*; + match column_type { + ColumnType::String => matches!(function, Equals | Contains | StartsWith | EndsWith), + ColumnType::Number | ColumnType::Date => matches!( + function, + Equals | GreaterThan | LessThan | GreaterThanOrEqual | LessThanOrEqual + ), + ColumnType::Boolean | ColumnType::Enum => function == Equals, + } +} + +/// Whether the blank operators may be applied to `column_type`. They are +/// allowed on `String`, `Date` and `Enum` only. +pub fn is_blank_operator_compatible(column_type: ColumnType) -> bool { + matches!( + column_type, + ColumnType::String | ColumnType::Date | ColumnType::Enum + ) +} + +/// Check one argument against a column, returning the reference's error message +/// on mismatch. +pub fn validate_value_type(value: &serde_json::Value, column: &ColumnDef) -> Result<(), String> { + let name = &column.name; + match column.column_type { + ColumnType::String => { + if !value.is_string() { + return Err(format!( + "Expected string for column '{name}', got {}", + js_typeof(value) + )); + } + } + ColumnType::Number => { + if !value.is_number() { + return Err(format!( + "Expected number for column '{name}', got {}", + js_typeof(value) + )); + } + } + ColumnType::Boolean => { + if !value.is_boolean() { + return Err(format!( + "Expected boolean for column '{name}', got {}", + js_typeof(value) + )); + } + } + ColumnType::Date => { + let Some(text) = value.as_str() else { + return Err(format!( + "Expected date string for column '{name}', got {}", + js_typeof(value) + )); + }; + if !is_iso_date_shape(text) { + return Err(format!( + "Invalid date format for column '{name}'. Expected YYYY-MM-DD or ISO datetime" + )); + } + } + ColumnType::Enum => { + if !value.is_string() { + return Err(format!( + "Expected string for enum column '{name}', got {}", + js_typeof(value) + )); + } + // The reference skips value-set validation: `ColumnDef` carries no + // enum members to check against. + } + } + Ok(()) +} + +/// The name JavaScript's `typeof` would give a JSON value, so that the "got X" +/// half of a message matches the reference. +fn js_typeof(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::String(_) => "string", + serde_json::Value::Number(_) => "number", + serde_json::Value::Bool(_) => "boolean", + // `typeof null` is `"object"` in JavaScript, and an unrepresentable + // number arrives here as JSON null. + serde_json::Value::Null => "object", + serde_json::Value::Array(_) | serde_json::Value::Object(_) => "object", + } +} + +/// Match `/^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d{3})?Z?)?$/`. +/// +/// This is a **shape** check, not a calendar check: `2024-13-99` passes here +/// because it passes in the bundle. Hand-rolled to avoid a regex dependency for +/// one pattern. +fn is_iso_date_shape(text: &str) -> bool { + let bytes = text.as_bytes(); + // YYYY-MM-DD is exactly ten characters. + if bytes.len() < 10 { + return false; + } + let digits = |range: std::ops::Range| bytes[range].iter().all(u8::is_ascii_digit); + if !digits(0..4) || bytes[4] != b'-' || !digits(5..7) || bytes[7] != b'-' || !digits(8..10) { + return false; + } + if bytes.len() == 10 { + return true; + } + // The optional time part: THH:MM:SS + if bytes[10] != b'T' || bytes.len() < 19 { + return false; + } + if !digits(11..13) || bytes[13] != b':' || !digits(14..16) || bytes[16] != b':' { + return false; + } + if !digits(17..19) { + return false; + } + let mut i = 19; + // The optional milliseconds: exactly three digits after the dot. + if bytes.get(i) == Some(&b'.') { + if bytes.len() < i + 4 || !digits(i + 1..i + 4) { + return false; + } + i += 4; + } + // The optional trailing Z. + if bytes.get(i) == Some(&b'Z') { + i += 1; + } + i == bytes.len() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse_query; + use serde_json::json; + + fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("created", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ] + } + + /// The single message a query produces, or a panic if it produced none. + fn message(query: &str) -> String { + let result = parse_query(query, &columns()); + let errors = result.errors(); + assert_eq!(errors.len(), 1, "{query}: {errors:?}"); + errors[0].message.clone() + } + + fn is_valid(query: &str) -> bool { + !parse_query(query, &columns()).has_errors() + } + + #[test] + fn unknown_column_is_reported() { + assert_eq!(message("[nope] = 1"), "Column 'nope' does not exist"); + assert_eq!(message("[ s ] = \"x\""), "Column ' s ' does not exist"); + // A non-existent column stops validation of that condition, so the + // operator/type mismatch that would otherwise follow is not also + // reported. The operand has to be well-typed for the *grammar* first: + // `[nope] contains 1` fails to parse, so validation never runs on it. + assert_eq!( + message("[nope] contains \"x\""), + "Column 'nope' does not exist" + ); + assert_eq!(message("[nope] is blank"), "Column 'nope' does not exist"); + assert_eq!( + message("[nope] contains 1"), + "mismatched input '1' expecting STRING" + ); + } + + #[test] + fn semantic_errors_carry_zero_positions() { + let result = parse_query("[nope] = 1", &columns()); + let e = &result.errors()[0]; + assert_eq!((e.start, e.end), (0, 0)); + assert_eq!(e.severity, crate::parser::ErrorSeverity::Error); + } + + #[test] + fn incompatible_operators_on_string() { + assert_eq!( + message("[name] > 1"), + "Operator 'greater than' is not compatible with type 'string'" + ); + assert_eq!( + message("[name] < 1"), + "Operator 'less than' is not compatible with type 'string'" + ); + assert_eq!( + message("[name] >= 1"), + "Operator 'greater than or equal' is not compatible with type 'string'" + ); + assert_eq!( + message("[name] <= 1"), + "Operator 'less than or equal' is not compatible with type 'string'" + ); + } + + #[test] + fn compatible_operators_on_string() { + assert!(is_valid("[name] = \"a\"")); + assert!(is_valid("[name] contains \"a\"")); + assert!(is_valid("[name] starts with \"a\"")); + assert!(is_valid("[name] ends with \"a\"")); + } + + #[test] + fn incompatible_operators_on_number() { + assert_eq!( + message("[age] contains \"x\""), + "Operator 'contains' is not compatible with type 'number'" + ); + assert_eq!( + message("[age] starts with \"x\""), + "Operator 'starts with' is not compatible with type 'number'" + ); + assert_eq!( + message("[age] ends with \"x\""), + "Operator 'ends with' is not compatible with type 'number'" + ); + } + + #[test] + fn compatible_operators_on_number_and_date() { + for op in ["=", ">", "<", ">=", "<="] { + assert!(is_valid(&format!("[age] {op} 1")), "age {op}"); + assert!( + is_valid(&format!("[created] {op} \"2024-01-01\"")), + "created {op}" + ); + } + } + + #[test] + fn incompatible_operators_on_boolean() { + assert_eq!( + message("[active] > 1"), + "Operator 'greater than' is not compatible with type 'boolean'" + ); + assert_eq!( + message("[active] contains \"x\""), + "Operator 'contains' is not compatible with type 'boolean'" + ); + assert!(is_valid("[active] = true")); + assert!(is_valid("[active] = false")); + } + + #[test] + fn incompatible_operators_on_enum() { + assert_eq!( + message("[kind] contains \"a\""), + "Operator 'contains' is not compatible with type 'enum'" + ); + assert_eq!( + message("[kind] > 1"), + "Operator 'greater than' is not compatible with type 'enum'" + ); + assert!(is_valid("[kind] = \"a\"")); + } + + #[test] + fn incompatible_operators_on_date() { + assert_eq!( + message("[created] contains \"a\""), + "Operator 'contains' is not compatible with type 'date'" + ); + } + + #[test] + fn blank_operators_are_string_date_and_enum_only() { + assert!(is_valid("[name] is blank")); + assert!(is_valid("[name] is not blank")); + assert!(is_valid("[created] is blank")); + assert!(is_valid("[kind] is blank")); + assert_eq!( + message("[active] is blank"), + "Operator 'is blank' is not compatible with type 'boolean'" + ); + assert_eq!( + message("[active] is not blank"), + "Operator 'is not blank' is not compatible with type 'boolean'" + ); + assert_eq!( + message("[age] is blank"), + "Operator 'is blank' is not compatible with type 'number'" + ); + } + + #[test] + fn wrong_value_types_are_reported() { + assert_eq!( + message("[age] > \"x\""), + "Expected number for column 'age', got string" + ); + assert_eq!( + message("[age] = true"), + "Expected number for column 'age', got boolean" + ); + assert_eq!( + message("[name] = 1"), + "Expected string for column 'name', got number" + ); + assert_eq!( + message("[name] = true"), + "Expected string for column 'name', got boolean" + ); + assert_eq!( + message("[active] = \"x\""), + "Expected boolean for column 'active', got string" + ); + assert_eq!( + message("[active] = 1"), + "Expected boolean for column 'active', got number" + ); + assert_eq!( + message("[created] > 1"), + "Expected date string for column 'created', got number" + ); + assert_eq!( + message("[kind] = 1"), + "Expected string for enum column 'kind', got number" + ); + } + + #[test] + fn date_check_is_shape_only() { + // Nonsense month and day, accepted because the reference accepts them. + assert!(is_valid("[created] > \"2024-13-99\"")); + assert!(is_valid("[created] > \"0000-00-00\"")); + } + + #[test] + fn date_shapes_that_are_accepted() { + for value in [ + "2024-01-01", + "2024-01-01T10:20:30", + "2024-01-01T10:20:30Z", + "2024-01-01T10:20:30.123", + "2024-01-01T10:20:30.123Z", + ] { + assert!(is_iso_date_shape(value), "{value}"); + assert!(is_valid(&format!("[created] > \"{value}\"")), "{value}"); + } + } + + #[test] + fn date_shapes_that_are_rejected() { + for value in [ + "2024-1-1", + "2024-01-01T10:20:30.12", + "2024-01-01T10:20:30.1234", + "2024-01-01T10:20", + "2024-01-01 10:20:30", + "2024-01-01T", + "2024-01-01Z", + "24-01-01", + "", + "x", + "2024-01-01T10:20:30.123ZZ", + "2024-01-01T10:20:30z", + ] { + assert!(!is_iso_date_shape(value), "{value}"); + } + assert_eq!( + message("[created] > \"2024-1-1\""), + "Invalid date format for column 'created'. Expected YYYY-MM-DD or ISO datetime" + ); + } + + #[test] + fn a_comparison_with_no_value_is_reported() { + // The grammar cannot produce this, but a hand-built AST can. + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![], + )]); + let errors = validate_filter_group(&group, &columns()); + assert_eq!(errors.len(), 1); + assert_eq!( + errors[0].message, + "Operator 'greater than' requires a value" + ); + } + + #[test] + fn nested_groups_are_validated() { + let result = parse_query("([nope] = 1 OR [age] = 2) AND [name] = 3", &columns()); + let messages: Vec<_> = result.errors().iter().map(|e| e.message.clone()).collect(); + assert_eq!( + messages, + vec![ + "Column 'nope' does not exist".to_string(), + "Expected string for column 'name', got number".to_string(), + ] + ); + } + + #[test] + fn an_empty_group_validates_clean() { + assert!(validate_filter_group(&FilterGroup::default(), &columns()).is_empty()); + } + + #[test] + fn a_filter_with_neither_side_is_ignored() { + let group = FilterGroup::and(vec![Filter::default()]); + assert!(validate_filter_group(&group, &columns()).is_empty()); + } + + #[test] + fn all_args_are_checked() { + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::Equals, + vec![json!(1), json!("x"), json!(true)], + )]); + let errors = validate_filter_group(&group, &columns()); + assert_eq!(errors.len(), 2); + assert_eq!( + errors[0].message, + "Expected number for column 'age', got string" + ); + assert_eq!( + errors[1].message, + "Expected number for column 'age', got boolean" + ); + } + + #[test] + fn a_null_arg_reports_the_javascript_typeof() { + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::Equals, + vec![serde_json::Value::Null], + )]); + let errors = validate_filter_group(&group, &columns()); + assert_eq!( + errors[0].message, + "Expected number for column 'age', got object" + ); + } + + #[test] + fn compatibility_helpers_cover_every_pair() { + use FilterFunction::*; + let all = [ + Equals, + GreaterThan, + LessThan, + GreaterThanOrEqual, + LessThanOrEqual, + Contains, + StartsWith, + EndsWith, + ]; + let types = [ + ColumnType::String, + ColumnType::Number, + ColumnType::Boolean, + ColumnType::Date, + ColumnType::Enum, + ]; + // Equality is the one operator every type accepts. + for t in types { + assert!(is_operator_compatible(t, Equals), "{t:?}"); + } + // And every pair has a definite answer, so no combination is unhandled. + for t in types { + for f in all { + let _ = is_operator_compatible(t, f); + } + } + assert!(is_blank_operator_compatible(ColumnType::String)); + assert!(is_blank_operator_compatible(ColumnType::Date)); + assert!(is_blank_operator_compatible(ColumnType::Enum)); + assert!(!is_blank_operator_compatible(ColumnType::Number)); + assert!(!is_blank_operator_compatible(ColumnType::Boolean)); + } + + #[test] + fn normalized_backend_types_validate_like_their_targets() { + let cols = vec![ + ColumnDef::normalized("nAge", "INT32"), + ColumnDef::normalized("tName", "TEXT"), + ColumnDef::normalized("weird", "GUID"), + ]; + assert!(!parse_query("[nAge] > 1", &cols).has_errors()); + assert!(!parse_query("[tName] contains \"a\"", &cols).has_errors()); + // An unknown backend type falls back to string, so `contains` is fine + // and `>` is not. + assert!(!parse_query("[weird] contains \"a\"", &cols).has_errors()); + let result = parse_query("[weird] > 1", &cols); + assert_eq!( + result.errors()[0].message, + "Operator 'greater than' is not compatible with type 'string'" + ); + } +} diff --git a/rusty-filter/tests/frontend_ast_compat.rs b/rusty-filter/tests/frontend_ast_compat.rs new file mode 100644 index 0000000..fddda04 --- /dev/null +++ b/rusty-filter/tests/frontend_ast_compat.rs @@ -0,0 +1,189 @@ +//! The AST this crate serializes must be byte-compatible with the JSON the +//! browser's `filter-query-editor` produces, because both halves of a filter can +//! be authored on either side and a `DataTable` query has to mean one thing. +//! +//! Every expected value below is the **verbatim output of the shipped bundle**, +//! captured by importing `dist/index.js` from +//! `src/frontend/node_modules/filter-query-editor` (version 2.2.0) and calling +//! `parseQuery(query, columns)` with the column schema in [`columns`]. Nothing +//! here is hand-written from the grammar: the point of this file is to catch +//! drift away from the reference, and a hand-written expectation would drift with +//! the implementation instead of pinning it. +//! +//! Reproducing the capture: +//! +//! ```js +//! const B = '/src/frontend/node_modules/filter-query-editor/dist/index.js'; +//! const { parseQuery } = await import(B); +//! console.log(JSON.stringify(parseQuery('[age] > 1', cols))); +//! ``` +//! +//! The one thing that is *not* asserted verbatim is a multi-error cascade: on a +//! syntax error this crate reports the first error and stops, which the crate +//! docs call out as a deliberate divergence. Single-error inputs are compared in +//! full, spans included. + +use rusty_filter::{parse_query, ColumnDef, ColumnType}; + +/// The schema the probe ran with. `n` and `x` exist only to keep the captured +/// mixed `AND`/`OR` query short. +fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("when", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ColumnDef::new("n", ColumnType::String), + ColumnDef::new("x", ColumnType::Boolean), + ] +} + +/// Assert that this crate's `ParseResult` serializes to exactly `expected`. +/// +/// Comparing `serde_json::Value` rather than strings keeps key *presence* +/// significant while leaving key order free, which is what matters: the +/// reference omits `negate` on comparisons and emits `negate: false` on text +/// operations, and a round trip through the browser must not change that. +#[track_caller] +fn assert_bundle_json(query: &str, expected: &str) { + let expected: serde_json::Value = + serde_json::from_str(expected).expect("the captured JSON parses"); + let actual = serde_json::to_value(parse_query(query, &columns())).expect("the AST serializes"); + assert_eq!(actual, expected, "query: {query}"); +} + +#[test] +fn a_comparison_omits_the_negate_key() { + assert_bundle_json( + "[age] > 1", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}}]}}"#, + ); + assert_bundle_json( + "[age] = 100", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[100]}}]}}"#, + ); +} + +#[test] +fn a_text_operation_emits_negate_false() { + assert_bundle_json( + r#"[name] contains "ab""#, + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"contains","args":["ab"]},"negate":false}]}}"#, + ); + assert_bundle_json( + r#"[name] starts with "a""#, + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"startsWith","args":["a"]},"negate":false}]}}"#, + ); +} + +#[test] +fn negated_forms_carry_negate_true() { + // `!=` is `equals` with `negate: true` — there is no `notEquals` function. + assert_bundle_json( + "[age] != 5", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[5]},"negate":true}]}}"#, + ); + assert_bundle_json( + r#"[name] not contains "ab""#, + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"contains","args":["ab"]},"negate":true}]}}"#, + ); + // `NOT` lands on the condition filter itself, not on a wrapper. + assert_bundle_json( + "NOT [age] > 1", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]},"negate":true}]}}"#, + ); + // And it *toggles*, so a double `NOT` is `false` rather than absent. + assert_bundle_json( + "not not [age] > 1", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]},"negate":false}]}}"#, + ); +} + +#[test] +fn an_existence_operation_has_empty_args_and_no_negate() { + assert_bundle_json( + "[name] is blank", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"isBlank","args":[]}}]}}"#, + ); + assert_bundle_json( + "[name] is not blank", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"isNotBlank","args":[]}}]}}"#, + ); +} + +#[test] +fn mixed_and_or_nests_the_and_arm_as_a_group() { + // `OR` is the root, and the `AND` arm is spliced in as `{group: ...}`. + assert_bundle_json( + r#"[age] > 1 AND [n] = "a" OR [x] = true"#, + r#"{"filters":{"op":"OR","filters":[{"group":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}},{"condition":{"column":"n","function":"equals","args":["a"]}}]}},{"condition":{"column":"x","function":"equals","args":[true]}}]}}"#, + ); +} + +#[test] +fn parentheses_are_never_collapsed() { + assert_bundle_json( + "(([age] > 1))", + r#"{"filters":{"op":"AND","filters":[{"group":{"op":"AND","filters":[{"group":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}}]}}]}}]}}"#, + ); +} + +#[test] +fn an_empty_query_is_an_empty_and_group() { + let empty = r#"{"filters":{"op":"AND","filters":[]}}"#; + assert_bundle_json("", empty); + assert_bundle_json(" ", empty); +} + +#[test] +fn numbers_serialize_as_javascript_writes_them() { + // JavaScript has one number type, so `007` is `7` and an integral fraction + // loses its decimal point. Whitespace inside a number is skipped. + assert_bundle_json( + "[age] = 007", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[7]}}]}}"#, + ); + assert_bundle_json( + "[age] = 1 . 5", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[1.5]}}]}}"#, + ); + assert_bundle_json( + "[age] = 1.000", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[1]}}]}}"#, + ); +} + +#[test] +fn a_semantic_error_matches_the_reference_wording_and_zero_span() { + assert_bundle_json( + "[nope] = 1", + r#"{"errors":[{"message":"Column 'nope' does not exist","start":0,"end":0,"severity":"error"}]}"#, + ); + assert_bundle_json( + r#"[age] contains "x""#, + r#"{"errors":[{"message":"Operator 'contains' is not compatible with type 'number'","start":0,"end":0,"severity":"error"}]}"#, + ); + assert_bundle_json( + r#"[age] > "x""#, + r#"{"errors":[{"message":"Expected number for column 'age', got string","start":0,"end":0,"severity":"error"}]}"#, + ); +} + +#[test] +fn a_syntax_error_matches_the_reference_message_and_span() { + // Single-error inputs only: the ANTLR cascade is out of scope, so an input + // the reference reports three errors for would not compare here. + assert_bundle_json( + "[age] > 1 1", + r#"{"errors":[{"message":"extraneous input '1' expecting ","start":10,"end":11,"severity":"error"}]}"#, + ); + assert_bundle_json( + "[age] = 1e5", + r#"{"errors":[{"message":"token recognition error at: 'e5'","start":9,"end":10,"severity":"error"}]}"#, + ); + assert_bundle_json( + "([age] > 1", + r#"{"errors":[{"message":"missing ')' at ''","start":10,"end":10,"severity":"error"}]}"#, + ); +} diff --git a/rusty-filter/tests/round_trip.rs b/rusty-filter/tests/round_trip.rs new file mode 100644 index 0000000..5d1db97 --- /dev/null +++ b/rusty-filter/tests/round_trip.rs @@ -0,0 +1,263 @@ +//! Printing an AST and parsing the result back. +//! +//! This is what makes [`canonical_key`] usable as a cache key: two subscribers +//! who wrote the same filter differently must land on one string, and that string +//! must still mean what they wrote. +//! +//! Assertions here are at the **AST** level, never on string equality. +//! `canonical_key` is not string-idempotent on the first pass — `[name] not +//! contains "a"` prints as `NOT ([name] contains "a")`, which is a different +//! source text — so a string round trip would fail on correct output. +//! +//! # Negated leaves are not AST-stable, in either implementation +//! +//! AST stability does not hold universally, and the reference does not have it +//! either. A negated *leaf* prints as `NOT (...)`, and re-parsing that reads the +//! parentheses as a group, so the negation moves from the condition filter onto a +//! wrapping group filter: +//! +//! ```text +//! [age] != 100 +//! -> {op:AND, filters:[{condition:{...equals...}, negate:true}]} +//! -> prints as "NOT ([age] equals 100)" +//! -> {op:AND, filters:[{group:{op:AND, filters:[{condition:{...}}]}, negate:true}]} +//! ``` +//! +//! Measured against `filter-query-editor` 2.2.0 over the 47 queries in +//! [`QUERIES`]: 39 are AST-stable and the 8 in [`NOT_AST_STABLE`] are not, with +//! the bundle producing the very same reshaped AST this crate does. So the +//! divergence is shared, not introduced here — and the tests below pin both +//! halves: exact AST equality for the stable set, and semantic equivalence plus +//! string idempotence for the rest. + +use rusty_filter::{ + canonical_key, evaluate, parse_query, parse_query_unchecked, to_query_string, ColumnDef, + ColumnType, FilterGroup, +}; + +fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("when", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ] +} + +/// Rows covering the truth table of every operator the queries below use. +fn rows() -> Vec { + vec![ + serde_json::json!({"age": 100, "name": "ab", "active": true, "when": "2024-01-01", "kind": "a"}), + serde_json::json!({"age": 101, "name": "abc", "active": false, "when": "2025-06-30T00:00:00Z", "kind": ""}), + serde_json::json!({"age": 1, "name": "zab", "active": true, "when": "2023-01-01", "kind": "b"}), + serde_json::json!({"age": -5, "name": "", "active": false, "when": null, "kind": null}), + serde_json::json!({"age": 1.5, "name": "a\"b", "active": true}), + serde_json::json!({"age": 7, "name": "a\\b", "active": false, "when": "2024-01-01"}), + serde_json::json!({}), + ] +} + +/// Every valid query shape the grammar admits, one per feature. +const QUERIES: &[&str] = &[ + // comparisons, symbolic and spelled out + "[age] > 100", + "[age] >= 100", + "[age] < 100", + "[age] <= 100", + "[age] = 100", + "[age] == 100", + "[age] != 100", + "[age] equals 100", + "[age] not equals 100", + "[age] not equal 100", + "[age] greater than 100", + "[age] greater than or equal 100", + "[age] less than 100", + "[age] less than or equal 100", + // text operations, negated and not + r#"[name] contains "ab""#, + r#"[name] not contains "ab""#, + r#"[name] starts with "ab""#, + r#"[name] not starts with "ab""#, + r#"[name] ends with "ab""#, + r#"[name] not ends with "ab""#, + // existence + "[name] is blank", + "[name] is not blank", + "[when] is blank", + "[kind] is not blank", + // literals + "[active] = true", + "[active] = false", + "[age] = -5", + "[age] = 1.5", + "[age] = 007", + r#"[when] = "2024-01-01""#, + r#"[when] > "2024-01-01T10:20:30.123Z""#, + // strings needing escapes + r#"[name] = "a\"b""#, + r#"[name] = "a\\b""#, + // logical structure + r#"[age] > 1 AND [name] = "a""#, + r#"[age] > 1 OR [name] = "a""#, + r#"[age] > 1 AND [name] = "a" AND [active] = true"#, + r#"[age] > 1 OR [name] = "a" OR [active] = true"#, + r#"[age] > 1 AND [name] = "a" OR [active] = true"#, + r#"[age] > 1 OR [name] = "a" AND [active] = true"#, + // negation and parentheses + "NOT [age] > 1", + "not not [age] > 1", + "(([age] > 1))", + r#"NOT ([age] > 1 AND [name] = "a")"#, + r#"([age] > 1 AND [name] = "a") OR [active] = true"#, + r#"([age] > 1 OR [name] = "a") AND [active] = true"#, + // empty + "", + " ", +]; + +/// The queries whose AST is reshaped by a print/parse round trip, because their +/// negation sits on a leaf and printing lifts it onto a `NOT (...)` group. +/// +/// This list is the bundle's, not this crate's: each entry was confirmed to be +/// AST-unstable in `filter-query-editor` 2.2.0 as well. Anything not listed here +/// must be exactly stable, which [`printing_then_reparsing_preserves_the_ast`] +/// enforces — so an implementation change that quietly reshapes one more query +/// fails the suite rather than growing this list. +const NOT_AST_STABLE: &[&str] = &[ + "[age] != 100", + "[age] not equals 100", + "[age] not equal 100", + r#"[name] not contains "ab""#, + r#"[name] not starts with "ab""#, + r#"[name] not ends with "ab""#, + "NOT [age] > 1", + "not not [age] > 1", +]; + +fn parse(query: &str) -> FilterGroup { + parse_query(query, &columns()) + .filters + .unwrap_or_else(|| panic!("{query} should parse")) +} + +#[test] +fn printing_then_reparsing_preserves_the_ast() { + let mut stable = 0; + for query in QUERIES { + let first = parse(query); + let printed = to_query_string(&first); + let second = parse(&printed); + if NOT_AST_STABLE.contains(query) { + assert_ne!( + second, first, + "{query} is listed as AST-unstable but round-tripped cleanly — \ + drop it from NOT_AST_STABLE" + ); + } else { + assert_eq!(second, first, "{query} printed as {printed:?}"); + stable += 1; + } + } + // Pins the split measured against the bundle: 39 of 47 stable. + assert_eq!(stable, QUERIES.len() - NOT_AST_STABLE.len()); + assert_eq!(stable, 39); +} + +#[test] +fn a_reshaped_round_trip_still_means_the_same_thing() { + // The negated-leaf reshaping moves `negate` onto a wrapping group, which is a + // different AST but the same predicate. That is the property that actually + // matters for a cache key, so it is asserted for *every* query rather than + // only the unstable ones. + for query in QUERIES { + let first = parse(query); + let second = parse(&to_query_string(&first)); + let cols = columns(); + for row in rows() { + assert_eq!( + evaluate(&first, &row, &cols), + evaluate(&second, &row, &cols), + "{query} disagreed on row {row}" + ); + } + } +} + +#[test] +fn a_second_round_trip_is_a_fixed_point() { + // Printing is idempotent from the second pass onwards even where the AST is + // reshaped on the first, so a cache key derived from a cache key is the same + // key. Without this, `canonical_key` would not be canonical. + for query in QUERIES { + let once = to_query_string(&parse(query)); + let twice = to_query_string(&parse(&once)); + assert_eq!(twice, once, "{query}"); + let thrice = to_query_string(&parse(&twice)); + assert_eq!(thrice, once, "{query}"); + } +} + +#[test] +fn equivalent_spellings_share_one_cache_key() { + for [a, b] in [ + ["[age] > 100", "[age] greater than 100"], + ["[age] >= 100", "[age] greater than or equal 100"], + ["[age] < 100", "[age] less than 100"], + ["[age] <= 100", "[age] less than or equal 100"], + ["[age] = 100", "[age] == 100"], + ["[age] != 100", "[age] not equals 100"], + ["[age] != 100", "[age] not equal 100"], + ["[age] = 7", "[age] = 007"], + ["[age] = 1.5", "[age] = 1 . 5"], + ["[age] > 1", "not not [age] > 1"], + // Keyword case is folded, so only the keywords differ here. + ["[age] > 1", "[age] GREATER THAN 1"], + [r#"[name] contains "a""#, r#"[name] CONTAINS "a""#], + ] { + let key_a = canonical_key(&parse_query_unchecked(a).expect("valid")); + let key_b = canonical_key(&parse_query_unchecked(b).expect("valid")); + assert_eq!(key_a, key_b, "{a:?} vs {b:?}"); + } +} + +#[test] +fn a_column_name_keeps_its_case_in_the_key() { + // Keyword case is folded but a column name is not, so two differently-cased + // column names are two different cache keys — as they must be, since the + // column lookup at evaluation time is case-sensitive too. + let lower = canonical_key(&parse_query_unchecked("[name] = \"a\"").expect("valid")); + let upper = canonical_key(&parse_query_unchecked("[NAME] = \"a\"").expect("valid")); + assert_ne!(lower, upper); +} + +#[test] +fn differently_meaning_queries_get_different_keys() { + // The other half of a cache key's contract: collapsing two *different* + // filters onto one key would serve one subscriber the other's rows. + let queries = [ + "[age] > 1", + "[age] >= 1", + "[age] > 2", + "[age] < 1", + "NOT [age] > 1", + r#"[age] > 1 AND [name] = "a""#, + r#"[age] > 1 OR [name] = "a""#, + r#"[name] contains "a""#, + r#"[name] not contains "a""#, + r#"[name] starts with "a""#, + r#"[name] ends with "a""#, + "[name] is blank", + "[name] is not blank", + "", + ]; + let keys: Vec = queries + .iter() + .map(|q| canonical_key(&parse_query_unchecked(q).expect("valid"))) + .collect(); + let mut unique = keys.clone(); + unique.sort(); + unique.dedup(); + assert_eq!(unique.len(), keys.len(), "keys collided: {keys:?}"); +} diff --git a/rusty/Cargo.toml b/rusty/Cargo.toml index 2f94d45..adc678a 100644 --- a/rusty/Cargo.toml +++ b/rusty/Cargo.toml @@ -16,6 +16,7 @@ tracing.workspace = true tower-http.workspace = true bytes.workspace = true rusty-macros = { path = "../rusty-macros" } +rusty-filter = { path = "../rusty-filter" } rusty-ivyml = { path = "../rusty-ivyml" } [dev-dependencies] diff --git a/rusty/src/core/query_cache.rs b/rusty/src/core/query_cache.rs index 8e15b80..e3ca3a8 100644 --- a/rusty/src/core/query_cache.rs +++ b/rusty/src/core/query_cache.rs @@ -685,6 +685,40 @@ impl QueryService { } } + /// Build a cache key from a base key and a parsed filter, so two subscribers + /// with equivalent filters share one entry. + /// + /// `base` comes back unchanged when there is nothing to filter by — `None`, + /// or a group holding no filters, which is what an empty query parses to. + /// Otherwise the filter follows a `?` separator in the canonical spelling + /// [`rusty_filter::canonical_key`] produces, so `[age] > 1` and + /// `[age] greater than 1` are one key rather than two. + /// + /// An associated function, not a method: it reads no cache state, which also + /// keeps it clear of this module's rule that the cache lock is never held + /// across an `.await`. + /// + /// ``` + /// use rusty::core::QueryService; + /// use rusty_filter::parse_query_unchecked; + /// + /// let a = parse_query_unchecked("[age] > 1").unwrap(); + /// let b = parse_query_unchecked("[age] greater than 1").unwrap(); + /// assert_eq!( + /// QueryService::filtered_key("people", Some(&a)), + /// QueryService::filtered_key("people", Some(&b)), + /// ); + /// assert_eq!(QueryService::filtered_key("people", None), "people"); + /// ``` + pub fn filtered_key(base: &str, filter: Option<&rusty_filter::FilterGroup>) -> String { + match filter { + Some(filter) if !filter.filters.is_empty() => { + format!("{base}?{}", rusty_filter::canonical_key(filter)) + } + _ => base.to_string(), + } + } + /// Transition an entry into `Fetching`/`Revalidating` and return the fetcher /// to spawn, or `None` when a fetch is already running. fn begin_fetch(entry: &mut QueryEntry, revalidating: bool) -> Option { @@ -1556,4 +1590,69 @@ mod tests { assert_eq!(service.entry_state("b"), Some(QueryEntryState::Empty)); assert_eq!(service.peek::("a"), None); } + + fn filter(query: &str) -> rusty_filter::FilterGroup { + rusty_filter::parse_query_unchecked(query).expect("valid query") + } + + #[test] + fn test_filtered_key_returns_the_base_when_there_is_no_filter() { + assert_eq!(QueryService::filtered_key("people", None), "people"); + // An empty query parses to an empty group, so a cleared filter box must + // land back on the unfiltered entry instead of creating `people?`. + assert_eq!( + QueryService::filtered_key("people", Some(&filter(""))), + "people" + ); + assert_eq!( + QueryService::filtered_key("people", Some(&rusty_filter::FilterGroup::default())), + "people" + ); + } + + #[test] + fn test_filtered_key_appends_the_canonical_filter() { + // The canonical spelling is the printer's, which is `>` rather than + // `greater than` — so the key does not echo the query as written. + assert_eq!( + QueryService::filtered_key("people", Some(&filter("[age] greater than 1"))), + "people?[age] > 1" + ); + assert_eq!( + QueryService::filtered_key("people", Some(&filter(r#"[name] CONTAINS "a""#))), + r#"people?[name] contains "a""# + ); + } + + #[test] + fn test_equivalent_filters_share_one_key() { + // The point of keying on the canonical spelling: two subscribers who + // wrote the same filter differently must not each get their own fetch. + for [a, b] in [ + ["[age] > 1", "[age] greater than 1"], + ["[age] = 7", "[age] = 007"], + ["[age] >= 1", "[age] GREATER THAN OR EQUAL 1"], + ] { + assert_eq!( + QueryService::filtered_key("people", Some(&filter(a))), + QueryService::filtered_key("people", Some(&filter(b))), + "{a:?} vs {b:?}" + ); + } + } + + #[test] + fn test_different_filters_get_different_keys() { + let keys = [ + QueryService::filtered_key("people", None), + QueryService::filtered_key("people", Some(&filter("[age] > 1"))), + QueryService::filtered_key("people", Some(&filter("[age] > 2"))), + QueryService::filtered_key("people", Some(&filter("[age] >= 1"))), + QueryService::filtered_key("other", Some(&filter("[age] > 1"))), + ]; + let mut unique = keys.to_vec(); + unique.sort(); + unique.dedup(); + assert_eq!(unique.len(), keys.len(), "keys collided: {keys:?}"); + } } diff --git a/rusty/src/lib.rs b/rusty/src/lib.rs index 1cad54c..b72694f 100644 --- a/rusty/src/lib.rs +++ b/rusty/src/lib.rs @@ -25,6 +25,14 @@ pub mod prelude { SubmitHandler, Validator, View, }; pub use crate::widgets::*; + /// The crate itself, so `rusty_filter::lexer` and the other modules are + /// reachable without a second dependency line in `Cargo.toml`. + pub use rusty_filter; + pub use rusty_filter::{ + canonical_key, count_matches, evaluate, parse_query, parse_query_unchecked, + retain_matching, to_query_string, validate_filter_group, ColumnDef, ColumnType, Condition, + ErrorSeverity, Filter, FilterFunction, FilterGroup, LogicalOp, ParseError, ParseResult, + }; } // Re-export the derive macro diff --git a/rusty/src/widgets/data_table.rs b/rusty/src/widgets/data_table.rs index 3bd023b..33888b6 100644 --- a/rusty/src/widgets/data_table.rs +++ b/rusty/src/widgets/data_table.rs @@ -1,5 +1,6 @@ use crate::shared::{Align, Color, Icon, Size}; use crate::views::view::Element; +use rusty_filter::{ColumnDef, ColumnType, ParseError}; use rusty_macros::Widget; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -19,6 +20,23 @@ pub enum ColType { Link, } +/// A `DataTable` has eight cell types but the filter grammar's validator knows +/// only five, so several collapse onto one. The mapping is the one +/// [`ColumnType::normalize`] applies to the backend type names, which is what +/// keeps a Rust-side filter agreeing with the browser's: `Icon` normalizes to +/// `String` there, and `Labels` and `Link` are unknown names that fall back to +/// `String`. +impl From for ColumnType { + fn from(col_type: ColType) -> Self { + match col_type { + ColType::Number => ColumnType::Number, + ColType::Boolean => ColumnType::Boolean, + ColType::Date | ColType::DateTime => ColumnType::Date, + ColType::Text | ColType::Icon | ColType::Labels | ColType::Link => ColumnType::String, + } + } +} + /// Sort direction applied to a column. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -276,7 +294,14 @@ pub struct RowActionArgs { /// A typed data grid with per-column formatting, sorting and cell events. /// /// Rows travel inline in the widget JSON, exactly as [`crate::widgets::Table`] does. -/// Ivy's `DataTableConnection` server-side query pipeline is not ported. +/// +/// Ivy's `DataTableConnection` server-side query pipeline is not ported. What is +/// available is filtering: [`DataTable::apply_filter`] runs a query in the +/// grammar the browser's filter editor uses and keeps the matching rows. It has +/// to be called by the app, though — the filter box in the rendered table talks +/// to a gRPC `DataTableService` that Rusty does not implement, so typing there +/// still goes nowhere. Sorting, aggregation and pagination are likewise not +/// ported. #[derive(Clone, Serialize, Deserialize, Widget)] pub struct DataTable { #[serde(skip_serializing_if = "Option::is_none")] @@ -363,6 +388,62 @@ impl DataTable { self } + /// The columns a filter query may name, as the filter grammar sees them. + /// + /// A column is offered only when it is `filterable` and not `hidden`, which + /// is the same test `DataTableFilterOption.tsx` applies before handing + /// columns to the browser's editor. Naming an excluded column is therefore a + /// `Column 'x' does not exist` error on both sides rather than a filter that + /// works on one of them. + pub fn filter_columns(&self) -> Vec { + self.columns + .iter() + .filter(|c| c.filterable && !c.hidden) + .map(|c| ColumnDef::new(c.name.clone(), c.col_type.into())) + .collect() + } + /// Keep only the rows matching `query`, which is parsed and validated against + /// [`DataTable::filter_columns`]. + /// + /// An empty or whitespace-only query is valid and matches every row, so a + /// cleared filter box needs no special case. + /// + /// # Errors + /// + /// Returns the parse and validation errors when `query` is not a valid + /// filter. A syntax error is reported alone; semantic errors come one per + /// offending condition. + /// + /// ``` + /// use rusty::widgets::{ColType, DataTable, DataTableColumn}; + /// + /// let table = DataTable::new(vec![ + /// DataTableColumn::new("name", "Name", ColType::Text), + /// DataTableColumn::new("age", "Age", ColType::Number), + /// ]) + /// .rows(vec![ + /// serde_json::json!({"name": "Ada", "age": 41}), + /// serde_json::json!({"name": "Bob", "age": 12}), + /// ]); + /// + /// let filtered = table.apply_filter("[age] > 18").expect("valid query"); + /// assert_eq!(filtered.rows.len(), 1); + /// assert_eq!(filtered.rows[0]["name"], "Ada"); + /// ``` + pub fn apply_filter(mut self, query: &str) -> Result> { + let columns = self.filter_columns(); + let result = rusty_filter::parse_query(query, &columns); + match result.filters { + Some(filter) => { + self.rows = rusty_filter::retain_matching(&filter, self.rows, &columns); + Ok(self) + } + // `filters` and `errors` are never both populated, so the fallback + // here is unreachable rather than a silently empty error list. + None => Err(result.errors.unwrap_or_default()), + } + } + pub fn into_element(self) -> Element { Element::Widget(Box::new(self)) } @@ -600,4 +681,117 @@ mod tests { let el: Element = DataTable::new(vec![]).into(); assert!(matches!(el, Element::Widget(_))); } + + #[test] + fn test_col_type_maps_onto_a_filter_column_type() { + // All eight variants, so adding a ninth to `ColType` fails to compile + // rather than silently filtering as text. + for (col_type, expected) in [ + (ColType::Number, ColumnType::Number), + (ColType::Boolean, ColumnType::Boolean), + (ColType::Date, ColumnType::Date), + (ColType::DateTime, ColumnType::Date), + (ColType::Text, ColumnType::String), + (ColType::Icon, ColumnType::String), + (ColType::Labels, ColumnType::String), + (ColType::Link, ColumnType::String), + ] { + assert_eq!(ColumnType::from(col_type), expected, "{col_type:?}"); + } + } + + #[test] + fn test_filter_columns_excludes_non_filterable_and_hidden() { + let table = DataTable::new(vec![ + DataTableColumn::new("name", "Name", ColType::Text), + DataTableColumn::new("age", "Age", ColType::Number), + DataTableColumn::new("secret", "Secret", ColType::Text).filterable(false), + DataTableColumn::new("internal", "Internal", ColType::Text).hidden(true), + // Excluded by either test alone, so still excluded by both. + DataTableColumn::new("both", "Both", ColType::Text) + .filterable(false) + .hidden(true), + ]); + + let columns = table.filter_columns(); + assert_eq!( + columns, + vec![ + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("age", ColumnType::Number), + ] + ); + } + + fn people() -> DataTable { + DataTable::new(vec![ + DataTableColumn::new("name", "Name", ColType::Text), + DataTableColumn::new("age", "Age", ColType::Number), + DataTableColumn::new("secret", "Secret", ColType::Text).filterable(false), + ]) + .rows(vec![ + json!({"name": "Ada", "age": 41, "secret": "x"}), + json!({"name": "Bob", "age": 12, "secret": "y"}), + json!({"name": "Ann", "age": 30, "secret": "x"}), + ]) + } + + #[test] + fn test_apply_filter_keeps_only_matching_rows() { + let filtered = people() + .apply_filter(r#"[age] > 18 AND [name] starts with "A""#) + .expect("valid query"); + let names: Vec<&str> = filtered + .rows + .iter() + .map(|r| r["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, ["Ada", "Ann"]); + // Everything else about the table survives. + assert_eq!(filtered.columns.len(), 3); + } + + #[test] + fn test_apply_filter_with_an_empty_query_keeps_every_row() { + assert_eq!(people().apply_filter("").expect("valid").rows.len(), 3); + assert_eq!(people().apply_filter(" ").expect("valid").rows.len(), 3); + } + + #[test] + fn test_apply_filter_on_a_non_filterable_column_says_it_does_not_exist() { + // The column is on the table but not in `filter_columns`, so the query + // gets the same error the browser's editor would give for a name it was + // never offered. + let errors = people() + .apply_filter(r#"[secret] = "x""#) + .expect_err("secret is not filterable"); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].message, "Column 'secret' does not exist"); + } + + #[test] + fn test_apply_filter_reports_a_syntax_error() { + let errors = people().apply_filter("[age] > ").expect_err("no operand"); + assert_eq!(errors.len(), 1); + assert!( + errors[0].message.contains(""), + "unexpected message: {}", + errors[0].message + ); + } + + #[test] + fn test_apply_filter_filters_a_datetime_column_by_iso_order() { + // `DateTime` normalizes to the grammar's `date`, so an ISO string + // comparison works rather than falling back to text comparison. + let table = DataTable::new(vec![DataTableColumn::new("at", "At", ColType::DateTime)]) + .rows(vec![ + json!({"at": "2024-06-01T09:00:00Z"}), + json!({"at": "2023-01-01T09:00:00Z"}), + ]) + .apply_filter(r#"[at] > "2024-01-01""#) + .expect("valid query"); + assert_eq!(table.rows.len(), 1); + assert_eq!(table.rows[0]["at"], "2024-06-01T09:00:00Z"); + } }