From 786324d41488201912f5f1b613c494c34806335f Mon Sep 17 00:00:00 2001 From: Mikael Rinne <40919111+rorychatt@users.noreply.github.com> Date: Sun, 2 Aug 2026 12:06:10 +0200 Subject: [PATCH 1/3] [00123] Add the rusty-filter crate: lexer, parser, AST, validator, evaluator and printer A dependency-light Rust port of the filter query grammar the frontend's filter-query-editor 2.2.0 bundle implements, so a query can be authored on either side and mean the same thing. Only serde and serde_json; no ANTLR runtime, no build script. Every behaviour was cross-checked against the shipped bundle by replaying a corpus through both implementations: 42 valid queries matched on both AST JSON and formatQuery output, 46 error cases matched on first message and byte span, and 29 filters over 6 rows matched on evaluateFilter and countMatches. Divergences, all documented in the module docs: error spans are byte offsets rather than UTF-16 code units, and a syntax error stops the parse instead of recovering into an ANTLR error cascade. --- Cargo.lock | 8 + Cargo.toml | 2 +- rusty-filter/Cargo.toml | 11 + rusty-filter/src/ast.rs | 304 +++++++ rusty-filter/src/column.rs | 166 ++++ rusty-filter/src/eval.rs | 846 ++++++++++++++++++ rusty-filter/src/lexer.rs | 792 +++++++++++++++++ rusty-filter/src/lib.rs | 92 ++ rusty-filter/src/parser.rs | 1617 ++++++++++++++++++++++++++++++++++ rusty-filter/src/print.rs | 526 +++++++++++ rusty-filter/src/validate.rs | 603 +++++++++++++ 11 files changed, 4966 insertions(+), 1 deletion(-) create mode 100644 rusty-filter/Cargo.toml create mode 100644 rusty-filter/src/ast.rs create mode 100644 rusty-filter/src/column.rs create mode 100644 rusty-filter/src/eval.rs create mode 100644 rusty-filter/src/lexer.rs create mode 100644 rusty-filter/src/lib.rs create mode 100644 rusty-filter/src/parser.rs create mode 100644 rusty-filter/src/print.rs create mode 100644 rusty-filter/src/validate.rs diff --git a/Cargo.lock b/Cargo.lock index 3e90f60..d75c3cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -795,6 +795,14 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rusty-filter" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "rusty-macros" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c7b5ff7..b650ec1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rusty", "rusty-macros", "rusty-server", "rusty-docs"] +members = ["rusty", "rusty-filter", "rusty-macros", "rusty-server", "rusty-docs"] resolver = "2" [workspace.package] diff --git a/rusty-filter/Cargo.toml b/rusty-filter/Cargo.toml new file mode 100644 index 0000000..4215181 --- /dev/null +++ b/rusty-filter/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rusty-filter" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Filter query grammar parser and AST for Rusty-Framework" + +[dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/rusty-filter/src/ast.rs b/rusty-filter/src/ast.rs new file mode 100644 index 0000000..a344c93 --- /dev/null +++ b/rusty-filter/src/ast.rs @@ -0,0 +1,304 @@ +//! The filter AST, mirroring `dist/types/filter.d.ts` of `filter-query-editor`. +//! +//! The shapes here are deliberately faithful to the TypeScript interfaces, +//! `Option` placement included, so that `serde_json` output is byte-compatible +//! with what the browser's editor produces. See the crate-level docs for the +//! rules governing which keys are present. + +use serde::{Deserialize, Serialize}; + +/// How the filters of a group are combined. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum LogicalOp { + /// Every filter must match. This is the default, matching the empty-query + /// result `{op: "AND", filters: []}`. + #[default] + #[serde(rename = "AND")] + And, + #[serde(rename = "OR")] + Or, +} + +/// The ten functions the reference implementation emits. +/// +/// There is deliberately no `NotEquals`: `!=`, `not equals` and `not equal` all +/// produce [`FilterFunction::Equals`] with `negate: Some(true)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FilterFunction { + Equals, + GreaterThan, + LessThan, + GreaterThanOrEqual, + LessThanOrEqual, + Contains, + StartsWith, + EndsWith, + IsBlank, + IsNotBlank, +} + +impl FilterFunction { + /// The human-readable name used in validation messages, from + /// `getOperatorDisplayName` in `dist/validator/TypeChecker.js`. + pub fn display_name(self) -> &'static str { + match self { + FilterFunction::Equals => "equals", + FilterFunction::GreaterThan => "greater than", + FilterFunction::LessThan => "less than", + FilterFunction::GreaterThanOrEqual => "greater than or equal", + FilterFunction::LessThanOrEqual => "less than or equal", + FilterFunction::Contains => "contains", + FilterFunction::StartsWith => "starts with", + FilterFunction::EndsWith => "ends with", + FilterFunction::IsBlank => "is blank", + FilterFunction::IsNotBlank => "is not blank", + } + } + + /// Whether this is one of the two argument-less existence operators. + pub fn is_blank_operator(self) -> bool { + matches!(self, FilterFunction::IsBlank | FilterFunction::IsNotBlank) + } +} + +/// A single filter expression, e.g. `[status] equals "open"`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Condition { + /// The column identifier, exactly as written between the brackets. + pub column: String, + /// The comparison or text operation. + pub function: FilterFunction, + /// Values to compare against. Empty for the two blank operators. + pub args: Vec, +} + +impl Condition { + pub fn new( + column: impl Into, + function: FilterFunction, + args: Vec, + ) -> Self { + Condition { + column: column.into(), + function, + args, + } + } +} + +/// A collection of filters combined with `AND` or `OR`. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct FilterGroup { + pub op: LogicalOp, + pub filters: Vec, +} + +impl FilterGroup { + /// A group combining `filters` with `AND`. + pub fn and(filters: Vec) -> Self { + FilterGroup { + op: LogicalOp::And, + filters, + } + } + + /// A group combining `filters` with `OR`. + pub fn or(filters: Vec) -> Self { + FilterGroup { + op: LogicalOp::Or, + filters, + } + } + + /// Whether this group holds no filters. An empty group is what the empty + /// query parses to, and it is the case [`crate::print::canonical_key`] + /// treats as "no filter at all". + pub fn is_empty(&self) -> bool { + self.filters.is_empty() + } +} + +/// One entry of a [`FilterGroup`]: a condition or a nested group, optionally +/// negated. +/// +/// `condition` and `group` are mutually exclusive but both optional, as in the +/// TypeScript interface. `negate` is `Option` on purpose: comparisons omit +/// the key entirely, text operations always emit it (`false` included). +/// Round-tripping the frontend's JSON must not change which keys are present. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct Filter { + #[serde(skip_serializing_if = "Option::is_none")] + pub condition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub group: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub negate: Option, +} + +impl Filter { + /// A condition filter with no `negate` key, as a comparison produces. + pub fn condition( + column: impl Into, + function: FilterFunction, + args: Vec, + ) -> Self { + Filter { + condition: Some(Condition::new(column, function, args)), + group: None, + negate: None, + } + } + + /// A filter wrapping a nested group, as `(...)` produces. + pub fn from_group(group: FilterGroup) -> Self { + Filter { + condition: None, + group: Some(group), + negate: None, + } + } + + /// This filter with `negate: Some(negate)`. Passing `false` emits the key + /// with a `false` value rather than dropping it, which is what a text + /// operation without `NOT` does. + pub fn negated(mut self, negate: bool) -> Self { + self.negate = Some(negate); + self + } + + /// Whether negation is in effect. A missing key and `Some(false)` both mean + /// "not negated", exactly as the JavaScript truthiness test does. + pub fn is_negated(&self) -> bool { + self.negate.unwrap_or(false) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn logical_op_defaults_to_and() { + assert_eq!(LogicalOp::default(), LogicalOp::And); + assert_eq!(FilterGroup::default(), FilterGroup::and(vec![])); + } + + #[test] + fn logical_op_serializes_uppercase() { + assert_eq!(serde_json::to_value(LogicalOp::And).unwrap(), json!("AND")); + assert_eq!(serde_json::to_value(LogicalOp::Or).unwrap(), json!("OR")); + } + + #[test] + fn filter_function_serializes_camel_case() { + let pairs = [ + (FilterFunction::Equals, "equals"), + (FilterFunction::GreaterThan, "greaterThan"), + (FilterFunction::LessThan, "lessThan"), + (FilterFunction::GreaterThanOrEqual, "greaterThanOrEqual"), + (FilterFunction::LessThanOrEqual, "lessThanOrEqual"), + (FilterFunction::Contains, "contains"), + (FilterFunction::StartsWith, "startsWith"), + (FilterFunction::EndsWith, "endsWith"), + (FilterFunction::IsBlank, "isBlank"), + (FilterFunction::IsNotBlank, "isNotBlank"), + ]; + for (func, expected) in pairs { + assert_eq!(serde_json::to_value(func).unwrap(), json!(expected)); + } + } + + #[test] + fn display_names_match_the_reference() { + assert_eq!(FilterFunction::Equals.display_name(), "equals"); + assert_eq!(FilterFunction::GreaterThan.display_name(), "greater than"); + assert_eq!(FilterFunction::LessThan.display_name(), "less than"); + assert_eq!( + FilterFunction::GreaterThanOrEqual.display_name(), + "greater than or equal" + ); + assert_eq!( + FilterFunction::LessThanOrEqual.display_name(), + "less than or equal" + ); + assert_eq!(FilterFunction::Contains.display_name(), "contains"); + assert_eq!(FilterFunction::StartsWith.display_name(), "starts with"); + assert_eq!(FilterFunction::EndsWith.display_name(), "ends with"); + assert_eq!(FilterFunction::IsBlank.display_name(), "is blank"); + assert_eq!(FilterFunction::IsNotBlank.display_name(), "is not blank"); + } + + #[test] + fn absent_negate_is_omitted_from_json() { + let f = Filter::condition("age", FilterFunction::GreaterThan, vec![json!(1)]); + let value = serde_json::to_value(&f).unwrap(); + assert_eq!( + value, + json!({"condition": {"column": "age", "function": "greaterThan", "args": [1]}}) + ); + assert!(value.get("negate").is_none()); + assert!(value.get("group").is_none()); + } + + #[test] + fn false_negate_is_present_in_json() { + let f = + Filter::condition("name", FilterFunction::Contains, vec![json!("a")]).negated(false); + let value = serde_json::to_value(&f).unwrap(); + assert_eq!(value.get("negate"), Some(&json!(false))); + } + + #[test] + fn is_negated_treats_missing_and_false_alike() { + let bare = Filter::condition("age", FilterFunction::Equals, vec![json!(1)]); + assert!(!bare.is_negated()); + assert!(!bare.clone().negated(false).is_negated()); + assert!(bare.negated(true).is_negated()); + } + + #[test] + fn group_constructors_set_the_operator() { + assert_eq!(FilterGroup::and(vec![]).op, LogicalOp::And); + assert_eq!(FilterGroup::or(vec![]).op, LogicalOp::Or); + assert!(FilterGroup::and(vec![]).is_empty()); + assert!(!FilterGroup::and(vec![Filter::default()]).is_empty()); + } + + #[test] + fn from_group_wraps_without_negation() { + let inner = FilterGroup::or(vec![Filter::condition( + "age", + FilterFunction::Equals, + vec![json!(1)], + )]); + let f = Filter::from_group(inner.clone()); + assert_eq!(f.group, Some(inner)); + assert!(f.condition.is_none()); + assert!(f.negate.is_none()); + } + + #[test] + fn blank_operators_are_recognised() { + assert!(FilterFunction::IsBlank.is_blank_operator()); + assert!(FilterFunction::IsNotBlank.is_blank_operator()); + assert!(!FilterFunction::Equals.is_blank_operator()); + assert!(!FilterFunction::Contains.is_blank_operator()); + } + + #[test] + fn round_trips_through_json() { + let group = FilterGroup::or(vec![ + Filter::from_group(FilterGroup::and(vec![ + Filter::condition("age", FilterFunction::GreaterThan, vec![json!(1)]), + Filter::condition("name", FilterFunction::Contains, vec![json!("a")]) + .negated(false), + ])), + Filter::condition("active", FilterFunction::Equals, vec![json!(true)]).negated(true), + ]); + let json = serde_json::to_string(&group).unwrap(); + let back: FilterGroup = serde_json::from_str(&json).unwrap(); + assert_eq!(back, group); + } +} diff --git a/rusty-filter/src/column.rs b/rusty-filter/src/column.rs new file mode 100644 index 0000000..a67e2b8 --- /dev/null +++ b/rusty-filter/src/column.rs @@ -0,0 +1,166 @@ +//! Column schema used for validation and evaluation. + +use serde::{Deserialize, Serialize}; + +/// The five column types the grammar's validator knows about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ColumnType { + #[default] + String, + Number, + Boolean, + Date, + Enum, +} + +impl ColumnType { + /// Map a backend type name onto a [`ColumnType`], reproducing + /// `normalizeColumnType` in `dist/validator/TypeChecker.js`. + /// + /// `INT32`, `INT64`, `DOUBLE`, `DECIMAL` and `NUMBER` become + /// [`ColumnType::Number`]; `TEXT`, `STRING` and `ICON` become + /// [`ColumnType::String`]; `BOOLEAN` becomes [`ColumnType::Boolean`]; + /// `DATE` and `DATETIME` become [`ColumnType::Date`]; `ENUM` becomes + /// [`ColumnType::Enum`]. Anything else becomes [`ColumnType::String`]. + /// The comparison is case-insensitive. + pub fn normalize(type_name: &str) -> ColumnType { + let upper = type_name.to_ascii_uppercase(); + match upper.as_str() { + "INT32" | "INT64" | "DOUBLE" | "DECIMAL" | "NUMBER" => ColumnType::Number, + "TEXT" | "STRING" | "ICON" => ColumnType::String, + "BOOLEAN" => ColumnType::Boolean, + "DATE" | "DATETIME" => ColumnType::Date, + "ENUM" => ColumnType::Enum, + _ => ColumnType::String, + } + } + + /// The name used in validation messages, matching the lowercase + /// `ColumnType` union of `dist/types/column.d.ts`. + pub fn as_str(self) -> &'static str { + match self { + ColumnType::String => "string", + ColumnType::Number => "number", + ColumnType::Boolean => "boolean", + ColumnType::Date => "date", + ColumnType::Enum => "enum", + } + } +} + +/// One filterable column. +/// +/// The TypeScript `ColumnDef` also carries a `width`; nothing in parsing, +/// validation or evaluation reads it, so it is deliberately dropped here. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ColumnDef { + pub name: String, + #[serde(rename = "type")] + pub column_type: ColumnType, +} + +impl ColumnDef { + pub fn new(name: impl Into, column_type: ColumnType) -> Self { + ColumnDef { + name: name.into(), + column_type, + } + } + + /// A column whose backend type name needs normalizing first. + pub fn normalized(name: impl Into, type_name: &str) -> Self { + ColumnDef::new(name, ColumnType::normalize(type_name)) + } +} + +/// Look a column up by exact name. Names are matched verbatim — the grammar +/// does not trim what is between the brackets, so `[ s ]` and `[s]` are +/// different columns. +pub(crate) fn find_column<'a>(columns: &'a [ColumnDef], name: &str) -> Option<&'a ColumnDef> { + columns.iter().find(|c| c.name == name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalizes_number_types() { + for name in ["INT32", "INT64", "DOUBLE", "DECIMAL", "NUMBER"] { + assert_eq!(ColumnType::normalize(name), ColumnType::Number, "{name}"); + assert_eq!( + ColumnType::normalize(&name.to_lowercase()), + ColumnType::Number, + "{name} lowercase" + ); + } + } + + #[test] + fn normalizes_string_types() { + for name in ["TEXT", "STRING", "ICON", "text", "Icon"] { + assert_eq!(ColumnType::normalize(name), ColumnType::String, "{name}"); + } + } + + #[test] + fn normalizes_boolean_date_and_enum() { + assert_eq!(ColumnType::normalize("BOOLEAN"), ColumnType::Boolean); + assert_eq!(ColumnType::normalize("boolean"), ColumnType::Boolean); + assert_eq!(ColumnType::normalize("DATE"), ColumnType::Date); + assert_eq!(ColumnType::normalize("DATETIME"), ColumnType::Date); + assert_eq!(ColumnType::normalize("datetime"), ColumnType::Date); + assert_eq!(ColumnType::normalize("ENUM"), ColumnType::Enum); + assert_eq!(ColumnType::normalize("enum"), ColumnType::Enum); + } + + #[test] + fn unknown_types_fall_back_to_string() { + for name in ["GUID", "", "Labels", "Link", "whatever"] { + assert_eq!(ColumnType::normalize(name), ColumnType::String, "{name}"); + } + } + + #[test] + fn default_is_string() { + assert_eq!(ColumnType::default(), ColumnType::String); + } + + #[test] + fn type_names_are_lowercase() { + assert_eq!(ColumnType::String.as_str(), "string"); + assert_eq!(ColumnType::Number.as_str(), "number"); + assert_eq!(ColumnType::Boolean.as_str(), "boolean"); + assert_eq!(ColumnType::Date.as_str(), "date"); + assert_eq!(ColumnType::Enum.as_str(), "enum"); + } + + #[test] + fn normalized_constructor_maps_the_type() { + let col = ColumnDef::normalized("age", "INT64"); + assert_eq!(col.name, "age"); + assert_eq!(col.column_type, ColumnType::Number); + } + + #[test] + fn lookup_is_exact() { + let cols = vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new(" s ", ColumnType::String), + ]; + assert_eq!(find_column(&cols, "age").unwrap().name, "age"); + assert_eq!(find_column(&cols, " s ").unwrap().name, " s "); + assert!(find_column(&cols, "s").is_none()); + assert!(find_column(&cols, "AGE").is_none()); + } + + #[test] + fn serializes_type_as_lowercase_string() { + let col = ColumnDef::new("age", ColumnType::Number); + assert_eq!( + serde_json::to_value(&col).unwrap(), + serde_json::json!({"name": "age", "type": "number"}) + ); + } +} diff --git a/rusty-filter/src/eval.rs b/rusty-filter/src/eval.rs new file mode 100644 index 0000000..5938395 --- /dev/null +++ b/rusty-filter/src/eval.rs @@ -0,0 +1,846 @@ +//! Evaluating a filter against row data. +//! +//! A port of `FilterEvaluator` and `Comparators`. Rows are +//! [`serde_json::Value`] objects, which is what `DataTable` already stores, so +//! nothing needs converting. +//! +//! Three behaviours are worth knowing before reading the code: +//! +//! * **An empty group is `true`, whatever its operator.** The reference returns +//! early on `filters.length === 0` before it looks at `op`, so an empty `OR` +//! group is vacuously true just like an empty `AND` group. Keep it that way: +//! the empty query parses to an empty group and must match every row. +//! * **The column type decides which operators can match.** `contains` on a +//! non-string column is `false`, not a string coercion; the orderings on +//! anything but a number or a date are `false`. Because [`ColumnDef`] stores a +//! normalized [`ColumnType`], a column declared `INT32` compares as a number +//! here, whereas the reference — which switches on the raw type name — +//! silently returns `false`. That divergence is deliberate. +//! * **A non-object row has no columns.** The reference reads `row[column]`, +//! which throws for `null` and is caught into a blanket `false`; here every +//! column of a non-object row reads as missing, so `isBlank` matches it. + +use std::collections::HashMap; + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup, LogicalOp}; +use crate::column::{ColumnDef, ColumnType}; + +/// The recursion limit from `FilterEvaluator`. Exceeding it fails the whole +/// evaluation rather than the offending branch, as the reference's throw and +/// blanket `catch` do. +const MAX_RECURSION_DEPTH: usize = 100; + +/// A name-to-column lookup. On a duplicate name the first definition wins, +/// matching [`crate::column::find_column`] and so keeping validation and +/// evaluation in agreement. +type ColumnMap<'a> = HashMap<&'a str, &'a ColumnDef>; + +fn column_map<'a>(columns: &'a [ColumnDef]) -> ColumnMap<'a> { + let mut map = ColumnMap::with_capacity(columns.len()); + for column in columns { + map.entry(column.name.as_str()).or_insert(column); + } + map +} + +/// Whether `row` matches `filter`. +/// +/// Returns `false` if the filter nests deeper than 100 groups. +pub fn evaluate(filter: &FilterGroup, row: &serde_json::Value, columns: &[ColumnDef]) -> bool { + eval_group(filter, row, &column_map(columns), 0).unwrap_or(false) +} + +/// Keep only the rows matching `filter`, preserving their order. +pub fn retain_matching( + filter: &FilterGroup, + rows: Vec, + columns: &[ColumnDef], +) -> Vec { + let map = column_map(columns); + rows.into_iter() + .filter(|row| eval_group(filter, row, &map, 0).unwrap_or(false)) + .collect() +} + +/// How many of `rows` match `filter`, without building a new collection. +pub fn count_matches( + filter: &FilterGroup, + rows: &[serde_json::Value], + columns: &[ColumnDef], +) -> usize { + let map = column_map(columns); + rows.iter() + .filter(|row| eval_group(filter, row, &map, 0).unwrap_or(false)) + .count() +} + +/// `None` stands for the reference's "maximum recursion depth exceeded" throw, +/// which its caller turns into `false` for the entire filter. +fn eval_group( + group: &FilterGroup, + row: &serde_json::Value, + columns: &ColumnMap<'_>, + depth: usize, +) -> Option { + if depth > MAX_RECURSION_DEPTH { + return None; + } + // Vacuous truth, checked before `op` is examined. + if group.filters.is_empty() { + return Some(true); + } + match group.op { + LogicalOp::And => { + for filter in &group.filters { + if !eval_filter(filter, row, columns, depth + 1)? { + return Some(false); + } + } + Some(true) + } + LogicalOp::Or => { + for filter in &group.filters { + if eval_filter(filter, row, columns, depth + 1)? { + return Some(true); + } + } + Some(false) + } + } +} + +fn eval_filter( + filter: &Filter, + row: &serde_json::Value, + columns: &ColumnMap<'_>, + depth: usize, +) -> Option { + // A condition wins over a group when both are set, and a filter with + // neither never matches — even negated, since the reference returns before + // it applies negation. + let result = if let Some(condition) = &filter.condition { + eval_condition(condition, row, columns) + } else if let Some(group) = &filter.group { + // The nested group keeps this depth rather than incrementing again. + eval_group(group, row, columns, depth)? + } else { + return Some(false); + }; + Some(result != filter.is_negated()) +} + +fn eval_condition(condition: &Condition, row: &serde_json::Value, columns: &ColumnMap<'_>) -> bool { + let Some(column) = columns.get(condition.column.as_str()) else { + // A condition on an unknown column never matches. + return false; + }; + let value = row.get(&condition.column); + apply( + condition.function, + value, + &condition.args, + column.column_type, + ) +} + +/// Whether `value` — `None` when the row has no such key — satisfies +/// `function` against `args` for a column of `column_type`. +pub fn apply( + function: FilterFunction, + value: Option<&serde_json::Value>, + args: &[serde_json::Value], + column_type: ColumnType, +) -> bool { + // The blank operators ignore their arguments and are the only ones a + // missing value can satisfy. + match function { + FilterFunction::IsBlank => return is_blank(value, column_type), + FilterFunction::IsNotBlank => return !is_blank(value, column_type), + _ => {} + } + + let (Some(value), Some(arg)) = (non_null(value), non_null(args.first())) else { + return false; + }; + + match function { + FilterFunction::Equals => strict_equals(value, arg), + FilterFunction::GreaterThan + | FilterFunction::LessThan + | FilterFunction::GreaterThanOrEqual + | FilterFunction::LessThanOrEqual => compare(function, value, arg, column_type), + FilterFunction::Contains | FilterFunction::StartsWith | FilterFunction::EndsWith => { + if column_type != ColumnType::String { + return false; + } + let (Some(haystack), Some(needle)) = (value.as_str(), arg.as_str()) else { + return false; + }; + // Case-sensitive on purpose: the reference does not fold case. + match function { + FilterFunction::Contains => haystack.contains(needle), + FilterFunction::StartsWith => haystack.starts_with(needle), + _ => haystack.ends_with(needle), + } + } + FilterFunction::IsBlank | FilterFunction::IsNotBlank => unreachable!("handled above"), + } +} + +/// A missing key and a JSON `null` are both the reference's "nullish". +fn non_null(value: Option<&serde_json::Value>) -> Option<&serde_json::Value> { + match value { + Some(serde_json::Value::Null) | None => None, + other => other, + } +} + +fn is_blank(value: Option<&serde_json::Value>, column_type: ColumnType) -> bool { + match non_null(value) { + None => true, + // The empty string counts as blank for string columns only. + Some(value) => column_type == ColumnType::String && value.as_str() == Some(""), + } +} + +/// JavaScript `===` on two JSON values. +/// +/// Strict, so no coercion: the number `1` never equals the string `"1"`. Numbers +/// are compared by value rather than by JSON representation, because `1` and +/// `1.0` are the same number in JavaScript and a row loaded from a database may +/// carry either. +fn strict_equals(a: &serde_json::Value, b: &serde_json::Value) -> bool { + match (a, b) { + (serde_json::Value::Number(x), serde_json::Value::Number(y)) => x.as_f64() == y.as_f64(), + // Arrays and objects compare by identity in JavaScript, so two distinct + // ones are never equal however alike they look. + (serde_json::Value::Array(_), _) + | (_, serde_json::Value::Array(_)) + | (serde_json::Value::Object(_), _) + | (_, serde_json::Value::Object(_)) => false, + _ => a == b, + } +} + +fn compare( + function: FilterFunction, + value: &serde_json::Value, + arg: &serde_json::Value, + column_type: ColumnType, +) -> bool { + let ordering = match column_type { + ColumnType::Number => { + let (Some(a), Some(b)) = (value.as_f64(), arg.as_f64()) else { + return false; + }; + a.partial_cmp(&b) + } + ColumnType::Date => { + let (Some(a), Some(b)) = (value.as_str(), arg.as_str()) else { + return false; + }; + let (Some(a), Some(b)) = (parse_iso_millis(a), parse_iso_millis(b)) else { + // An unparseable date is the reference's `NaN`, which fails + // every comparison. + return false; + }; + Some(a.cmp(&b)) + } + // The orderings are not supported for other types. + ColumnType::String | ColumnType::Boolean | ColumnType::Enum => return false, + }; + let Some(ordering) = ordering else { + return false; + }; + match function { + FilterFunction::GreaterThan => ordering.is_gt(), + FilterFunction::LessThan => ordering.is_lt(), + FilterFunction::GreaterThanOrEqual => ordering.is_ge(), + FilterFunction::LessThanOrEqual => ordering.is_le(), + _ => false, + } +} + +/// Parse an ISO 8601 date or datetime into milliseconds since the Unix epoch, +/// returning `None` where `Date.parse` would return `NaN`. +/// +/// This follows ECMAScript's Date Time String Format, which is what the +/// reference's `new Date(...)` reaches for: the field ranges are checked +/// (month 1–12, day 1–31, hour 0–24, minute and second 0–59, and hour 24 only at +/// exactly `24:00:00`), then the arithmetic is allowed to roll over, so +/// `2024-02-30` is 1 March and `2024-01-01T24:00:00` is 2 January — while +/// `2024-13-01` and `2024-01-32` are unparseable. +/// +/// **One deliberate divergence.** A datetime with no trailing `Z` is read as +/// UTC, whereas the browser reads it in the machine's local zone. Following the +/// browser would make server-side filtering depend on the server's timezone and +/// disagree with the same filter run in the client, so a date-only value and a +/// naked datetime compare equal here — `2024-01-01T00:00:00 > 2024-01-01` is +/// false in both implementations, but for different reasons. +fn parse_iso_millis(text: &str) -> Option { + let bytes = text.as_bytes(); + let digits = |start: usize, len: usize| -> Option { + let slice = bytes.get(start..start + len)?; + if !slice.iter().all(u8::is_ascii_digit) { + return None; + } + std::str::from_utf8(slice).ok()?.parse().ok() + }; + + let year = digits(0, 4)?; + if bytes.get(4) != Some(&b'-') { + return None; + } + let month = digits(5, 2)?; + if bytes.get(7) != Some(&b'-') { + return None; + } + let day = digits(8, 2)?; + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + return None; + } + + let mut hour = 0; + let mut minute = 0; + let mut second = 0; + let mut milli = 0; + let mut end = 10; + if bytes.len() > 10 { + if bytes[10] != b'T' { + return None; + } + hour = digits(11, 2)?; + if bytes.get(13) != Some(&b':') { + return None; + } + minute = digits(14, 2)?; + if bytes.get(16) != Some(&b':') { + return None; + } + second = digits(17, 2)?; + end = 19; + if bytes.get(end) == Some(&b'.') { + milli = digits(end + 1, 3)?; + end += 4; + } + if !(0..=24).contains(&hour) || minute > 59 || second > 59 { + return None; + } + // Hour 24 names midnight ending the day, so nothing may follow it. + if hour == 24 && (minute, second, milli) != (0, 0, 0) { + return None; + } + if bytes.get(end) == Some(&b'Z') { + end += 1; + } + } + if end != bytes.len() { + return None; + } + + let days = days_from_civil(year, month, day); + Some(((days * 24 + hour) * 60 + minute) * 60_000 + second * 1_000 + milli) +} + +/// Days from 1970-01-01 to `year-month-day` in the proleptic Gregorian +/// calendar. A day past the end of its month rolls into the next one, as +/// ECMAScript's `MakeDay` does. +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + // Shift so that the leap day falls at the end of the 400-year cycle. + let y = if month <= 2 { year - 1 } else { year }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let year_of_era = y - era * 400; + let month_shifted = if month > 2 { month - 3 } else { month + 9 }; + let day_of_year = (153 * month_shifted + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse_query_unchecked; + use serde_json::json; + + fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("created", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ] + } + + fn matches(query: &str, row: serde_json::Value) -> bool { + let group = parse_query_unchecked(query).expect(query); + evaluate(&group, &row, &columns()) + } + + #[test] + fn the_probed_rows_behave_as_measured() { + let query = "[age] > 1 AND [name] contains \"a\""; + assert!(matches(query, json!({"age": 2, "name": "abc"}))); + assert!(!matches(query, json!({"age": 0, "name": "abc"}))); + assert!(!matches(query, json!({"age": 2, "name": "xyz"}))); + assert!(!matches(query, json!({"age": null, "name": "abc"}))); + } + + #[test] + fn or_needs_only_one_arm() { + let query = "[age] > 10 OR [name] = \"a\""; + assert!(matches(query, json!({"age": 11, "name": "z"}))); + assert!(matches(query, json!({"age": 0, "name": "a"}))); + assert!(!matches(query, json!({"age": 0, "name": "z"}))); + } + + #[test] + fn an_empty_group_is_true_whatever_its_operator() { + let row = json!({"age": 1}); + // This is the divergence from the plan's table: an empty OR group is + // true in the bundle, not false. + assert!(evaluate(&FilterGroup::and(vec![]), &row, &columns())); + assert!(evaluate(&FilterGroup::or(vec![]), &row, &columns())); + // And the empty query parses to exactly that group. + assert!(matches("", row.clone())); + assert!(matches(" ", row)); + } + + #[test] + fn an_empty_group_matches_even_a_non_object_row() { + assert!(evaluate(&FilterGroup::default(), &json!(null), &columns())); + assert!(evaluate(&FilterGroup::default(), &json!(7), &columns())); + } + + #[test] + fn negation_inverts_the_filter() { + assert!(matches("NOT [age] > 1", json!({"age": 0}))); + assert!(!matches("NOT [age] > 1", json!({"age": 2}))); + assert!(matches("[name] not contains \"a\"", json!({"name": "xyz"}))); + assert!(!matches( + "[name] not contains \"a\"", + json!({"name": "abc"}) + )); + // A false `negate` key must not invert anything. + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![json!(1)], + ) + .negated(false)]); + assert!(evaluate(&group, &json!({"age": 2}), &columns())); + } + + #[test] + fn negation_applies_to_groups_too() { + assert!(matches( + "NOT ([age] > 1 AND [name] = \"a\")", + json!({"age": 2, "name": "b"}) + )); + assert!(!matches( + "NOT ([age] > 1 AND [name] = \"a\")", + json!({"age": 2, "name": "a"}) + )); + } + + #[test] + fn a_null_value_fails_everything_but_is_blank() { + for query in [ + "[name] = \"a\"", + "[name] contains \"a\"", + "[name] starts with \"a\"", + "[name] ends with \"a\"", + "[name] is not blank", + ] { + assert!(!matches(query, json!({"name": null})), "{query}"); + assert!(!matches(query, json!({})), "{query} missing"); + } + assert!(matches("[name] is blank", json!({"name": null}))); + assert!(matches("[name] is blank", json!({}))); + } + + #[test] + fn a_missing_column_definition_never_matches() { + let group = parse_query_unchecked("[nope] is blank").unwrap(); + assert!(!evaluate(&group, &json!({"nope": null}), &columns())); + } + + #[test] + fn is_blank_covers_the_empty_string_for_strings_only() { + assert!(matches("[name] is blank", json!({"name": ""}))); + assert!(!matches("[name] is blank", json!({"name": "a"}))); + assert!(!matches("[name] is not blank", json!({"name": ""}))); + // An empty string in a date or enum column is not blank. + assert!(!matches("[created] is blank", json!({"created": ""}))); + assert!(!matches("[kind] is blank", json!({"kind": ""}))); + assert!(matches("[kind] is not blank", json!({"kind": ""}))); + } + + #[test] + fn equals_is_strict() { + assert!(!matches("[name] = \"1\"", json!({"name": 1}))); + assert!(matches("[name] = \"1\"", json!({"name": "1"}))); + assert!(!matches("[age] = 1", json!({"age": "1"}))); + assert!(!matches("[age] = 1", json!({"age": true}))); + assert!(!matches("[active] = true", json!({"active": 1}))); + assert!(matches("[active] = true", json!({"active": true}))); + assert!(matches("[active] = false", json!({"active": false}))); + assert!(!matches("[active] = false", json!({"active": true}))); + } + + #[test] + fn equals_compares_numbers_by_value_not_representation() { + // A row loaded from a database may hold an integer where the query + // holds a float, and JavaScript would call those equal. + assert!(matches("[age] = 1", json!({"age": 1}))); + assert!(matches("[age] = 1", json!({"age": 1.0}))); + assert!(matches("[age] = 1.0", json!({"age": 1}))); + assert!(matches("[age] = 1.5", json!({"age": 1.5}))); + assert!(!matches("[age] = 1.5", json!({"age": 1.6}))); + } + + #[test] + fn equals_never_matches_a_composite_value() { + let group = FilterGroup::and(vec![Filter::condition( + "name", + FilterFunction::Equals, + vec![json!([1])], + )]); + assert!(!evaluate(&group, &json!({"name": [1]}), &columns())); + let group = FilterGroup::and(vec![Filter::condition( + "name", + FilterFunction::Equals, + vec![json!({"a": 1})], + )]); + assert!(!evaluate(&group, &json!({"name": {"a": 1}}), &columns())); + } + + #[test] + fn number_orderings() { + assert!(matches("[age] > 1", json!({"age": 1.5}))); + assert!(!matches("[age] > 1", json!({"age": 1}))); + assert!(matches("[age] >= 1", json!({"age": 1}))); + assert!(matches("[age] < 1", json!({"age": 0.5}))); + assert!(!matches("[age] < 1", json!({"age": 1}))); + assert!(matches("[age] <= 1", json!({"age": 1}))); + assert!(matches("[age] > -1", json!({"age": 0}))); + // A non-numeric value in a numeric column cannot be ordered. + assert!(!matches("[age] > 1", json!({"age": "5"}))); + assert!(!matches("[age] > 1", json!({"age": true}))); + } + + #[test] + fn orderings_are_unsupported_for_string_boolean_and_enum() { + // The grammar and validator both reject these, but a hand-built AST + // can carry them, and then they must not match. + for column in ["name", "active", "kind"] { + let group = FilterGroup::and(vec![Filter::condition( + column, + FilterFunction::GreaterThan, + vec![json!("a")], + )]); + let row = json!({"name": "b", "active": true, "kind": "b"}); + assert!(!evaluate(&group, &row, &columns()), "{column}"); + } + } + + #[test] + fn contains_and_friends_are_case_sensitive() { + assert!(matches("[name] contains \"bc\"", json!({"name": "abcd"}))); + assert!(!matches("[name] contains \"BC\"", json!({"name": "abcd"}))); + assert!(matches( + "[name] starts with \"ab\"", + json!({"name": "abcd"}) + )); + assert!(!matches( + "[name] starts with \"AB\"", + json!({"name": "abcd"}) + )); + assert!(matches("[name] ends with \"cd\"", json!({"name": "abcd"}))); + assert!(!matches("[name] ends with \"CD\"", json!({"name": "abcd"}))); + // The empty needle is contained in everything. + assert!(matches("[name] contains \"\"", json!({"name": "abcd"}))); + } + + #[test] + fn text_operations_need_a_string_column() { + // `[kind]` is an enum, so `contains` cannot match however alike the + // values look. + let group = FilterGroup::and(vec![Filter::condition( + "kind", + FilterFunction::Contains, + vec![json!("b")], + )]); + assert!(!evaluate(&group, &json!({"kind": "abc"}), &columns())); + } + + #[test] + fn date_orderings() { + assert!(matches( + "[created] > \"2024-01-01\"", + json!({"created": "2024-06-01"}) + )); + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": "2023-06-01"}) + )); + assert!(matches( + "[created] >= \"2024-01-01\"", + json!({"created": "2024-01-01"}) + )); + assert!(matches( + "[created] < \"2024-01-01T12:00:00Z\"", + json!({"created": "2024-01-01T11:59:59Z"}) + )); + // Different shapes still compare on the instant they name. + assert!(matches( + "[created] <= \"2024-01-01\"", + json!({"created": "2024-01-01T00:00:00Z"}) + )); + assert!(matches( + "[created] >= \"2024-01-01\"", + json!({"created": "2024-01-01T00:00:00Z"}) + )); + } + + #[test] + fn a_naked_datetime_does_not_exceed_the_bare_date() { + // The measured browser answer, reached here without depending on the + // machine's timezone. + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": "2024-01-01T00:00:00"}) + )); + } + + #[test] + fn an_unparseable_date_fails_every_ordering() { + for query in [ + "[created] > \"2024-13-99\"", + "[created] < \"2024-13-99\"", + "[created] >= \"2024-13-99\"", + "[created] <= \"2024-13-99\"", + ] { + assert!(!matches(query, json!({"created": "2024-01-01"})), "{query}"); + } + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": "not a date"}) + )); + // A non-string value in a date column cannot be ordered either. + assert!(!matches( + "[created] > \"2024-01-01\"", + json!({"created": 1}) + )); + } + + #[test] + fn iso_parsing_matches_the_javascript_engine() { + // Values measured with `Date.parse` in Node on 2026-08-01. + assert_eq!(parse_iso_millis("2024-01-01"), Some(1_704_067_200_000)); + assert_eq!( + parse_iso_millis("2024-01-01T00:00:00Z"), + Some(1_704_067_200_000) + ); + assert_eq!( + parse_iso_millis("2024-12-31T23:59:59.999Z"), + Some(1_735_689_599_999) + ); + assert_eq!(parse_iso_millis("0000-01-01"), Some(-62_167_219_200_000)); + assert_eq!(parse_iso_millis("9999-12-31"), Some(253_402_214_400_000)); + assert_eq!(parse_iso_millis("2024-02-29"), Some(1_709_164_800_000)); + // Out-of-range fields are unparseable, exactly as in the engine. + for text in [ + "2024-13-01", + "2024-00-01", + "2024-01-32", + "2024-01-00", + "2024-01-01T25:00:00", + "2024-01-01T23:60:00", + "2024-01-01T23:59:60", + "2024-01-01T24:00:01", + "2024-01-01T24:59:59", + "2024-1-1", + "", + "x", + "2024-01-01 00:00:00", + "2024-01-01T00:00", + "2024-01-01T00:00:00.12", + "2024-01-01T00:00:00ZZ", + ] { + assert_eq!(parse_iso_millis(text), None, "{text}"); + } + } + + #[test] + fn iso_parsing_rolls_over_like_make_day_and_make_time() { + // Both measured in Node: an over-long month rolls into the next one, + // and hour 24 names the following midnight. + assert_eq!( + parse_iso_millis("2024-02-30"), + parse_iso_millis("2024-03-01") + ); + assert_eq!( + parse_iso_millis("2023-02-29"), + parse_iso_millis("2023-03-01") + ); + assert_eq!( + parse_iso_millis("2024-04-31"), + parse_iso_millis("2024-05-01") + ); + assert_eq!( + parse_iso_millis("2024-01-01T24:00:00Z"), + parse_iso_millis("2024-01-02") + ); + } + + #[test] + fn days_from_civil_anchors_on_the_epoch() { + assert_eq!(days_from_civil(1970, 1, 1), 0); + assert_eq!(days_from_civil(1970, 1, 2), 1); + assert_eq!(days_from_civil(1969, 12, 31), -1); + assert_eq!(days_from_civil(2000, 3, 1), 11_017); + assert_eq!(days_from_civil(1900, 3, 1), -25_508); + } + + #[test] + fn normalized_types_widen_what_can_match() { + // The reference switches on the raw type name, so an `INT32` column + // fails every ordering there. Here the type is normalized first. + let cols = vec![ + ColumnDef::normalized("n", "INT32"), + ColumnDef::normalized("t", "TEXT"), + ]; + let group = parse_query_unchecked("[n] > 1 AND [t] contains \"a\"").unwrap(); + assert!(evaluate(&group, &json!({"n": 2, "t": "abc"}), &cols)); + assert!(!evaluate(&group, &json!({"n": 1, "t": "abc"}), &cols)); + } + + #[test] + fn a_non_object_row_has_no_columns() { + let group = parse_query_unchecked("[name] is blank").unwrap(); + assert!(evaluate(&group, &json!(null), &columns())); + assert!(evaluate(&group, &json!([1, 2]), &columns())); + let group = parse_query_unchecked("[name] = \"a\"").unwrap(); + assert!(!evaluate(&group, &json!(null), &columns())); + } + + #[test] + fn a_filter_with_neither_side_never_matches() { + let group = FilterGroup::and(vec![Filter::default()]); + assert!(!evaluate(&group, &json!({}), &columns())); + // Negation does not rescue it: the reference returns before negating. + let group = FilterGroup::and(vec![Filter::default().negated(true)]); + assert!(!evaluate(&group, &json!({}), &columns())); + } + + #[test] + fn a_condition_wins_over_a_group_on_the_same_filter() { + let filter = Filter { + condition: Some(crate::ast::Condition::new( + "age", + FilterFunction::GreaterThan, + vec![json!(1)], + )), + group: Some(FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::LessThan, + vec![json!(0)], + )])), + negate: None, + }; + let group = FilterGroup::and(vec![filter]); + assert!(evaluate(&group, &json!({"age": 2}), &columns())); + } + + #[test] + fn nesting_past_the_depth_limit_fails_the_whole_filter() { + let deepest = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![json!(1)], + )]); + let nest = |depth: usize| { + let mut group = deepest.clone(); + for _ in 0..depth { + group = FilterGroup::and(vec![Filter::from_group(group)]); + } + group + }; + let row = json!({"age": 2}); + assert!(evaluate(&nest(99), &row, &columns())); + assert!(evaluate(&nest(100), &row, &columns())); + assert!(!evaluate(&nest(101), &row, &columns())); + } + + #[test] + fn retain_matching_keeps_order() { + let group = parse_query_unchecked("[age] > 1").unwrap(); + let rows = vec![ + json!({"age": 3, "name": "c"}), + json!({"age": 1, "name": "a"}), + json!({"age": 2, "name": "b"}), + ]; + let kept = retain_matching(&group, rows.clone(), &columns()); + assert_eq!(kept, vec![rows[0].clone(), rows[2].clone()]); + assert_eq!(count_matches(&group, &rows, &columns()), 2); + } + + #[test] + fn retain_matching_on_an_empty_filter_keeps_everything() { + let rows = vec![json!({"age": 1}), json!({"age": 2})]; + let kept = retain_matching(&FilterGroup::default(), rows.clone(), &columns()); + assert_eq!(kept, rows); + assert_eq!(count_matches(&FilterGroup::default(), &rows, &columns()), 2); + } + + #[test] + fn duplicate_column_names_resolve_to_the_first() { + let cols = vec![ + ColumnDef::new("x", ColumnType::Number), + ColumnDef::new("x", ColumnType::String), + ]; + let group = FilterGroup::and(vec![Filter::condition( + "x", + FilterFunction::GreaterThan, + vec![json!(1)], + )]); + assert!(evaluate(&group, &json!({"x": 2}), &cols)); + } + + #[test] + fn apply_is_usable_on_its_own() { + assert!(apply( + FilterFunction::Contains, + Some(&json!("abc")), + &[json!("b")], + ColumnType::String + )); + assert!(!apply( + FilterFunction::Contains, + None, + &[json!("b")], + ColumnType::String + )); + // No argument means nothing to compare against. + assert!(!apply( + FilterFunction::Equals, + Some(&json!("a")), + &[], + ColumnType::String + )); + assert!(apply( + FilterFunction::IsBlank, + None, + &[], + ColumnType::String + )); + // Only the first argument is read. + assert!(apply( + FilterFunction::Equals, + Some(&json!("a")), + &[json!("a"), json!("b")], + ColumnType::String + )); + } +} diff --git a/rusty-filter/src/lexer.rs b/rusty-filter/src/lexer.rs new file mode 100644 index 0000000..60b0dda --- /dev/null +++ b/rusty-filter/src/lexer.rs @@ -0,0 +1,792 @@ +//! Hand-written lexer for the `Filters.g4` token set. +//! +//! One [`Token`] variant per lexer rule of the grammar. Every rule below was +//! checked against the shipped `filter-query-editor` 2.2.0 bundle, both through +//! `parseQuery` and by driving the generated ANTLR lexer directly; see the crate +//! docs for the probe method. +//! +//! Whitespace (space, tab, CR, LF) is skipped between tokens, which is why +//! `1 . 5` lexes as three tokens and parses as `1.5`. +//! +//! # How lexer errors are shaped +//! +//! ANTLR's lexer walks its DFA as far as the input stays a *viable prefix* of +//! some rule, and its error message quotes everything it walked over plus the +//! character that broke viability. That is why `1e5` complains about `e5` rather +//! than `e`, and why `[age] ! 5` complains about `'! '` — bang and space — since +//! `!` is a viable prefix of `!=` and the space is what rules it out. +//! [`viable_len`] is that walk, and it is the reason the error text is built the +//! way it is instead of just naming the offending character. + +/// A lexed token together with its byte span in the input. +#[derive(Debug, Clone, PartialEq)] +pub struct SpannedToken { + pub token: Token, + /// The raw lexeme, exactly as it appeared in the input: brackets included + /// for a field, quotes included for a string, original case for a keyword. + /// Empty for [`Token::Eof`]. Error messages quote this rather than a + /// canonical spelling, so `[age] GREATER 5` complains about `GREATER`. + pub text: String, + /// Byte offset of the first character of the token. + pub start: usize, + /// Byte offset one past the last character of the token. + pub end: usize, +} + +impl SpannedToken { + /// The spelling an error message uses for this token. + pub fn describe(&self) -> &str { + if self.token == Token::Eof { + "" + } else { + &self.text + } + } +} + +/// The terminals of the grammar. +/// +/// [`Token::Field`] carries the text *between* the brackets, verbatim: no +/// trimming and no unescaping. [`Token::Str`] carries the *unescaped* body of +/// the string literal. Both keep their raw lexeme in [`SpannedToken::text`]. +#[derive(Debug, Clone, PartialEq)] +pub enum Token { + Field(String), + Str(String), + Contains, + Greater, + Starts, + Equals, + Equal, + Blank, + Less, + Than, + Ends, + With, + Not, + Or, + Is, + And, + True, + False, + LParen, + RParen, + Eq, + Eq2, + Neq, + Gt, + Ge, + Lt, + Le, + Dot, + Sign(char), + Digits(String), + Eof, +} + +impl Token { + /// The grammar's name for this token, used where a message names an + /// *expected* token rather than quoting the offending one — as in + /// `missing BLANK at ''`. + pub fn name(&self) -> &'static str { + match self { + Token::Field(_) => "FIELD", + Token::Str(_) => "STRING", + Token::Contains => "CONTAINS", + Token::Greater => "GREATER", + Token::Starts => "STARTS", + Token::Equals => "EQUALS", + Token::Equal => "EQUAL", + Token::Blank => "BLANK", + Token::Less => "LESS", + Token::Than => "THAN", + Token::Ends => "ENDS", + Token::With => "WITH", + Token::Not => "NOT", + Token::Or => "OR", + Token::Is => "IS", + Token::And => "AND", + Token::True => "TRUE", + Token::False => "FALSE", + Token::LParen => "'('", + Token::RParen => "')'", + Token::Eq => "EQUAL_SIGN", + Token::Eq2 => "EQUAL_SIGN2", + Token::Neq => "NOT_EQUAL_SIGN", + Token::Gt => "GT", + Token::Ge => "GTE", + Token::Lt => "LT", + Token::Le => "LTE", + Token::Dot => "DOT", + Token::Sign(_) => "SIGN", + Token::Digits(_) => "DIGITS", + Token::Eof => "", + } + } +} + +/// A lexing failure, with the byte span of the offending text. +#[derive(Debug, Clone, PartialEq)] +pub struct LexError { + pub message: String, + pub start: usize, + pub end: usize, +} + +/// The keyword table. Matching is ASCII-case-insensitive, and the table is also +/// what [`viable_len`] consults to decide whether a partial word could still +/// grow into a keyword. +const KEYWORDS: &[(&str, Token)] = &[ + ("contains", Token::Contains), + ("greater", Token::Greater), + ("starts", Token::Starts), + ("equals", Token::Equals), + ("equal", Token::Equal), + ("blank", Token::Blank), + ("false", Token::False), + ("less", Token::Less), + ("than", Token::Than), + ("ends", Token::Ends), + ("with", Token::With), + ("true", Token::True), + ("not", Token::Not), + ("and", Token::And), + ("or", Token::Or), + ("is", Token::Is), +]; + +/// The multi-character symbolic tokens, longest first. +const SYMBOLS2: &[(&str, Token)] = &[ + ("==", Token::Eq2), + ("!=", Token::Neq), + (">=", Token::Ge), + ("<=", Token::Le), +]; + +/// Lex `input` into tokens, always terminated by [`Token::Eof`]. +/// +/// Returns the first error encountered rather than recovering; see the crate +/// docs on the single-error divergence from the ANTLR reference. +pub fn tokenize(input: &str) -> Result, LexError> { + let bytes = input.as_bytes(); + let mut tokens = Vec::new(); + let mut i = 0usize; + + while i < bytes.len() { + let c = bytes[i]; + // WS: space, tab, CR, LF — skipped, not emitted. + if matches!(c, b' ' | b'\t' | b'\r' | b'\n') { + i += 1; + continue; + } + match longest_match(input, i) { + Some((token, len)) => { + tokens.push(SpannedToken { + token, + text: input[i..i + len].to_string(), + start: i, + end: i + len, + }); + i += len; + } + None => return Err(recognition_error(input, i)), + } + } + + tokens.push(SpannedToken { + token: Token::Eof, + text: String::new(), + start: input.len(), + end: input.len(), + }); + Ok(tokens) +} + +/// The longest complete token starting at byte offset `at`, with its length. +/// +/// Maximal munch: `==` beats `=`, `equals` beats `equal`, and a digit run is +/// taken whole. A word that is only a *prefix* of a keyword matches nothing, +/// which is what makes `end` an error while `ends` is a token. +fn longest_match(input: &str, at: usize) -> Option<(Token, usize)> { + let bytes = input.as_bytes(); + let rest = &input[at..]; + let c = bytes[at]; + + // FIELD: '[' ~[\r\n\]]+ ']' + if c == b'[' { + let mut j = at + 1; + while j < bytes.len() { + match bytes[j] { + b'\r' | b'\n' => return None, + // The `+` needs at least one inner character, so `[]` is not a + // field. + b']' if j > at + 1 => { + return Some((Token::Field(input[at + 1..j].to_string()), j + 1 - at)); + } + b']' => return None, + _ => j += 1, + } + } + return None; + } + + // STRING: '"' ( '\\' . | ~[\\"\r\n] )* '"' + if c == b'"' { + let mut j = at + 1; + let mut body = String::new(); + while j < bytes.len() { + match bytes[j] { + // An escape consumes the next character whatever it is, so a + // trailing backslash swallows the closing quote. + b'\\' => match char_at(input, j + 1) { + Some((ch, width)) => { + body.push('\\'); + body.push(ch); + j += 1 + width; + } + None => return None, + }, + b'"' => return Some((Token::Str(unescape(&body)), j + 1 - at)), + b'\r' | b'\n' => return None, + _ => { + let (ch, width) = char_at(input, j).expect("index inside input"); + body.push(ch); + j += width; + } + } + } + return None; + } + + // Two-character operators before their one-character prefixes. + for (text, token) in SYMBOLS2 { + if rest.starts_with(text) { + return Some((token.clone(), text.len())); + } + } + let single = match c { + b'(' => Some(Token::LParen), + b')' => Some(Token::RParen), + b'=' => Some(Token::Eq), + b'>' => Some(Token::Gt), + b'<' => Some(Token::Lt), + b'.' => Some(Token::Dot), + b'+' => Some(Token::Sign('+')), + b'-' => Some(Token::Sign('-')), + _ => None, + }; + if let Some(token) = single { + return Some((token, 1)); + } + + // DIGITS: [0-9]+ + if c.is_ascii_digit() { + let len = bytes[at..] + .iter() + .take_while(|b| b.is_ascii_digit()) + .count(); + return Some((Token::Digits(input[at..at + len].to_string()), len)); + } + + // Keywords, longest complete spelling wins. + let mut best: Option<(Token, usize)> = None; + for (word, token) in KEYWORDS { + if rest.len() >= word.len() && rest[..word.len()].eq_ignore_ascii_case(word) { + let better = best.as_ref().is_none_or(|(_, len)| word.len() > *len); + if better { + best = Some((token.clone(), word.len())); + } + } + } + best +} + +/// How many bytes from `at` stay a viable prefix of some lexer rule. +/// +/// Zero for a character that starts no rule at all. This is the DFA walk ANTLR +/// performs before it gives up, and the length its error message is built from. +fn viable_len(input: &str, at: usize) -> usize { + let bytes = input.as_bytes(); + let rest = &input[at..]; + let c = bytes[at]; + + // A field or string stays viable until a forbidden character appears; the + // whole remainder is viable if it simply runs out unterminated. + if c == b'[' { + let mut j = at + 1; + while j < bytes.len() && !matches!(bytes[j], b'\r' | b'\n' | b']') { + j += 1; + } + return j - at; + } + if c == b'"' { + let mut j = at + 1; + while j < bytes.len() { + match bytes[j] { + b'\\' => match char_at(input, j + 1) { + Some((_, width)) => j += 1 + width, + None => { + j += 1; + break; + } + }, + b'\r' | b'\n' => break, + _ => j += char_at(input, j).expect("index inside input").1, + } + } + return j - at; + } + + // `!` alone is viable only because `!=` exists. + if c == b'!' { + return 1; + } + // Every other symbolic token and a digit run are complete as soon as they + // start, so nothing beyond `longest_match` is viable. + if let Some((_, len)) = longest_match(input, at) { + if !c.is_ascii_alphabetic() { + return len; + } + } + + // A word is viable while it is a case-insensitive prefix of some keyword. + let mut best = 0; + for (word, _) in KEYWORDS { + let mut shared = 0; + while shared < word.len() + && shared < rest.len() + && rest.as_bytes()[shared].eq_ignore_ascii_case(&word.as_bytes()[shared]) + { + shared += 1; + } + best = best.max(shared); + } + best +} + +/// Build the `token recognition error` the reference reports at `at`. +/// +/// The quoted text runs from `at` over every viable byte plus the character that +/// broke viability, or to the end of the input if viability ran out there. The +/// *span*, by contrast, covers only the first character: the reference's error +/// listener gets no token for a lexer error and falls back to a one-unit span at +/// the reported position. +fn recognition_error(input: &str, at: usize) -> LexError { + let viable = viable_len(input, at); + let end = match char_at(input, at + viable) { + Some((_, width)) => at + viable + width, + None => input.len(), + }; + let (_, first_width) = char_at(input, at).expect("index inside input"); + LexError { + message: format!("token recognition error at: '{}'", &input[at..end]), + start: at, + end: at + first_width, + } +} + +/// The character starting at byte offset `at`, with its UTF-8 width. +fn char_at(input: &str, at: usize) -> Option<(char, usize)> { + input + .get(at..) + .and_then(|s| s.chars().next()) + .map(|c| (c, c.len_utf8())) +} + +/// Unescape a string literal body exactly as `extractStringValue` does. +/// +/// The reference applies three ordered replacements to the quoted body: +/// `\"` then `\'` then `\\`. Only those three escapes are recognised; every +/// other escape **keeps its backslash**, so `\t` stays as backslash-t and +/// `\z` stays as backslash-z. Because the replacements run left to right over +/// the whole string rather than as a single scan, a `\\` pair is only collapsed +/// after the quote escapes have been rewritten — which is why `\\'` yields +/// backslash-quote rather than backslash-backslash-quote. +fn unescape(body: &str) -> String { + let stage1 = body.replace("\\\"", "\""); + let stage2 = stage1.replace("\\'", "'"); + stage2.replace("\\\\", "\\") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn toks(input: &str) -> Vec { + tokenize(input) + .expect("expected the input to lex") + .into_iter() + .map(|t| t.token) + .collect() + } + + fn err(input: &str) -> LexError { + tokenize(input).expect_err("expected the input to fail lexing") + } + + #[test] + fn lexes_every_token_variant() { + let tokens = toks( + "[f] \"s\" contains greater starts equals equal blank less than ends with \ + not or is and true false ( ) = == != > >= < <= . + - 12", + ); + assert_eq!( + tokens, + vec![ + Token::Field("f".to_string()), + Token::Str("s".to_string()), + Token::Contains, + Token::Greater, + Token::Starts, + Token::Equals, + Token::Equal, + Token::Blank, + Token::Less, + Token::Than, + Token::Ends, + Token::With, + Token::Not, + Token::Or, + Token::Is, + Token::And, + Token::True, + Token::False, + Token::LParen, + Token::RParen, + Token::Eq, + Token::Eq2, + Token::Neq, + Token::Gt, + Token::Ge, + Token::Lt, + Token::Le, + Token::Dot, + Token::Sign('+'), + Token::Sign('-'), + Token::Digits("12".to_string()), + Token::Eof, + ] + ); + } + + #[test] + fn field_keeps_inner_text_verbatim() { + assert_eq!(toks("[a b]")[0], Token::Field("a b".to_string())); + assert_eq!(toks("[a[b]")[0], Token::Field("a[b".to_string())); + assert_eq!(toks("[café]")[0], Token::Field("café".to_string())); + assert_eq!(toks("[a\tb]")[0], Token::Field("a\tb".to_string())); + // No trimming: a single space is a legal one-character column name. + assert_eq!(toks("[ ]")[0], Token::Field(" ".to_string())); + assert_eq!(toks("[ s ]")[0], Token::Field(" s ".to_string())); + } + + #[test] + fn raw_text_is_kept_alongside_the_token() { + let tokens = tokenize("[age] GREATER \"x\"").unwrap(); + assert_eq!(tokens[0].text, "[age]"); + // Original case, so an error message can quote what was typed. + assert_eq!(tokens[1].text, "GREATER"); + assert_eq!(tokens[2].text, "\"x\""); + assert_eq!(tokens[3].describe(), ""); + assert_eq!(tokens[3].text, ""); + } + + #[test] + fn empty_field_is_rejected() { + // Viability stops after `[`, so the `]` that broke it is quoted too. + let e = err("[] = \"x\""); + assert_eq!(e.message, "token recognition error at: '[]'"); + assert_eq!((e.start, e.end), (0, 1)); + } + + #[test] + fn field_with_escaped_bracket_is_rejected() { + // `FIELD` has no escape rule, so the backslash-bracket closes the field + // and the remainder fails, exactly as in the bundle. + let tokens = tokenize("[a\\]").unwrap(); + assert_eq!(tokens[0].token, Token::Field("a\\".to_string())); + let e = err("[a\\]b] = \"x\""); + assert_eq!(e.message, "token recognition error at: 'b]'"); + assert_eq!((e.start, e.end), (4, 5)); + } + + #[test] + fn field_with_raw_newline_is_rejected() { + let e = err("[a\nb] = \"x\""); + assert_eq!(e.message, "token recognition error at: '[a\n'"); + assert_eq!((e.start, e.end), (0, 1)); + let e = err("[a\rb] = \"x\""); + assert_eq!(e.message, "token recognition error at: '[a\r'"); + } + + #[test] + fn unterminated_field_quotes_the_whole_remainder() { + let e = err("[age"); + assert_eq!(e.message, "token recognition error at: '[age'"); + assert_eq!((e.start, e.end), (0, 1)); + } + + #[test] + fn string_escapes_follow_the_reference() { + // Only the three recognised escapes are rewritten. + assert_eq!(toks("\"a\\\"b\"")[0], Token::Str("a\"b".to_string())); + assert_eq!(toks("\"a\\'b\"")[0], Token::Str("a'b".to_string())); + assert_eq!(toks("\"a\\\\b\"")[0], Token::Str("a\\b".to_string())); + // Everything else keeps its backslash. + assert_eq!(toks("\"a\\tb\"")[0], Token::Str("a\\tb".to_string())); + assert_eq!(toks("\"a\\nb\"")[0], Token::Str("a\\nb".to_string())); + assert_eq!(toks("\"a\\zb\"")[0], Token::Str("a\\zb".to_string())); + } + + #[test] + fn string_escape_replacement_order_is_sequential() { + // `\\'` collapses to backslash-quote because the `\'` rewrite runs + // before the `\\` one; a single scan would give backslash-backslash-quote. + assert_eq!(toks("\"\\\\'\"")[0], Token::Str("\\'".to_string())); + assert_eq!(toks("\"\\\\\\\\'\"")[0], Token::Str("\\\\'".to_string())); + assert_eq!(toks("\"\\\\t\"")[0], Token::Str("\\t".to_string())); + } + + #[test] + fn raw_newline_in_string_is_rejected() { + let e = err("[s] = \"a\nb\""); + assert_eq!(e.message, "token recognition error at: '\"a\n'"); + assert_eq!((e.start, e.end), (6, 7)); + } + + #[test] + fn raw_tab_in_string_is_accepted() { + assert_eq!(toks("\"a\tb\"")[0], Token::Str("a\tb".to_string())); + } + + #[test] + fn unterminated_string_is_rejected() { + let e = err("\"abc"); + assert_eq!(e.message, "token recognition error at: '\"abc'"); + // A trailing backslash consumes the closing quote. + let e = err("[s] = \"a\\\""); + assert_eq!(e.message, "token recognition error at: '\"a\\\"'"); + assert_eq!((e.start, e.end), (6, 7)); + } + + #[test] + fn single_quoted_string_is_rejected() { + let e = err("[name] = 'x'"); + assert_eq!(e.message, "token recognition error at: '''"); + assert_eq!((e.start, e.end), (9, 10)); + } + + #[test] + fn two_character_operators_win_maximal_munch() { + assert_eq!(toks("==")[0], Token::Eq2); + assert_eq!(toks("!=")[0], Token::Neq); + assert_eq!(toks(">=")[0], Token::Ge); + assert_eq!(toks("<=")[0], Token::Le); + } + + #[test] + fn spaced_two_character_operator_lexes_as_two_tokens() { + assert_eq!(toks("> ="), vec![Token::Gt, Token::Eq, Token::Eof]); + assert_eq!(toks("< ="), vec![Token::Lt, Token::Eq, Token::Eof]); + assert_eq!(toks("= ="), vec![Token::Eq, Token::Eq, Token::Eof]); + } + + #[test] + fn lone_bang_is_rejected_and_quotes_the_next_character() { + // `!` is a viable prefix of `!=`, so the space that rules it out is part + // of the quoted text. + let e = err("[age] ! 5"); + assert_eq!(e.message, "token recognition error at: '! '"); + assert_eq!((e.start, e.end), (6, 7)); + assert_eq!(err("[age] !x").message, "token recognition error at: '!x'"); + assert_eq!(err("!").message, "token recognition error at: '!'"); + } + + #[test] + fn exponent_syntax_is_rejected() { + // `e` could still become `equal`, `equals` or `ends`, so the `5` that + // rules all three out is quoted with it. + let e = err("[age] = 1e5"); + assert_eq!(e.message, "token recognition error at: 'e5'"); + assert_eq!((e.start, e.end), (9, 10)); + assert_eq!( + err("[age] = 1E5").message, + "token recognition error at: 'E5'" + ); + assert_eq!( + err("[age] = 1ee5").message, + "token recognition error at: 'ee'" + ); + assert_eq!(err("[age] = 1e").message, "token recognition error at: 'e'"); + } + + #[test] + fn a_keyword_followed_by_letters_lexes_then_fails() { + // Maximal munch takes the keyword, then the leftover letters fail — so + // the input as a whole does not lex and the error names only the tail. + // `tokenize` reports that first failure instead of the tokens preceding + // it, which is why this asserts through `err` and not `toks`. + let e = err("andx"); + assert_eq!(e.message, "token recognition error at: 'x'"); + assert_eq!((e.start, e.end), (3, 4)); + assert_eq!(err("notx").message, "token recognition error at: 'x'"); + assert_eq!(err("equalx").message, "token recognition error at: 'x'"); + assert_eq!(err("equalsx").message, "token recognition error at: 'x'"); + assert_eq!(err("blanks").message, "token recognition error at: 's'"); + // `a` is viable as a prefix of `and`, so the space is quoted with it. + assert_eq!(err("nota ").message, "token recognition error at: 'a '"); + } + + #[test] + fn adjacent_keywords_lex_without_separators() { + assert_eq!(toks("andor"), vec![Token::And, Token::Or, Token::Eof]); + assert_eq!(toks("isnot"), vec![Token::Is, Token::Not, Token::Eof]); + assert_eq!( + toks("isnotblank"), + vec![Token::Is, Token::Not, Token::Blank, Token::Eof] + ); + assert_eq!( + toks("lessthanorequal"), + vec![ + Token::Less, + Token::Than, + Token::Or, + Token::Equal, + Token::Eof + ] + ); + assert_eq!( + toks("startswith"), + vec![Token::Starts, Token::With, Token::Eof] + ); + } + + #[test] + fn a_keyword_prefix_alone_is_an_error() { + // Measured: `ends` lexes, `end`, `en` and `e` do not. + assert_eq!(toks("ends"), vec![Token::Ends, Token::Eof]); + assert_eq!(err("end").message, "token recognition error at: 'end'"); + assert_eq!(err("en").message, "token recognition error at: 'en'"); + assert_eq!(err("e").message, "token recognition error at: 'e'"); + } + + #[test] + fn an_unviable_character_quotes_only_itself() { + for (input, text) in [ + ("z", "z"), + ("q", "q"), + ("@", "@"), + ("#", "#"), + ("~", "~"), + ("zz", "z"), + ] { + assert_eq!( + err(input).message, + format!("token recognition error at: '{text}'"), + "{input}" + ); + } + // But a viable prefix drags in the character that broke it. + assert_eq!(err("ab").message, "token recognition error at: 'ab'"); + assert_eq!(err("abc").message, "token recognition error at: 'ab'"); + assert_eq!(err("an ").message, "token recognition error at: 'an '"); + assert_eq!(err("sta ").message, "token recognition error at: 'sta '"); + assert_eq!(err("i ").message, "token recognition error at: 'i '"); + } + + #[test] + fn keywords_are_case_insensitive() { + for spelling in ["AND", "and", "AnD"] { + assert_eq!(toks(spelling)[0], Token::And); + } + for spelling in ["CONTAINS", "contains", "CoNtAiNs"] { + assert_eq!(toks(spelling)[0], Token::Contains); + } + assert_eq!(toks("TRUE")[0], Token::True); + assert_eq!(toks("False")[0], Token::False); + } + + #[test] + fn equals_wins_over_equal() { + assert_eq!(toks("equals"), vec![Token::Equals, Token::Eof]); + assert_eq!(toks("equal"), vec![Token::Equal, Token::Eof]); + } + + #[test] + fn digits_are_grouped_and_dot_is_separate() { + assert_eq!( + toks("1 . 5"), + vec![ + Token::Digits("1".to_string()), + Token::Dot, + Token::Digits("5".to_string()), + Token::Eof, + ] + ); + assert_eq!(toks("007")[0], Token::Digits("007".to_string())); + } + + #[test] + fn keywords_need_no_surrounding_space() { + assert_eq!( + toks("[age]>1AND[age]<5"), + vec![ + Token::Field("age".to_string()), + Token::Gt, + Token::Digits("1".to_string()), + Token::And, + Token::Field("age".to_string()), + Token::Lt, + Token::Digits("5".to_string()), + Token::Eof, + ] + ); + } + + #[test] + fn whitespace_is_skipped() { + assert_eq!(toks(" \t\r\n "), vec![Token::Eof]); + assert_eq!(toks(""), vec![Token::Eof]); + } + + #[test] + fn spans_are_byte_offsets() { + let tokens = tokenize("[café] = \"x\"").unwrap(); + assert_eq!((tokens[0].start, tokens[0].end), (0, 7)); + assert_eq!((tokens[1].start, tokens[1].end), (8, 9)); + assert_eq!((tokens[2].start, tokens[2].end), (10, 13)); + assert_eq!(tokens[3].token, Token::Eof); + assert_eq!(tokens[3].start, "[café] = \"x\"".len()); + } + + #[test] + fn a_multibyte_offending_character_gets_a_whole_char_span() { + // The reference would report a one-code-unit span here; a byte-offset + // port reports the character's full width so the span stays sliceable. + let e = err("é"); + assert_eq!(e.message, "token recognition error at: 'é'"); + assert_eq!((e.start, e.end), (0, 2)); + assert_eq!(&"é"[e.start..e.end], "é"); + } + + #[test] + fn token_names_are_the_grammars() { + assert_eq!(Token::Blank.name(), "BLANK"); + assert_eq!(Token::With.name(), "WITH"); + assert_eq!(Token::RParen.name(), "')'"); + assert_eq!(Token::Digits("1".to_string()).name(), "DIGITS"); + assert_eq!(Token::Eof.name(), ""); + } + + #[test] + fn viability_is_zero_for_a_character_that_starts_nothing() { + assert_eq!(viable_len("z", 0), 0); + assert_eq!(viable_len("!", 0), 1); + assert_eq!(viable_len("e", 0), 1); + assert_eq!(viable_len("end", 0), 3); + assert_eq!(viable_len("ends", 0), 4); + assert_eq!(viable_len("[ab", 0), 3); + assert_eq!(viable_len("=", 0), 1); + assert_eq!(viable_len("12", 0), 2); + } +} diff --git a/rusty-filter/src/lib.rs b/rusty-filter/src/lib.rs new file mode 100644 index 0000000..232df4b --- /dev/null +++ b/rusty-filter/src/lib.rs @@ -0,0 +1,92 @@ +//! Filter query grammar, AST, validator, evaluator and printer. +//! +//! Rusty's `DataTable` renders the same filter query editor Ivy uses, so a query +//! typed in the browser has to mean the same thing on the server. This crate is +//! the Rust half: it parses a query string into the same AST, validates it +//! against a column schema, evaluates it against rows, and prints it back to a +//! canonical string suitable for a cache key. +//! +//! # Where the grammar comes from +//! +//! The grammar is not invented here. It is fixed by `docs/grammar/Filters.g4` in +//! `Ivy-Interactive/Ivy-Query-Editor` and by the shipped `filter-query-editor` +//! bundle under `src/frontend/node_modules/`. Behaviour was matched against +//! version **2.2.0** by calling `parseQuery`, `validateFilters`, `formatQuery` +//! and `evaluateFilter` over roughly a hundred inputs. Nothing in this crate +//! links against an ANTLR runtime: the lexer and the recursive-descent parser +//! are hand-written, and the only dependencies are `serde` and `serde_json`. +//! +//! # Getting started +//! +//! ``` +//! use rusty_filter::{parse_query, retain_matching, ColumnDef, ColumnType}; +//! +//! let columns = vec![ +//! ColumnDef::new("age", ColumnType::Number), +//! ColumnDef::new("name", ColumnType::String), +//! ]; +//! let result = parse_query("[age] > 30 AND [name] starts with \"A\"", &columns); +//! let filter = result.filters.expect("the query is valid"); +//! +//! let rows = vec![ +//! serde_json::json!({"age": 41, "name": "Ada"}), +//! serde_json::json!({"age": 41, "name": "Bob"}), +//! serde_json::json!({"age": 12, "name": "Ann"}), +//! ]; +//! let kept = retain_matching(&filter, rows, &columns); +//! assert_eq!(kept.len(), 1); +//! ``` +//! +//! An invalid query returns its errors instead: +//! +//! ``` +//! use rusty_filter::{parse_query, ColumnDef, ColumnType}; +//! +//! let columns = vec![ColumnDef::new("age", ColumnType::Number)]; +//! let result = parse_query("[age] contains \"3\"", &columns); +//! assert!(result.has_errors()); +//! assert_eq!( +//! result.errors()[0].message, +//! "Operator 'contains' is not compatible with type 'number'" +//! ); +//! ``` +//! +//! # Known divergences from the reference +//! +//! Four, all deliberate, each restated where it applies: +//! +//! * **Error spans are byte offsets, not UTF-16 code units.** The two agree for +//! ASCII queries and part company after a non-BMP character. Anything +//! highlighting a span from a Rust-side error has to account for that. +//! * **A syntax error stops the parse.** ANTLR recovers and can report a +//! cascade; here the first error is the only one. Semantic errors are still +//! reported in full, one per offending condition. +//! * **Evaluation uses the normalized column type.** The reference validates +//! against a normalized type but evaluates against the raw one, so a column +//! declared `INT32` passes validation for `>` and then silently matches +//! nothing. [`ColumnDef`] stores a normalized [`ColumnType`], so the same +//! filter matches here. See [`eval`]. +//! * **A datetime with no `Z` is read as UTC.** The browser reads it in the +//! machine's local zone, which would make server-side filtering depend on the +//! server's timezone. See [`eval`]. +//! +//! # Module layout +//! +//! [`lexer`] turns a query into tokens, [`parser`] turns tokens into the +//! [`ast`], [`validate`] checks that AST against a [`column`] schema, +//! [`eval`] runs it over rows, and [`print`] turns it back into text. + +pub mod ast; +pub mod column; +pub mod eval; +pub mod lexer; +pub mod parser; +pub mod print; +pub mod validate; + +pub use ast::{Condition, Filter, FilterFunction, FilterGroup, LogicalOp}; +pub use column::{ColumnDef, ColumnType}; +pub use eval::{count_matches, evaluate, retain_matching}; +pub use parser::{parse_query, parse_query_unchecked, ErrorSeverity, ParseError, ParseResult}; +pub use print::{canonical_key, to_query_string}; +pub use validate::validate_filter_group; diff --git a/rusty-filter/src/parser.rs b/rusty-filter/src/parser.rs new file mode 100644 index 0000000..1b78a61 --- /dev/null +++ b/rusty-filter/src/parser.rs @@ -0,0 +1,1617 @@ +//! Recursive-descent parser for the filter grammar, one function per parser rule. +//! +//! Precedence, from loosest to tightest: `OR`, `AND`, `NOT`, primary. The +//! AST-shaping rules are those of `dist/parser/ASTBuilder.js` and are reproduced +//! rather than simplified — in particular parentheses are never collapsed and +//! `NOT` *toggles* negation instead of forcing it true. +//! +//! # How error messages are worded +//! +//! ANTLR has four error shapes, and which one appears is decided mechanically. +//! Reproducing that decision is the only way a Rust-side message can read like +//! the browser's, so each rule below is implemented rather than approximated: +//! +//! * `extraneous input 'X' expecting T` — deleting the current token would let +//! the match succeed. +//! * `missing T at 'X'` — the current token could legally *follow* the token +//! that is missing, so one is assumed to have been left out. +//! * `mismatched input 'X' expecting T` — neither repair works. +//! * `no viable alternative at input 'X'` — a decision needing more than one +//! token of lookahead ran out of alternatives. `X` is every token's raw text +//! from the start of that decision onwards, run together. +//! +//! Deletion is tried before insertion, which is why `[age] = --5` blames the +//! extra sign instead of reporting a missing digit. Messages quote the raw +//! lexeme, so `[age] GREATER 5` complains about `GREATER`, not `greater`. +//! +//! # Divergences from the reference +//! +//! * `start` and `end` in [`ParseError`] are **byte** offsets into the input; +//! the TypeScript version reports UTF-16 code units. They agree for ASCII +//! input and differ once a multi-byte character appears before the error. +//! * On a syntax error this parser stops and reports **one** error. The ANTLR +//! reference recovers and can emit a cascade of several errors for one input; +//! matching that cascade is out of scope. The first error agrees. + +use serde::{Deserialize, Serialize}; + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup}; +use crate::column::ColumnDef; +use crate::lexer::{tokenize, SpannedToken, Token}; +use crate::validate::validate_filter_group; + +/// How serious a [`ParseError`] is. The reference emits `error` for everything +/// this crate produces; `Warning` exists to mirror the TypeScript union. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ErrorSeverity { + #[default] + Error, + Warning, +} + +/// A syntax or semantic error, with the byte span it applies to. +/// +/// Semantic errors carry `start: 0` and `end: 0` because the reference +/// validator has no position information to report — that is faithful, not a +/// defect to be "fixed". +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ParseError { + pub message: String, + pub start: usize, + pub end: usize, + pub severity: ErrorSeverity, +} + +impl ParseError { + pub fn new(message: impl Into, start: usize, end: usize) -> Self { + ParseError { + message: message.into(), + start, + end, + severity: ErrorSeverity::Error, + } + } +} + +/// The result of parsing: filters *or* errors, never both, mirroring +/// `dist/types/parser.d.ts` and the early returns of `parseQuery`. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct ParseResult { + #[serde(skip_serializing_if = "Option::is_none")] + pub filters: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub errors: Option>, +} + +impl ParseResult { + fn ok(filters: FilterGroup) -> Self { + ParseResult { + filters: Some(filters), + errors: None, + } + } + + fn failed(errors: Vec) -> Self { + ParseResult { + filters: None, + errors: Some(errors), + } + } + + /// Whether the parse produced errors. + pub fn has_errors(&self) -> bool { + self.errors.as_ref().is_some_and(|e| !e.is_empty()) + } + + /// The errors, or an empty slice on success. + pub fn errors(&self) -> &[ParseError] { + self.errors.as_deref().unwrap_or(&[]) + } +} + +/// Parse `query` and validate it against `columns`. +/// +/// Whitespace-only and empty input yield an empty `AND` group with no errors. +/// Syntax errors are reported before semantic ones, and semantic validation only +/// runs on a syntactically valid query — the same order `parseQuery` uses. +pub fn parse_query(query: &str, columns: &[ColumnDef]) -> ParseResult { + let group = match parse_query_unchecked(query) { + Ok(group) => group, + Err(errors) => return ParseResult::failed(errors), + }; + let semantic = validate_filter_group(&group, columns); + if semantic.is_empty() { + ParseResult::ok(group) + } else { + ParseResult::failed(semantic) + } +} + +/// Parse `query` without semantic validation, so no column schema is needed. +pub fn parse_query_unchecked(query: &str) -> Result> { + if query.trim().is_empty() { + return Ok(FilterGroup::default()); + } + let tokens = tokenize(query).map_err(|e| vec![ParseError::new(e.message, e.start, e.end)])?; + let mut parser = Parser::new(&tokens); + let group = parser.formula().map_err(|e| vec![e])?; + Ok(group) +} + +/// The result of visiting one rule: either a bare filter or a whole group, +/// exactly the untagged union `ASTBuilder`'s visitors return. +#[derive(Debug, Clone)] +enum Node { + Filter(Filter), + Group(FilterGroup), +} + +impl Node { + /// Wrap into a single `Filter`, as `visitGroup` does when the inner + /// expression came back as a group. + fn into_filter(self) -> Filter { + match self { + Node::Filter(f) => f, + Node::Group(g) => Filter::from_group(g), + } + } + + /// Lift into a `FilterGroup`, as `visitAndExpr` does for a single arm. + fn into_group(self) -> FilterGroup { + match self { + Node::Filter(f) => FilterGroup::and(vec![f]), + Node::Group(g) => g, + } + } + + /// Toggle negation on whichever variant this is. `visitUnaryExpr` sets + /// `negate = !negate` on the visited node, and a group node carries its own + /// `negate` only once it has been wrapped in a filter. + fn toggle_negate(self) -> Node { + match self { + Node::Filter(mut f) => { + f.negate = Some(!f.is_negated()); + Node::Filter(f) + } + Node::Group(g) => { + // A raw group has no `negate` field of its own, so the reference + // sets one on the object it is holding. Wrapping it in a filter + // is the faithful Rust equivalent: `NOT (a AND b)` becomes a + // group filter with `negate: true`. + let mut f = Filter::from_group(g); + f.negate = Some(true); + Node::Filter(f) + } + } + } +} + +struct Parser<'a> { + tokens: &'a [SpannedToken], + pos: usize, + /// How many `(` are currently open. Only the error wording reads this: a + /// token's legal followers depend on the enclosing groups, so `)` follows a + /// completed primary only inside one and `` only outside them all. + depth: usize, +} + +impl<'a> Parser<'a> { + fn new(tokens: &'a [SpannedToken]) -> Self { + Parser { + tokens, + pos: 0, + depth: 0, + } + } + + fn peek(&self) -> &'a Token { + &self.tokens[self.pos.min(self.tokens.len() - 1)].token + } + + fn peek_at(&self, offset: usize) -> &'a Token { + let idx = (self.pos + offset).min(self.tokens.len() - 1); + &self.tokens[idx].token + } + + fn current(&self) -> &'a SpannedToken { + &self.tokens[self.pos.min(self.tokens.len() - 1)] + } + + fn advance(&mut self) -> &'a SpannedToken { + let tok = self.current(); + if self.pos < self.tokens.len() - 1 { + self.pos += 1; + } + tok + } + + fn eat(&mut self, expected: &Token) -> bool { + if self.peek() == expected { + self.advance(); + true + } else { + false + } + } + + fn error(&self, message: impl Into) -> ParseError { + let tok = self.current(); + ParseError::new(message, tok.start, tok.end) + } + + /// Match a single expected token, or word the failure as ANTLR would. + /// + /// `expecting` names the token, `matches` recognises it (needed because + /// `STRING` and `DIGITS` carry payloads), and `follows` says whether a token + /// may legally appear *after* the expected one — which is what licenses the + /// `missing` wording. Deletion is tried before insertion. + fn expect( + &mut self, + expecting: &str, + matches: fn(&Token) -> bool, + follows: Follows, + ) -> Result<&'a SpannedToken, ParseError> { + if matches(self.peek()) { + return Ok(self.advance()); + } + let quoted = self.current().describe(); + if matches(self.peek_at(1)) { + return Err(self.error(format!("extraneous input '{quoted}' expecting {expecting}"))); + } + if follows(self.peek(), self.depth) { + return Err(self.error(format!("missing {expecting} at '{quoted}'"))); + } + Err(self.error(format!("mismatched input '{quoted}' expecting {expecting}"))) + } + + /// The failure for a position that expects one of a *set* of tokens. + /// + /// No `missing` wording here: with several candidates the reference never + /// picks one to insert, so only deletion is offered. + fn expected_set(&self, expecting: &str, matches: fn(&Token) -> bool) -> ParseError { + let quoted = self.current().describe(); + if matches(self.peek_at(1)) { + self.error(format!("extraneous input '{quoted}' expecting {expecting}")) + } else { + self.error(format!("mismatched input '{quoted}' expecting {expecting}")) + } + } + + /// The failure for a decision that ran out of alternatives. + /// + /// The quoted text is every token's raw spelling from `from` through the + /// token where prediction gave up, run together with no separators — so + /// `[age] not blank` reports `'[age]notblank'`. The span is that last + /// token's. + fn no_viable(&self, from: usize, failed_at: usize) -> ParseError { + let last = failed_at.min(self.tokens.len() - 1); + let text: String = self.tokens[from..=last] + .iter() + .map(|t| t.text.as_str()) + .collect(); + let tok = &self.tokens[last]; + ParseError::new( + format!("no viable alternative at input '{text}'"), + tok.start, + tok.end, + ) + } + + /// `formula : expr EOF` + fn formula(&mut self) -> Result { + let node = self.expr()?; + self.expect("", is_eof, never)?; + Ok(node.into_group()) + } + + /// `expr : orExpr` + fn expr(&mut self) -> Result { + self.or_expr() + } + + /// `orExpr : andExpr (OR andExpr)*` + /// + /// A single arm is returned unchanged. Several arms produce an `OR` group in + /// which each arm that came back as a group is spliced in bare when it holds + /// exactly one filter, and wrapped as `{group: ...}` otherwise. + fn or_expr(&mut self) -> Result { + let first = self.and_expr()?; + if self.peek() != &Token::Or { + return Ok(first); + } + let mut arms = vec![first]; + while self.eat(&Token::Or) { + arms.push(self.and_expr()?); + } + let mut filters = Vec::with_capacity(arms.len()); + for arm in arms { + match arm { + Node::Group(group) => { + if group.filters.len() == 1 { + filters.push(group.filters.into_iter().next().expect("length checked")); + } else { + filters.push(Filter::from_group(group)); + } + } + Node::Filter(f) => filters.push(f), + } + } + Ok(Node::Group(FilterGroup::or(filters))) + } + + /// `andExpr : unaryExpr (AND unaryExpr)*` + /// + /// A single arm that is already a group is returned as-is; a single filter is + /// wrapped in an `AND` group. Several arms are pushed as-is with no splicing. + fn and_expr(&mut self) -> Result { + let first = self.unary_expr()?; + if self.peek() != &Token::And { + return Ok(Node::Group(first.into_group())); + } + let mut filters = vec![first.into_filter()]; + while self.eat(&Token::And) { + filters.push(self.unary_expr()?.into_filter()); + } + Ok(Node::Group(FilterGroup::and(filters))) + } + + /// `unaryExpr : NOT unaryExpr | primary` + fn unary_expr(&mut self) -> Result { + if self.eat(&Token::Not) { + let inner = self.unary_expr()?; + return Ok(inner.toggle_negate()); + } + self.primary() + } + + /// `primary : group | comparison | textOperation | existenceOperation` + /// + /// The three field-led alternatives are chosen by looking past the field + /// reference and an optional `NOT`. When no alternative survives that + /// lookahead the failure is a `no viable alternative` quoting from the field, + /// because the field is where all three alternatives begin. + fn primary(&mut self) -> Result { + if self.peek() == &Token::LParen { + return self.group(); + } + let field = self.pos; + let Token::Field(column) = self.peek().clone() else { + return Err(self.expected_set("{FIELD, NOT, '('}", starts_primary)); + }; + match self.peek_at(1) { + Token::Contains | Token::Starts | Token::Ends => { + self.advance(); + self.text_operation(column) + } + Token::Is => { + self.advance(); + self.existence_operation(column, field) + } + // After `NOT` only negated equality and the text operators remain. + Token::Not => match self.peek_at(2) { + Token::Equals | Token::Equal => { + self.advance(); + self.comparison(column) + } + Token::Contains | Token::Starts | Token::Ends => { + self.advance(); + self.text_operation(column) + } + _ => Err(self.no_viable(field, self.pos + 2)), + }, + next if starts_comp_op(next) => { + self.advance(); + self.comparison(column) + } + _ => Err(self.no_viable(field, self.pos + 1)), + } + } + + /// `group : LPAREN expr RPAREN` + /// + /// Parentheses are never collapsed: the inner group is wrapped as a group + /// filter, so `(([age] > 1))` keeps both levels. + fn group(&mut self) -> Result { + self.advance(); // LPAREN + self.depth += 1; + let inner = self.expr()?; + // The `)` this rule is about is the one being closed, so its own follow + // set is that of the *enclosing* group: `([age] > 1` reports a missing + // `)` at ``, while `(([age] > 1)` reports the same at depth 1. + self.depth -= 1; + self.expect("')'", is_rparen, ends_primary)?; + Ok(match inner { + Node::Group(g) => Node::Filter(Filter::from_group(g)), + Node::Filter(f) => Node::Filter(f), + }) + } + + /// `comparison : fieldRef compOp operand` + /// + /// `negate` is `Some(true)` for the three not-equal spellings and `None` + /// otherwise — comparisons never emit `negate: false`. + fn comparison(&mut self, column: String) -> Result { + let (function, negate) = self.comp_op(self.pos)?; + let value = self.operand()?; + let mut filter = Filter { + condition: Some(Condition::new(column, function, vec![value])), + group: None, + negate: None, + }; + if negate { + filter.negate = Some(true); + } + Ok(Node::Filter(filter)) + } + + /// `compOp` — returns the mapped function and whether it negates. + /// + /// `op_start` is where a `no viable alternative` message should quote from: + /// the word operators are their own decision, so `[age] greater 5` quotes + /// `'greater5'` and not the field, while a failure that only becomes visible + /// here is still attributed to the whole primary. + fn comp_op(&mut self, op_start: usize) -> Result<(FilterFunction, bool), ParseError> { + let tok = self.peek().clone(); + match tok { + Token::Eq | Token::Eq2 => { + self.advance(); + Ok((FilterFunction::Equals, false)) + } + Token::Neq => { + self.advance(); + Ok((FilterFunction::Equals, true)) + } + Token::Gt => { + self.advance(); + Ok((FilterFunction::GreaterThan, false)) + } + Token::Ge => { + self.advance(); + Ok((FilterFunction::GreaterThanOrEqual, false)) + } + Token::Lt => { + self.advance(); + Ok((FilterFunction::LessThan, false)) + } + Token::Le => { + self.advance(); + Ok((FilterFunction::LessThanOrEqual, false)) + } + Token::Equals => { + self.advance(); + Ok((FilterFunction::Equals, false)) + } + // `NOT EQUALS` and `NOT EQUAL` both mean negated equality. A bare + // `EQUAL` without `NOT` is not a `compOp` alternative, which is why + // `primary` never routes one here. + Token::Not => { + self.advance(); + self.advance(); // EQUALS or EQUAL, checked by `primary` + Ok((FilterFunction::Equals, true)) + } + Token::Greater | Token::Less => { + let is_greater = tok == Token::Greater; + let word_start = self.pos; + self.advance(); + if self.peek() != &Token::Than { + // `greater`/`less` without `than` exhausts the alternatives + // for this decision, so the message quotes from the word. + return Err(self.no_viable(word_start, self.pos)); + } + self.advance(); + // The optional `OR EQUAL` tail. `OR EQUALS` (plural) is rejected + // by the grammar, which spells this alternative `OR EQUAL`. + if self.eat(&Token::Or) { + self.expect("EQUAL", is_equal, operand_follows)?; + Ok(( + if is_greater { + FilterFunction::GreaterThanOrEqual + } else { + FilterFunction::LessThanOrEqual + }, + false, + )) + } else { + Ok(( + if is_greater { + FilterFunction::GreaterThan + } else { + FilterFunction::LessThan + }, + false, + )) + } + } + // Unreachable: `primary` only routes a `compOp` start here. + _ => Err(self.no_viable(op_start, self.pos)), + } + } + + /// `textOperation : fieldRef NOT? textOp stringLiteral` + /// + /// `negate` is **always** `Some`, `false` included. + fn text_operation(&mut self, column: String) -> Result { + let has_not = self.eat(&Token::Not); + let function = self.text_op()?; + let token = self.expect("STRING", is_string, ends_primary)?; + let Token::Str(value) = &token.token else { + unreachable!("`expect` matched a STRING"); + }; + let filter = Filter { + condition: Some(Condition::new( + column, + function, + vec![serde_json::Value::String(value.clone())], + )), + group: None, + negate: Some(has_not), + }; + Ok(Node::Filter(filter)) + } + + /// `textOp : CONTAINS | STARTS WITH | ENDS WITH` + fn text_op(&mut self) -> Result { + match self.peek().clone() { + Token::Contains => { + self.advance(); + Ok(FilterFunction::Contains) + } + Token::Starts => { + self.advance(); + self.expect("WITH", is_with, string_follows)?; + Ok(FilterFunction::StartsWith) + } + // Unreachable for anything but `ENDS`: `primary` chose this rule. + _ => { + self.advance(); + self.expect("WITH", is_with, string_follows)?; + Ok(FilterFunction::EndsWith) + } + } + } + + /// `existenceOperation : fieldRef IS BLANK | fieldRef IS NOT BLANK` + /// + /// Produces `IsBlank` or `IsNotBlank` with empty `args` and no `negate`. + /// `field` is where a `no viable alternative` quotes from, since `[age] is 5` + /// abandons the whole primary rather than just this rule. + fn existence_operation(&mut self, column: String, field: usize) -> Result { + self.advance(); // IS + let negated = self.eat(&Token::Not); + if self.peek() != &Token::Blank { + // `IS NOT` has committed to this alternative, so a missing `BLANK` + // is reported as such. A bare `IS` followed by junk has not, and + // fails the primary decision instead. + if negated { + self.expect("BLANK", is_blank, ends_primary)?; + } else { + return Err(self.no_viable(field, self.pos)); + } + } + self.advance(); + let function = if negated { + FilterFunction::IsNotBlank + } else { + FilterFunction::IsBlank + }; + Ok(Node::Filter(Filter::condition(column, function, vec![]))) + } + + /// `operand : number | stringLiteral | booleanLiteral` + fn operand(&mut self) -> Result { + match self.peek().clone() { + Token::Str(s) => { + self.advance(); + Ok(serde_json::Value::String(s)) + } + Token::True => { + self.advance(); + Ok(serde_json::Value::Bool(true)) + } + Token::False => { + self.advance(); + Ok(serde_json::Value::Bool(false)) + } + Token::Sign(_) | Token::Digits(_) => self.number(), + _ => Err(self.expected_set(OPERAND_SET, starts_operand)), + } + } + + /// `number : SIGN? DIGITS (DOT DIGITS)?` + /// + /// Both sides of the dot are required, so `.5` and `5.` are errors. Since + /// whitespace is skipped between tokens, `1 . 5` is `1.5`. + fn number(&mut self) -> Result { + let negative = match self.peek() { + Token::Sign(c) => { + let neg = *c == '-'; + self.advance(); + Some(neg) + } + _ => None, + }; + // A sign with no digits ends the primary, so the follow set is the + // primary's: `[age] = -` reports a missing DIGITS at ``, while + // `[age] = - "x"` is a mismatch because a string cannot follow either. + let int_token = self.expect("DIGITS", is_digits, ends_primary)?; + let Token::Digits(int_part) = &int_token.token else { + unreachable!("`expect` matched DIGITS"); + }; + let mut text = int_part.clone(); + // The fraction is committed to on sight of the dot: the reference's + // adaptive prediction would look ahead, but every input where the two + // could differ is a syntax error either way. + if self.peek() == &Token::Dot { + self.advance(); + let frac_token = self.expect("DIGITS", is_digits, ends_primary)?; + let Token::Digits(frac) = &frac_token.token else { + unreachable!("`expect` matched DIGITS"); + }; + text.push('.'); + text.push_str(frac); + } + Ok(js_number(&text, negative == Some(true))) + } +} + +/// The token set an `operand` may start with, spelled as the message spells it. +const OPERAND_SET: &str = "{STRING, TRUE, FALSE, SIGN, DIGITS}"; + +/// Build the JSON number `parseFloat` would produce for `text`. +/// +/// The distinction matters for more than tidiness: `serde_json` remembers +/// whether a number was written as an integer, and `1.0` is not equal to `1`. +/// JavaScript has one number type, so an integral value must come back as an +/// integer or every AST comparison against the frontend's JSON fails. +fn js_number(text: &str, negative: bool) -> serde_json::Value { + // `parseFloat` on an over-long literal yields `Infinity`, which JSON cannot + // represent; the reference emits `null` for that arg. + let Ok(magnitude) = text.parse::() else { + return serde_json::Value::Null; + }; + let value = if negative { -magnitude } else { magnitude }; + if !value.is_finite() { + return serde_json::Value::Null; + } + // `-0` is `0` in JSON, and an integral f64 must serialize without a `.0`. + if value == value.trunc() && value.abs() < 9.007_199_254_740_992e15 { + return serde_json::Value::Number((value as i64).into()); + } + serde_json::Number::from_f64(value) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null) +} + +// The token predicates `expect` and `expected_set` are parameterised over. Each +// is a plain fn so the call sites stay allocation-free. + +/// Whether a token may legally follow the one `expect` is looking for, given how +/// many groups are currently open. The depth is what makes `missing STRING at +/// ')'` correct inside a group and `mismatched input ')' expecting STRING` +/// correct outside one. +type Follows = fn(&Token, usize) -> bool; + +fn never(_: &Token, _: usize) -> bool { + false +} + +fn is_eof(token: &Token) -> bool { + token == &Token::Eof +} + +fn is_rparen(token: &Token) -> bool { + token == &Token::RParen +} + +fn is_equal(token: &Token) -> bool { + token == &Token::Equal +} + +fn is_with(token: &Token) -> bool { + token == &Token::With +} + +fn is_blank(token: &Token) -> bool { + token == &Token::Blank +} + +fn is_string(token: &Token) -> bool { + matches!(token, Token::Str(_)) +} + +fn is_digits(token: &Token) -> bool { + matches!(token, Token::Digits(_)) +} + +/// The tokens a `primary` may start with: `{FIELD, NOT, '('}`. +fn starts_primary(token: &Token) -> bool { + matches!(token, Token::Field(_) | Token::Not | Token::LParen) +} + +/// The tokens an `operand` may start with. +fn starts_operand(token: &Token) -> bool { + matches!( + token, + Token::Str(_) | Token::True | Token::False | Token::Sign(_) | Token::Digits(_) + ) +} + +/// The tokens a `compOp` may start with. +fn starts_comp_op(token: &Token) -> bool { + matches!( + token, + Token::Eq + | Token::Eq2 + | Token::Neq + | Token::Gt + | Token::Ge + | Token::Lt + | Token::Le + | Token::Equals + | Token::Greater + | Token::Less + ) +} + +/// What may legally follow a completed `primary`, and so what licenses a +/// `missing` rather than a `mismatched` inside one. +/// +/// The depth is load-bearing, because ANTLR computes the follow set from the +/// rule invocation stack rather than from the grammar alone. `)` can only follow +/// while a group is open and `` only once every group has closed, which is +/// why `([name] contains` reports `mismatched input '' expecting STRING` +/// while the unparenthesised `[name] contains` reports `missing STRING at +/// ''`. Measured against the 2.2.0 bundle over `STRING`, `BLANK`, `DIGITS` +/// and `')'` at both depths. +fn ends_primary(token: &Token, depth: usize) -> bool { + match token { + Token::And | Token::Or => true, + Token::RParen => depth > 0, + Token::Eof => depth == 0, + _ => false, + } +} + +/// [`starts_operand`] as a [`Follows`]: the operand set does not depend on how +/// many groups are open. +fn operand_follows(token: &Token, _depth: usize) -> bool { + starts_operand(token) +} + +/// [`is_string`] as a [`Follows`], for the `WITH` in `STARTS WITH`. +fn string_follows(token: &Token, _depth: usize) -> bool { + is_string(token) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::LogicalOp; + use serde_json::json; + + fn parse(query: &str) -> FilterGroup { + parse_query_unchecked(query).unwrap_or_else(|e| panic!("{query:?} failed to parse: {e:?}")) + } + + fn parse_err(query: &str) -> ParseError { + parse_query_unchecked(query) + .map(|g| panic!("{query:?} unexpectedly parsed to {g:?}")) + .unwrap_err() + .into_iter() + .next() + .expect("at least one error") + } + + fn cond(column: &str, function: FilterFunction, args: Vec) -> Filter { + Filter::condition(column, function, args) + } + + // --- one assertion per row of the measured table ----------------------- + + #[test] + fn measured_greater_than() { + assert_eq!( + parse("[age] > 100"), + FilterGroup::and(vec![cond( + "age", + FilterFunction::GreaterThan, + vec![json!(100)] + )]) + ); + } + + #[test] + fn measured_not_equal_is_negated_equals() { + let expected = FilterGroup::and(vec![ + cond("age", FilterFunction::Equals, vec![json!(5)]).negated(true) + ]); + // All three not-equal spellings agree, and none produces a `notEquals`. + assert_eq!(parse("[age] != 5"), expected); + assert_eq!(parse("[age] not equals 5"), expected); + assert_eq!(parse("[age] not equal 5"), expected); + } + + #[test] + fn measured_not_contains() { + assert_eq!( + parse("[name] not contains \"ab\""), + FilterGroup::and(vec![cond( + "name", + FilterFunction::Contains, + vec![json!("ab")] + ) + .negated(true)]) + ); + } + + #[test] + fn measured_contains_emits_negate_false() { + let group = parse("[name] contains \"ab\""); + assert_eq!(group.filters[0].negate, Some(false)); + let value = serde_json::to_value(&group).unwrap(); + assert_eq!(value["filters"][0]["negate"], json!(false)); + } + + #[test] + fn measured_comparison_omits_negate() { + let group = parse("[age] > 1"); + assert_eq!(group.filters[0].negate, None); + let value = serde_json::to_value(&group).unwrap(); + assert!(value["filters"][0].get("negate").is_none()); + } + + #[test] + fn measured_or_root_with_spliced_and_arm() { + assert_eq!( + parse("[age] > 1 AND [n] = \"a\" OR [x] = true"), + FilterGroup::or(vec![ + Filter::from_group(FilterGroup::and(vec![ + cond("age", FilterFunction::GreaterThan, vec![json!(1)]), + cond("n", FilterFunction::Equals, vec![json!("a")]), + ])), + cond("x", FilterFunction::Equals, vec![json!(true)]), + ]) + ); + } + + #[test] + fn measured_not_lands_on_the_condition_filter() { + assert_eq!( + parse("NOT [age] > 1"), + FilterGroup::and(vec![cond( + "age", + FilterFunction::GreaterThan, + vec![json!(1)] + ) + .negated(true)]) + ); + } + + #[test] + fn measured_not_toggles() { + let group = parse("not not [age] > 1"); + assert_eq!(group.filters[0].negate, Some(false)); + assert_eq!(parse("NOT NOT NOT [age] > 1").filters[0].negate, Some(true)); + } + + #[test] + fn measured_parens_are_not_collapsed() { + assert_eq!( + parse("(([age] > 1))"), + FilterGroup::and(vec![Filter::from_group(FilterGroup::and(vec![ + Filter::from_group(FilterGroup::and(vec![cond( + "age", + FilterFunction::GreaterThan, + vec![json!(1)] + )])) + ]))]) + ); + // A third level nests a third time. + let deep = parse("((([age] > 1)))"); + let lvl1 = deep.filters[0].group.as_ref().unwrap(); + let lvl2 = lvl1.filters[0].group.as_ref().unwrap(); + let lvl3 = lvl2.filters[0].group.as_ref().unwrap(); + assert!(lvl3.filters[0].condition.is_some()); + } + + #[test] + fn measured_empty_and_blank_input() { + assert_eq!(parse(""), FilterGroup::default()); + assert_eq!(parse(" "), FilterGroup::default()); + assert_eq!(parse("\t\r\n"), FilterGroup::default()); + // and no errors + assert!(!parse_query("", &[]).has_errors()); + assert!(!parse_query(" ", &[]).has_errors()); + } + + #[test] + fn measured_whitespace_inside_a_number() { + for query in ["[age] = 1 . 5", "[age] = 1 .5", "[age] = 1. 5"] { + assert_eq!( + parse(query).filters[0].condition.as_ref().unwrap().args[0], + json!(1.5), + "{query}" + ); + } + } + + #[test] + fn measured_leading_zeros_are_dropped() { + assert_eq!( + parse("[age] = 007").filters[0] + .condition + .as_ref() + .unwrap() + .args[0], + json!(7) + ); + assert_eq!( + parse("[age] = 000000123").filters[0] + .condition + .as_ref() + .unwrap() + .args[0], + json!(123) + ); + } + + #[test] + fn measured_single_quotes_are_not_string_delimiters() { + assert_eq!( + parse_err("[name] = 'x'").message, + "token recognition error at: '''" + ); + } + + #[test] + fn measured_exponent_is_rejected() { + assert_eq!( + parse_err("[age] = 1e5").message, + "token recognition error at: 'e5'" + ); + } + + #[test] + fn measured_both_sides_of_the_dot_are_required() { + // `.5` — the dot is not a valid operand start. + assert!(parse_err("[age] = .5").message.contains("expecting")); + // `5.` — the fraction digits are missing. + assert_eq!(parse_err("[age] = 5.").message, "missing DIGITS at ''"); + assert_eq!( + parse_err("[age] = 1 .").message, + "missing DIGITS at ''" + ); + } + + #[test] + fn measured_nested_bracket_is_a_legal_column_name() { + assert_eq!( + parse("[a[b] = \"x\"").filters[0] + .condition + .as_ref() + .unwrap() + .column, + "a[b" + ); + } + + #[test] + fn measured_single_space_is_a_legal_column_name() { + assert_eq!( + parse("[ ] = \"x\"").filters[0] + .condition + .as_ref() + .unwrap() + .column, + " " + ); + assert!(parse_query_unchecked("[] = \"x\"").is_err()); + } + + // --- operator spellings ------------------------------------------------ + + #[test] + fn symbolic_operators_map_to_functions() { + let cases = [ + ("=", FilterFunction::Equals), + ("==", FilterFunction::Equals), + (">", FilterFunction::GreaterThan), + (">=", FilterFunction::GreaterThanOrEqual), + ("<", FilterFunction::LessThan), + ("<=", FilterFunction::LessThanOrEqual), + ]; + for (op, expected) in cases { + let group = parse(&format!("[age] {op} 5")); + assert_eq!( + group.filters[0].condition.as_ref().unwrap().function, + expected, + "{op}" + ); + } + } + + #[test] + fn word_operators_map_to_functions() { + let cases = [ + ("equals", FilterFunction::Equals), + ("greater than", FilterFunction::GreaterThan), + ("greater than or equal", FilterFunction::GreaterThanOrEqual), + ("less than", FilterFunction::LessThan), + ("less than or equal", FilterFunction::LessThanOrEqual), + ("GREATER THAN OR EQUAL", FilterFunction::GreaterThanOrEqual), + ]; + for (op, expected) in cases { + let group = parse(&format!("[age] {op} 5")); + assert_eq!( + group.filters[0].condition.as_ref().unwrap().function, + expected, + "{op}" + ); + } + } + + #[test] + fn bare_equal_without_not_is_rejected() { + assert_eq!( + parse_err("[age] equal 5").message, + "no viable alternative at input '[age]equal'" + ); + } + + #[test] + fn plural_or_equals_is_rejected() { + // The grammar spells this tail `OR EQUAL`, singular. + assert_eq!( + parse_err("[age] greater than or equals 5").message, + "mismatched input 'equals' expecting EQUAL" + ); + assert_eq!( + parse_err("[age] less than or equals 5").message, + "mismatched input 'equals' expecting EQUAL" + ); + } + + #[test] + fn text_operators_and_existence_operators() { + assert_eq!( + parse("[name] starts with \"a\"").filters[0] + .condition + .as_ref() + .unwrap() + .function, + FilterFunction::StartsWith + ); + assert_eq!( + parse("[name] ends with \"a\"").filters[0] + .condition + .as_ref() + .unwrap() + .function, + FilterFunction::EndsWith + ); + assert_eq!( + parse("[name] is blank"), + FilterGroup::and(vec![cond("name", FilterFunction::IsBlank, vec![])]) + ); + assert_eq!( + parse("[name] is not blank"), + FilterGroup::and(vec![cond("name", FilterFunction::IsNotBlank, vec![])]) + ); + } + + #[test] + fn existence_operations_carry_no_negate_key() { + for query in ["[name] is blank", "[name] is not blank"] { + let group = parse(query); + assert_eq!(group.filters[0].negate, None, "{query}"); + assert!(group.filters[0].condition.as_ref().unwrap().args.is_empty()); + } + } + + #[test] + fn negated_text_operations_always_emit_negate() { + for (query, expected) in [ + ("[name] contains \"a\"", false), + ("[name] not contains \"a\"", true), + ("[name] starts with \"a\"", false), + ("[name] not starts with \"a\"", true), + ("[name] ends with \"a\"", false), + ("[name] not ends with \"a\"", true), + ] { + assert_eq!(parse(query).filters[0].negate, Some(expected), "{query}"); + } + } + + #[test] + fn text_operation_rejects_a_non_string_argument() { + assert_eq!( + parse_err("[name] contains 5").message, + "mismatched input '5' expecting STRING" + ); + } + + // --- structural shaping ------------------------------------------------ + + #[test] + fn three_arm_and_stays_flat() { + let group = parse("[age] > 1 AND [age] < 5 AND [age] != 3"); + assert_eq!(group.op, LogicalOp::And); + assert_eq!(group.filters.len(), 3); + assert!(group.filters.iter().all(|f| f.condition.is_some())); + assert_eq!(group.filters[2].negate, Some(true)); + } + + #[test] + fn three_arm_or_stays_flat() { + let group = parse("[age] > 1 OR [age] < 5 OR [age] != 3"); + assert_eq!(group.op, LogicalOp::Or); + assert_eq!(group.filters.len(), 3); + assert!(group.filters.iter().all(|f| f.condition.is_some())); + } + + #[test] + fn or_splices_single_filter_arms_and_wraps_the_rest() { + // Middle arm has two filters, so it stays wrapped; the outer arms are bare. + let group = parse("[age] > 1 OR [age] < 2 AND [age] > 3 OR [age] < 4"); + assert_eq!(group.op, LogicalOp::Or); + assert_eq!(group.filters.len(), 3); + assert!(group.filters[0].condition.is_some()); + assert_eq!( + group.filters[1].group.as_ref().map(|g| g.filters.len()), + Some(2) + ); + assert!(group.filters[2].condition.is_some()); + } + + #[test] + fn an_explicit_group_arm_of_or_keeps_its_wrapper() { + // `([age] > 1)` is a *group filter* inside a one-filter AND group, so the + // splice hands the group filter through rather than unwrapping the paren. + let group = parse("([age] > 1) OR [age] < 2"); + assert_eq!(group.op, LogicalOp::Or); + assert_eq!( + group.filters[0] + .group + .as_ref() + .map(|g| (g.op, g.filters.len())), + Some((LogicalOp::And, 1)) + ); + assert!(group.filters[1].condition.is_some()); + } + + #[test] + fn a_group_inside_and_is_pushed_as_is() { + let group = parse("([age] > 1 OR [age] < 0) AND [name] = \"a\""); + assert_eq!(group.op, LogicalOp::And); + assert_eq!(group.filters.len(), 2); + assert_eq!( + group.filters[0].group.as_ref().map(|g| g.op), + Some(LogicalOp::Or) + ); + assert!(group.filters[1].condition.is_some()); + } + + #[test] + fn not_before_a_group_negates_the_group_filter() { + let group = parse("NOT ([age] > 1 AND [age] < 5)"); + assert_eq!(group.filters.len(), 1); + assert_eq!(group.filters[0].negate, Some(true)); + assert_eq!( + group.filters[0].group.as_ref().map(|g| g.filters.len()), + Some(2) + ); + } + + #[test] + fn not_before_a_group_toggles_too() { + let group = parse("NOT NOT ([age] > 1)"); + assert_eq!(group.filters[0].negate, Some(false)); + assert!(group.filters[0].group.is_some()); + } + + #[test] + fn not_inside_an_and_chain() { + let group = parse("[age] > 1 AND NOT [age] < 5"); + assert_eq!(group.filters.len(), 2); + assert_eq!(group.filters[0].negate, None); + assert_eq!(group.filters[1].negate, Some(true)); + } + + // --- syntax errors ----------------------------------------------------- + + #[test] + fn trailing_operator_with_no_operand_is_rejected() { + assert_eq!( + parse_err("[age] >").message, + "mismatched input '' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}" + ); + assert_eq!( + parse_err("[age] equals").message, + "mismatched input '' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}" + ); + } + + #[test] + fn dangling_and_is_rejected() { + assert_eq!( + parse_err("[age] > 1 AND").message, + "mismatched input '' expecting {FIELD, NOT, '('}" + ); + assert_eq!( + parse_err("[age] > 1 OR").message, + "mismatched input '' expecting {FIELD, NOT, '('}" + ); + } + + #[test] + fn trailing_extra_literal_is_rejected() { + let e = parse_err("[age] > 1 1"); + assert_eq!(e.message, "extraneous input '1' expecting "); + assert_eq!((e.start, e.end), (10, 11)); + } + + #[test] + fn a_second_operator_is_rejected() { + // `mismatched`, not `extraneous`: nothing may follow ``, so deleting + // the offending token would not let the match succeed either. + assert_eq!( + parse_err("[age] = 5 = 5").message, + "mismatched input '=' expecting " + ); + assert_eq!( + parse_err("[age] = 1.5.5").message, + "mismatched input '.' expecting " + ); + // A second *literal* is deletable, because a literal is what `` + // would follow — hence `extraneous` for this one. + assert_eq!( + parse_err("[age] > 1 1").message, + "extraneous input '1' expecting " + ); + } + + #[test] + fn spaced_two_character_operator_is_rejected() { + for query in ["[age] > = 5", "[age] < = 5", "[age] = = 5"] { + let e = parse_err(query); + assert!( + e.message + .contains("expecting {STRING, TRUE, FALSE, SIGN, DIGITS}"), + "{query}: {}", + e.message + ); + } + } + + #[test] + fn double_sign_is_rejected() { + // Deletion is tried first and the second `-` is followed by a digit, so + // the extra sign is blamed rather than a digit called missing. + assert_eq!( + parse_err("[age] = --5").message, + "extraneous input '-' expecting DIGITS" + ); + assert_eq!( + parse_err("[age] = -.5").message, + "extraneous input '.' expecting DIGITS" + ); + } + + #[test] + fn a_bare_literal_or_field_is_rejected() { + assert_eq!( + parse_err("5").message, + "mismatched input '5' expecting {FIELD, NOT, '('}" + ); + assert_eq!( + parse_err("()").message, + "mismatched input ')' expecting {FIELD, NOT, '('}" + ); + } + + #[test] + fn a_field_with_no_operator_quotes_from_the_field() { + // The three field-led alternatives all begin at the field, so the + // message runs the tokens together from there rather than naming just + // the token that failed. + assert_eq!( + parse_err("[age]").message, + "no viable alternative at input '[age]'" + ); + assert_eq!( + parse_err("[age] not").message, + "no viable alternative at input '[age]not'" + ); + assert_eq!( + parse_err("[age] not 5").message, + "no viable alternative at input '[age]not5'" + ); + assert_eq!( + parse_err("[age] not blank").message, + "no viable alternative at input '[age]notblank'" + ); + assert_eq!( + parse_err("[age] is").message, + "no viable alternative at input '[age]is'" + ); + assert_eq!( + parse_err("[age] is 5").message, + "no viable alternative at input '[age]is5'" + ); + assert_eq!( + parse_err("[age] not >").message, + "no viable alternative at input '[age]not>'" + ); + // Nesting and position do not change where the quoting starts. + assert_eq!( + parse_err("([age] equal 5)").message, + "no viable alternative at input '[age]equal'" + ); + assert_eq!( + parse_err("[age] > 1 AND [name] equal \"a\"").message, + "no viable alternative at input '[name]equal'" + ); + } + + #[test] + fn a_word_operator_with_no_than_quotes_from_the_word() { + // `greater`/`less` open a decision of their own, so the field is not + // part of the quoted text. + assert_eq!( + parse_err("[age] greater 5").message, + "no viable alternative at input 'greater5'" + ); + assert_eq!( + parse_err("[age] less 5").message, + "no viable alternative at input 'less5'" + ); + assert_eq!( + parse_err("[age] greater").message, + "no viable alternative at input 'greater'" + ); + assert_eq!( + parse_err("[age] less").message, + "no viable alternative at input 'less'" + ); + assert_eq!( + parse_err("([age] greater 5)").message, + "no viable alternative at input 'greater5'" + ); + } + + #[test] + fn quoted_text_keeps_the_original_case() { + assert_eq!( + parse_err("[age] EQUAL 5").message, + "no viable alternative at input '[age]EQUAL'" + ); + assert_eq!( + parse_err("[age] GREATER 5").message, + "no viable alternative at input 'GREATER5'" + ); + assert_eq!( + parse_err("[age] NOT 5").message, + "no viable alternative at input '[age]NOT5'" + ); + } + + #[test] + fn unbalanced_parens_are_rejected() { + assert_eq!(parse_err("([age] > 1").message, "missing ')' at ''"); + assert_eq!( + parse_err("([age] > 1 AND [age] < 2").message, + "missing ')' at ''" + ); + // A token that cannot follow the group is a mismatch, not an omission. + assert_eq!( + parse_err("([age] > 1 5").message, + "mismatched input '5' expecting ')'" + ); + assert_eq!( + parse_err("[age] > 1)").message, + "extraneous input ')' expecting " + ); + } + + #[test] + fn a_missing_token_is_reported_as_missing_when_one_would_do() { + // Insertion fires only where the offending token could legally follow + // the one left out. + for (query, expected) in [ + ("[age] is not", "missing BLANK at ''"), + ("[age] is not AND [age] > 1", "missing BLANK at 'AND'"), + ("[name] contains", "missing STRING at ''"), + ("[name] not contains", "missing STRING at ''"), + ("[name] starts with", "missing STRING at ''"), + ("[name] contains AND [age] > 1", "missing STRING at 'AND'"), + ("[age] greater than or 5", "missing EQUAL at '5'"), + ("[age] = -", "missing DIGITS at ''"), + ("[age] = 5.", "missing DIGITS at ''"), + ("[age] = 5. AND [age] > 1", "missing DIGITS at 'AND'"), + ("([age] = 5.)", "missing DIGITS at ')'"), + ] { + assert_eq!(parse_err(query).message, expected, "{query}"); + } + } + + #[test] + fn a_deletable_token_is_reported_as_extraneous() { + // Deletion is tried before insertion, so the doubled sign is blamed + // rather than a digit being called missing. + for (query, expected) in [ + ("[age] = --5", "extraneous input '-' expecting DIGITS"), + ("[age] = -.5", "extraneous input '.' expecting DIGITS"), + ( + "[age] = .5", + "extraneous input '.' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}", + ), + ( + "[age] > = 5", + "extraneous input '=' expecting {STRING, TRUE, FALSE, SIGN, DIGITS}", + ), + ( + "AND [age] > 1", + "extraneous input 'AND' expecting {FIELD, NOT, '('}", + ), + ("[age] > 1 1", "extraneous input '1' expecting "), + ( + "[age] is blank blank", + "extraneous input 'blank' expecting ", + ), + ] { + assert_eq!(parse_err(query).message, expected, "{query}"); + } + } + + #[test] + fn an_undeletable_unomittable_token_is_a_mismatch() { + for (query, expected) in [ + ("[name] starts 5", "mismatched input '5' expecting WITH"), + ("[name] ends 5", "mismatched input '5' expecting WITH"), + ("[name] starts", "mismatched input '' expecting WITH"), + ("[name] ends", "mismatched input '' expecting WITH"), + ( + "[name] starts AND [age] > 1", + "mismatched input 'AND' expecting WITH", + ), + ("[name] is not 5", "mismatched input '5' expecting BLANK"), + ("[name] contains 5", "mismatched input '5' expecting STRING"), + ( + "[name] contains true", + "mismatched input 'true' expecting STRING", + ), + ("[age] = 5.true", "mismatched input 'true' expecting DIGITS"), + ( + "[age] greater than or", + "mismatched input '' expecting EQUAL", + ), + ("[age] = 5 = 5", "mismatched input '=' expecting "), + ("[age] > 1 1 1", "mismatched input '1' expecting "), + ] { + assert_eq!(parse_err(query).message, expected, "{query}"); + } + } + + // --- number handling --------------------------------------------------- + + #[test] + fn signs_are_applied() { + for query in ["[age] = -5", "[age] = - 5"] { + assert_eq!( + parse(query).filters[0].condition.as_ref().unwrap().args[0], + json!(-5), + "{query}" + ); + } + for query in ["[age] = +5", "[age] = + 5"] { + assert_eq!( + parse(query).filters[0].condition.as_ref().unwrap().args[0], + json!(5), + "{query}" + ); + } + } + + #[test] + fn trailing_fraction_zeros_are_normalized() { + assert_eq!( + parse("[age] = 1.500").filters[0] + .condition + .as_ref() + .unwrap() + .args[0], + json!(1.5) + ); + // `1.000` is integral, and the reference emits `1` for it — JavaScript + // has one number type, so a written fraction that happens to be whole + // still serializes without a decimal point. + let group = parse("[age] = 1.000"); + let one = &group.filters[0].condition.as_ref().unwrap().args[0]; + assert_eq!(one, &json!(1)); + assert_eq!(one.to_string(), "1"); + } + + #[test] + fn an_unrepresentable_number_becomes_null() { + // `parseFloat` overflows to Infinity in the reference, which serializes + // as `null`; the same arg must appear here. + let query = format!("[age] = {}", "9".repeat(400)); + assert_eq!( + parse(&query).filters[0].condition.as_ref().unwrap().args[0], + serde_json::Value::Null + ); + } + + // --- parse_query and ParseResult --------------------------------------- + + #[test] + fn parse_result_is_filters_or_errors_never_both() { + let cols = vec![ColumnDef::new("age", crate::column::ColumnType::Number)]; + let ok = parse_query("[age] > 1", &cols); + assert!(ok.filters.is_some()); + assert!(ok.errors.is_none()); + assert!(!ok.has_errors()); + assert!(ok.errors().is_empty()); + + let bad = parse_query("[nope] > 1", &cols); + assert!(bad.filters.is_none()); + assert!(bad.has_errors()); + assert_eq!(bad.errors().len(), 1); + } + + #[test] + fn parse_result_json_omits_the_absent_side() { + let ok = ParseResult::ok(FilterGroup::default()); + let value = serde_json::to_value(&ok).unwrap(); + assert!(value.get("errors").is_none()); + assert!(value.get("filters").is_some()); + + let bad = ParseResult::failed(vec![ParseError::new("x", 0, 0)]); + let value = serde_json::to_value(&bad).unwrap(); + assert!(value.get("filters").is_none()); + assert_eq!(value["errors"][0]["severity"], json!("error")); + } + + #[test] + fn syntax_errors_preempt_semantic_ones() { + // `[nope]` does not exist *and* the trailing `1` is extraneous; the + // syntax error is the one reported. + let cols = vec![ColumnDef::new("age", crate::column::ColumnType::Number)]; + let result = parse_query("[nope] > 1 1", &cols); + assert_eq!( + result.errors()[0].message, + "extraneous input '1' expecting " + ); + } + + #[test] + fn error_spans_are_byte_offsets() { + // The documented divergence, pinned as a difference rather than an + // agreement: `é` is two bytes but one UTF-16 code unit, so the reference + // reports 11 for this input where the byte offset is 12. The span must + // still slice the input correctly, which is what makes it the useful one. + let query = "[café] > 1 1"; + let e = parse_err(query); + assert_eq!((e.start, e.end), (12, 13)); + assert_eq!(&query[e.start..e.end], "1"); + // The same query in pure ASCII agrees with the reference exactly, and the + // gap between the two is precisely the extra byte `é` costs. + let ascii = parse_err("[cafe] > 1 1"); + assert_eq!((ascii.start, ascii.end), (11, 12)); + assert_eq!(e.start - ascii.start, query.len() - "[cafe] > 1 1".len()); + } + + #[test] + fn severity_serializes_lowercase() { + assert_eq!( + serde_json::to_value(ErrorSeverity::Error).unwrap(), + json!("error") + ); + assert_eq!( + serde_json::to_value(ErrorSeverity::Warning).unwrap(), + json!("warning") + ); + assert_eq!(ErrorSeverity::default(), ErrorSeverity::Error); + } +} diff --git a/rusty-filter/src/print.rs b/rusty-filter/src/print.rs new file mode 100644 index 0000000..308d51e --- /dev/null +++ b/rusty-filter/src/print.rs @@ -0,0 +1,526 @@ +//! Printing an AST back to a query string. +//! +//! A port of `ASTPrinter` in canonical mode only — the non-canonical option +//! combinations have no caller in this repo, so the options struct is not +//! reproduced. +//! +//! The canonical spelling is deliberately asymmetric: `=` prints as the word +//! `equals` while the four orderings stay symbolic. That is what the bundle +//! does, and changing it would make the two sides disagree on cache keys. + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup, LogicalOp}; + +/// Print `group` as a canonical query string. +/// +/// An empty group prints as the empty string. Output is stable for a given AST +/// but is not always re-parseable to the same *text*: `[name] not contains "a"` +/// prints as `NOT ([name] contains "a")`, which parses back to the same AST. +pub fn to_query_string(group: &FilterGroup) -> String { + if group.filters.is_empty() { + return String::new(); + } + print_group(group, false) +} + +/// The canonical string of `group`, for use as a cache key. +/// +/// This is [`to_query_string`] under a name that says what it is for. Two ASTs +/// that filter identically produce the same key, and the frontend's +/// `formatQuery` produces that same key for the same AST. +pub fn canonical_key(group: &FilterGroup) -> String { + to_query_string(group) +} + +fn print_group(group: &FilterGroup, nested: bool) -> String { + if group.filters.is_empty() { + return String::new(); + } + let separator = match group.op { + LogicalOp::And => " AND ", + LogicalOp::Or => " OR ", + }; + let joined = group + .filters + .iter() + .map(print_filter) + .collect::>() + .join(separator); + if nested { + format!("({joined})") + } else { + joined + } +} + +fn print_filter(filter: &Filter) -> String { + // A filter with neither side prints as nothing, and an empty nested group + // prints as nothing too — which is how the reference ends up emitting a + // dangling `AND` for `[age] equals 1 AND `. The parser never + // builds either shape; a hand-built AST can, and reproducing the oddity + // keeps the two sides' cache keys in step. + let mut result = if let Some(condition) = &filter.condition { + print_condition(condition) + } else if let Some(group) = &filter.group { + print_group(group, true) + } else { + String::new() + }; + if filter.is_negated() { + // Parenthesize unless the text is already a group, so `NOT` always + // takes a bracketed operand — an empty operand included, as `NOT ()`. + if !result.starts_with('(') { + result = format!("({result})"); + } + result = format!("NOT {result}"); + } + result +} + +fn print_condition(condition: &Condition) -> String { + let column = print_column(&condition.column); + match condition.function { + FilterFunction::IsBlank => return format!("{column} IS BLANK"), + FilterFunction::IsNotBlank => return format!("{column} IS NOT BLANK"), + _ => {} + } + let operator = print_operator(condition.function); + // Only the first argument is printed; the AST never carries more. + match condition.args.first() { + Some(arg) => format!("{column} {operator} {}", print_value(arg)), + None => format!("{column} {operator}"), + } +} + +/// Bracket the column name, unless it already opens with a bracket — a name +/// stored as `[age]` would otherwise become `[[age]]`. +fn print_column(name: &str) -> String { + if name.starts_with('[') { + name.to_string() + } else { + format!("[{name}]") + } +} + +fn print_operator(function: FilterFunction) -> &'static str { + match function { + FilterFunction::Equals => "equals", + FilterFunction::GreaterThan => ">", + FilterFunction::LessThan => "<", + FilterFunction::GreaterThanOrEqual => ">=", + FilterFunction::LessThanOrEqual => "<=", + FilterFunction::Contains => "contains", + FilterFunction::StartsWith => "STARTS WITH", + FilterFunction::EndsWith => "ENDS WITH", + FilterFunction::IsBlank => "IS BLANK", + FilterFunction::IsNotBlank => "IS NOT BLANK", + } +} + +fn print_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(text) => { + // The backslash is escaped first, then the quote, so a literal + // backslash does not swallow the quote's escape. + let escaped = text.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") + } + serde_json::Value::Bool(true) => "true".to_string(), + serde_json::Value::Bool(false) => "false".to_string(), + serde_json::Value::Number(number) => match number.as_f64() { + Some(value) => js_number_to_string(value), + // Unreachable for finite JSON numbers, but a large integer literal + // is better printed than dropped. + None => number.to_string(), + }, + serde_json::Value::Null => "null".to_string(), + // The reference falls through to `String(value)` here. + other => js_string_of(other), + } +} + +/// Format a number the way JavaScript's `Number.prototype.toString` does, so +/// that a key built here matches one built in the browser. +/// +/// Rust's own `Display` agrees for everyday values but never switches to +/// exponential notation, where JavaScript does so above 10^21 and below 10^-6: +/// `1e21` and `1e-7` print as `1e+21` and `1e-7`, not as long digit strings. +fn js_number_to_string(value: f64) -> String { + if value == 0.0 { + // Covers -0.0, which JavaScript prints as `0`. + return "0".to_string(); + } + if value.is_nan() { + return "NaN".to_string(); + } + if value.is_infinite() { + return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_string(); + } + if value < 0.0 { + return format!("-{}", js_number_to_string(-value)); + } + + // `{:e}` gives the shortest round-tripping digits plus a decimal exponent, + // which is exactly the `s` and `n` of the spec: `s` is the digit string and + // `n` is one past the exponent. + let exp_form = format!("{value:e}"); + let (mantissa, exponent) = exp_form.split_once('e').expect("{:e} always emits an e"); + let digits: String = mantissa.chars().filter(|c| *c != '.').collect(); + let k = digits.len() as i32; + let n = exponent + .parse::() + .expect("{:e} emits a decimal exponent") + + 1; + + if k <= n && n <= 21 { + // Whole number: the digits then n - k zeros. + let mut out = digits; + out.extend(std::iter::repeat_n('0', (n - k) as usize)); + out + } else if 0 < n && n <= 21 { + // A decimal point inside the digits. + format!("{}.{}", &digits[..n as usize], &digits[n as usize..]) + } else if -6 < n && n <= 0 { + // A leading zero, then -n zeros, then the digits. + let zeros = "0".repeat((-n) as usize); + format!("0.{zeros}{digits}") + } else { + // Exponential, with an explicit `+` on a positive exponent. + let sign = if n >= 1 { "+" } else { "-" }; + let magnitude = (n - 1).abs(); + if k == 1 { + format!("{digits}e{sign}{magnitude}") + } else { + format!("{}.{}e{sign}{magnitude}", &digits[..1], &digits[1..]) + } + } +} + +/// JavaScript's `String(value)` for the JSON values `printValue` does not +/// special-case: an array joins its elements with commas, treating null and +/// undefined as empty, and any other object is `[object Object]`. +fn js_string_of(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Array(items) => items + .iter() + .map(|item| match item { + serde_json::Value::Null => String::new(), + serde_json::Value::String(text) => text.clone(), + serde_json::Value::Bool(true) => "true".to_string(), + serde_json::Value::Bool(false) => "false".to_string(), + serde_json::Value::Number(number) => match number.as_f64() { + Some(value) => js_number_to_string(value), + None => number.to_string(), + }, + nested => js_string_of(nested), + }) + .collect::>() + .join(","), + serde_json::Value::Object(_) => "[object Object]".to_string(), + serde_json::Value::Null => "null".to_string(), + serde_json::Value::String(text) => text.clone(), + serde_json::Value::Bool(true) => "true".to_string(), + serde_json::Value::Bool(false) => "false".to_string(), + serde_json::Value::Number(number) => match number.as_f64() { + Some(value) => js_number_to_string(value), + None => number.to_string(), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse_query_unchecked; + use serde_json::json; + + fn printed(query: &str) -> String { + to_query_string(&parse_query_unchecked(query).expect(query)) + } + + #[test] + fn the_measured_round_trips_are_reproduced() { + assert_eq!(printed("[age]>100"), "[age] > 100"); + assert_eq!(printed("[age] greater than 100"), "[age] > 100"); + assert_eq!( + printed("[name] not contains \"a\""), + "NOT ([name] contains \"a\")" + ); + assert_eq!(printed("NOT [age] > 1"), "NOT ([age] > 1)"); + assert_eq!( + printed("[age] > 1 AND [name] = \"a\" OR [active] = true"), + "([age] > 1 AND [name] equals \"a\") OR [active] equals true" + ); + assert_eq!(printed("[name] is not blank"), "[name] IS NOT BLANK"); + } + + #[test] + fn every_operator_has_a_canonical_spelling() { + let group = FilterGroup::and( + [ + FilterFunction::GreaterThan, + FilterFunction::LessThan, + FilterFunction::GreaterThanOrEqual, + FilterFunction::LessThanOrEqual, + FilterFunction::Equals, + FilterFunction::Contains, + FilterFunction::StartsWith, + FilterFunction::EndsWith, + ] + .into_iter() + .map(|f| Filter::condition("c", f, vec![json!("v")])) + .collect(), + ); + assert_eq!( + to_query_string(&group), + "[c] > \"v\" AND [c] < \"v\" AND [c] >= \"v\" AND [c] <= \"v\" \ + AND [c] equals \"v\" AND [c] contains \"v\" \ + AND [c] STARTS WITH \"v\" AND [c] ENDS WITH \"v\"" + ); + } + + #[test] + fn blank_operators_print_as_keywords() { + let group = FilterGroup::and(vec![ + Filter::condition("c", FilterFunction::IsBlank, vec![]), + Filter::condition("c", FilterFunction::IsNotBlank, vec![]), + ]); + assert_eq!(to_query_string(&group), "[c] IS BLANK AND [c] IS NOT BLANK"); + // Stray args on a blank operator are ignored, as in the reference. + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::IsBlank, + vec![json!("x")], + )]); + assert_eq!(to_query_string(&group), "[c] IS BLANK"); + } + + #[test] + fn an_empty_group_prints_as_nothing() { + assert_eq!(to_query_string(&FilterGroup::default()), ""); + assert_eq!(to_query_string(&FilterGroup::or(vec![])), ""); + assert_eq!(printed(""), ""); + assert_eq!(printed(" "), ""); + } + + #[test] + fn a_false_negate_key_prints_nothing_extra() { + let group = FilterGroup::and(vec![Filter::condition( + "name", + FilterFunction::Contains, + vec![json!("a")], + ) + .negated(false)]); + assert_eq!(to_query_string(&group), "[name] contains \"a\""); + // Which is exactly what the parser produces for the un-negated form. + assert_eq!(printed("[name] contains \"a\""), "[name] contains \"a\""); + } + + #[test] + fn nested_groups_are_parenthesized() { + assert_eq!( + printed("([age] > 1 AND [b] = 2) OR [c] = 3"), + "([age] > 1 AND [b] equals 2) OR [c] equals 3" + ); + assert_eq!( + printed("(([age] > 1))"), + "(([age] > 1))", + "parens are not collapsed" + ); + assert_eq!( + printed("NOT ([age] > 1 AND [b] = 2)"), + "NOT ([age] > 1 AND [b] equals 2)" + ); + } + + #[test] + fn negation_of_an_empty_group_still_brackets() { + let group = FilterGroup::and(vec![ + Filter::from_group(FilterGroup::default()).negated(true) + ]); + assert_eq!(to_query_string(&group), "NOT ()"); + let group = FilterGroup::and(vec![Filter::default().negated(true)]); + assert_eq!(to_query_string(&group), "NOT ()"); + } + + #[test] + fn a_filter_with_neither_side_prints_as_nothing() { + let group = FilterGroup::and(vec![Filter::default()]); + assert_eq!(to_query_string(&group), ""); + // And an empty arm leaves a dangling operator, faithfully. + let group = FilterGroup::and(vec![ + Filter::condition("age", FilterFunction::Equals, vec![json!(1)]), + Filter::from_group(FilterGroup::default()), + ]); + assert_eq!(to_query_string(&group), "[age] equals 1 AND "); + } + + #[test] + fn a_condition_with_no_args_prints_the_bare_operator() { + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![], + )]); + assert_eq!(to_query_string(&group), "[age] >"); + } + + #[test] + fn columns_are_bracketed_at_most_once() { + let group = FilterGroup::and(vec![ + Filter::condition("age", FilterFunction::Equals, vec![json!(1)]), + Filter::condition("[age]", FilterFunction::Equals, vec![json!(1)]), + Filter::condition("a]b", FilterFunction::Equals, vec![json!(1)]), + ]); + assert_eq!( + to_query_string(&group), + "[age] equals 1 AND [age] equals 1 AND [a]b] equals 1" + ); + } + + #[test] + fn strings_escape_the_backslash_before_the_quote() { + let cases = [ + ("a", "\"a\""), + ("", "\"\""), + ("\"", "\"\\\"\""), + ("\\", "\"\\\\\""), + ("\\\"", "\"\\\\\\\"\""), + ("\\\\", "\"\\\\\\\\\""), + // Whitespace and single quotes pass through untouched. + ("\t", "\"\t\""), + ("\n", "\"\n\""), + ("'", "\"'\""), + ]; + for (input, expected) in cases { + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!(input)], + )]); + assert_eq!( + to_query_string(&group), + format!("[c] equals {expected}"), + "{input:?}" + ); + } + } + + #[test] + fn a_quoted_string_survives_the_round_trip() { + let query = "[c] = \"a\\\"b\\\\c\""; + let group = parse_query_unchecked(query).unwrap(); + let printed = to_query_string(&group); + assert_eq!(printed, "[c] equals \"a\\\"b\\\\c\""); + assert_eq!(parse_query_unchecked(&printed).unwrap(), group); + } + + #[test] + fn booleans_and_null_print_as_literals() { + let group = FilterGroup::and(vec![ + Filter::condition("b", FilterFunction::Equals, vec![json!(true)]), + Filter::condition("b", FilterFunction::Equals, vec![json!(false)]), + Filter::condition("b", FilterFunction::Equals, vec![serde_json::Value::Null]), + ]); + assert_eq!( + to_query_string(&group), + "[b] equals true AND [b] equals false AND [b] equals null" + ); + } + + #[test] + fn numbers_print_the_way_javascript_prints_them() { + // Every expectation measured against `String(n)` in Node. + let cases = [ + (1.0, "1"), + (-1.0, "-1"), + (0.0, "0"), + (-0.0, "0"), + (1.5, "1.5"), + (0.1, "0.1"), + (100.0, "100"), + (1e21, "1e+21"), + (1e-7, "1e-7"), + (-1e-7, "-1e-7"), + (1e20, "100000000000000000000"), + (1e-6, "0.000001"), + (1.2345e2, "123.45"), + (1.5e22, "1.5e+22"), + (1.25e-8, "1.25e-8"), + (f64::INFINITY, "Infinity"), + (f64::NEG_INFINITY, "-Infinity"), + (f64::NAN, "NaN"), + ]; + for (value, expected) in cases { + assert_eq!(js_number_to_string(value), expected, "{value}"); + } + } + + #[test] + fn numbers_from_the_parser_print_back_identically() { + for query in [ + "[n] = 0", + "[n] = 1", + "[n] = -1", + "[n] = 1.5", + "[n] = 0.1", + "[n] = 100", + "[n] = 100000000000000000000", + ] { + assert_eq!(printed(query), query.replace('=', "equals"), "{query}"); + } + // A literal too large for an f64 becomes `Infinity`, which the + // reference stores as null; printing says `null` rather than inventing + // a value. + let huge = format!("[n] = 1{}", "0".repeat(400)); + assert_eq!(printed(&huge), "[n] equals null"); + } + + #[test] + fn composite_args_fall_back_to_the_javascript_string_conversion() { + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!([1, 2])], + )]); + assert_eq!(to_query_string(&group), "[c] equals 1,2"); + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!({"a": 1})], + )]); + assert_eq!(to_query_string(&group), "[c] equals [object Object]"); + // Null inside an array is the empty string, not `null`. + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!([null, "a", [2, 3]])], + )]); + assert_eq!(to_query_string(&group), "[c] equals ,a,2,3"); + } + + #[test] + fn only_the_first_arg_is_printed() { + let group = FilterGroup::and(vec![Filter::condition( + "c", + FilterFunction::Equals, + vec![json!("a"), json!("b")], + )]); + assert_eq!(to_query_string(&group), "[c] equals \"a\""); + } + + #[test] + fn canonical_key_is_the_query_string() { + let group = parse_query_unchecked("[age]>100").unwrap(); + assert_eq!(canonical_key(&group), to_query_string(&group)); + assert_eq!(canonical_key(&FilterGroup::default()), ""); + } + + #[test] + fn synonyms_collapse_onto_one_key() { + // The point of a canonical key: differently spelled equivalents agree. + let keys = ["[age]>100", "[age] > 100", "[age] greater than 100"] + .map(|q| canonical_key(&parse_query_unchecked(q).unwrap())); + assert_eq!(keys[0], keys[1]); + assert_eq!(keys[1], keys[2]); + } +} diff --git a/rusty-filter/src/validate.rs b/rusty-filter/src/validate.rs new file mode 100644 index 0000000..6a23927 --- /dev/null +++ b/rusty-filter/src/validate.rs @@ -0,0 +1,603 @@ +//! Semantic validation against a column schema. +//! +//! A port of `SemanticValidator` and `TypeChecker`, message strings included, so +//! that a Rust-side error reads exactly like the browser's. +//! +//! Every error carries `start: 0` and `end: 0`: the reference validator has no +//! position information to report, and the frontend's own `parseQuery` passes +//! those zeros straight through. That is faithful behaviour, not a defect. + +use crate::ast::{Condition, Filter, FilterFunction, FilterGroup}; +use crate::column::{find_column, ColumnDef, ColumnType}; +use crate::parser::ParseError; + +/// Validate `group` against `columns`, returning every error found. +/// +/// Validation walks nested groups. A condition naming an unknown column reports +/// only that error and is not checked further, matching the reference's early +/// return. +pub fn validate_filter_group(group: &FilterGroup, columns: &[ColumnDef]) -> Vec { + let mut errors = Vec::new(); + walk_group(group, columns, &mut errors); + errors +} + +fn walk_group(group: &FilterGroup, columns: &[ColumnDef], errors: &mut Vec) { + for filter in &group.filters { + walk_filter(filter, columns, errors); + } +} + +fn walk_filter(filter: &Filter, columns: &[ColumnDef], errors: &mut Vec) { + // `condition` wins when both are somehow set, as the `if / else if` does. + if let Some(condition) = &filter.condition { + validate_condition(condition, columns, errors); + } else if let Some(group) = &filter.group { + walk_group(group, columns, errors); + } +} + +fn validate_condition(condition: &Condition, columns: &[ColumnDef], errors: &mut Vec) { + let Some(column) = find_column(columns, &condition.column) else { + errors.push(error(format!( + "Column '{}' does not exist", + condition.column + ))); + return; + }; + let column_type = column.column_type; + + if condition.function.is_blank_operator() { + if !is_blank_operator_compatible(column_type) { + errors.push(incompatible(condition.function, column_type)); + } + // Blank operators take no arguments, so there is nothing left to check. + return; + } + + if !is_operator_compatible(column_type, condition.function) { + errors.push(incompatible(condition.function, column_type)); + return; + } + + if condition.args.is_empty() { + errors.push(error(format!( + "Operator '{}' requires a value", + condition.function.display_name() + ))); + return; + } + + for arg in &condition.args { + if let Err(message) = validate_value_type(arg, column) { + errors.push(error(message)); + } + } +} + +fn error(message: String) -> ParseError { + ParseError::new(message, 0, 0) +} + +fn incompatible(function: FilterFunction, column_type: ColumnType) -> ParseError { + error(format!( + "Operator '{}' is not compatible with type '{}'", + function.display_name(), + column_type.as_str() + )) +} + +/// Whether `function` may be applied to `column_type`. +/// +/// `String` allows equals, contains, startsWith and endsWith; `Number` and +/// `Date` allow equals plus the four orderings; `Boolean` and `Enum` allow +/// equals only. +pub fn is_operator_compatible(column_type: ColumnType, function: FilterFunction) -> bool { + use FilterFunction::*; + match column_type { + ColumnType::String => matches!(function, Equals | Contains | StartsWith | EndsWith), + ColumnType::Number | ColumnType::Date => matches!( + function, + Equals | GreaterThan | LessThan | GreaterThanOrEqual | LessThanOrEqual + ), + ColumnType::Boolean | ColumnType::Enum => function == Equals, + } +} + +/// Whether the blank operators may be applied to `column_type`. They are +/// allowed on `String`, `Date` and `Enum` only. +pub fn is_blank_operator_compatible(column_type: ColumnType) -> bool { + matches!( + column_type, + ColumnType::String | ColumnType::Date | ColumnType::Enum + ) +} + +/// Check one argument against a column, returning the reference's error message +/// on mismatch. +pub fn validate_value_type(value: &serde_json::Value, column: &ColumnDef) -> Result<(), String> { + let name = &column.name; + match column.column_type { + ColumnType::String => { + if !value.is_string() { + return Err(format!( + "Expected string for column '{name}', got {}", + js_typeof(value) + )); + } + } + ColumnType::Number => { + if !value.is_number() { + return Err(format!( + "Expected number for column '{name}', got {}", + js_typeof(value) + )); + } + } + ColumnType::Boolean => { + if !value.is_boolean() { + return Err(format!( + "Expected boolean for column '{name}', got {}", + js_typeof(value) + )); + } + } + ColumnType::Date => { + let Some(text) = value.as_str() else { + return Err(format!( + "Expected date string for column '{name}', got {}", + js_typeof(value) + )); + }; + if !is_iso_date_shape(text) { + return Err(format!( + "Invalid date format for column '{name}'. Expected YYYY-MM-DD or ISO datetime" + )); + } + } + ColumnType::Enum => { + if !value.is_string() { + return Err(format!( + "Expected string for enum column '{name}', got {}", + js_typeof(value) + )); + } + // The reference skips value-set validation: `ColumnDef` carries no + // enum members to check against. + } + } + Ok(()) +} + +/// The name JavaScript's `typeof` would give a JSON value, so that the "got X" +/// half of a message matches the reference. +fn js_typeof(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::String(_) => "string", + serde_json::Value::Number(_) => "number", + serde_json::Value::Bool(_) => "boolean", + // `typeof null` is `"object"` in JavaScript, and an unrepresentable + // number arrives here as JSON null. + serde_json::Value::Null => "object", + serde_json::Value::Array(_) | serde_json::Value::Object(_) => "object", + } +} + +/// Match `/^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d{3})?Z?)?$/`. +/// +/// This is a **shape** check, not a calendar check: `2024-13-99` passes here +/// because it passes in the bundle. Hand-rolled to avoid a regex dependency for +/// one pattern. +fn is_iso_date_shape(text: &str) -> bool { + let bytes = text.as_bytes(); + // YYYY-MM-DD is exactly ten characters. + if bytes.len() < 10 { + return false; + } + let digits = |range: std::ops::Range| bytes[range].iter().all(u8::is_ascii_digit); + if !digits(0..4) || bytes[4] != b'-' || !digits(5..7) || bytes[7] != b'-' || !digits(8..10) { + return false; + } + if bytes.len() == 10 { + return true; + } + // The optional time part: THH:MM:SS + if bytes[10] != b'T' || bytes.len() < 19 { + return false; + } + if !digits(11..13) || bytes[13] != b':' || !digits(14..16) || bytes[16] != b':' { + return false; + } + if !digits(17..19) { + return false; + } + let mut i = 19; + // The optional milliseconds: exactly three digits after the dot. + if bytes.get(i) == Some(&b'.') { + if bytes.len() < i + 4 || !digits(i + 1..i + 4) { + return false; + } + i += 4; + } + // The optional trailing Z. + if bytes.get(i) == Some(&b'Z') { + i += 1; + } + i == bytes.len() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::parse_query; + use serde_json::json; + + fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("created", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ] + } + + /// The single message a query produces, or a panic if it produced none. + fn message(query: &str) -> String { + let result = parse_query(query, &columns()); + let errors = result.errors(); + assert_eq!(errors.len(), 1, "{query}: {errors:?}"); + errors[0].message.clone() + } + + fn is_valid(query: &str) -> bool { + !parse_query(query, &columns()).has_errors() + } + + #[test] + fn unknown_column_is_reported() { + assert_eq!(message("[nope] = 1"), "Column 'nope' does not exist"); + assert_eq!(message("[ s ] = \"x\""), "Column ' s ' does not exist"); + // A non-existent column stops validation of that condition, so the + // operator/type mismatch that would otherwise follow is not also + // reported. The operand has to be well-typed for the *grammar* first: + // `[nope] contains 1` fails to parse, so validation never runs on it. + assert_eq!( + message("[nope] contains \"x\""), + "Column 'nope' does not exist" + ); + assert_eq!(message("[nope] is blank"), "Column 'nope' does not exist"); + assert_eq!( + message("[nope] contains 1"), + "mismatched input '1' expecting STRING" + ); + } + + #[test] + fn semantic_errors_carry_zero_positions() { + let result = parse_query("[nope] = 1", &columns()); + let e = &result.errors()[0]; + assert_eq!((e.start, e.end), (0, 0)); + assert_eq!(e.severity, crate::parser::ErrorSeverity::Error); + } + + #[test] + fn incompatible_operators_on_string() { + assert_eq!( + message("[name] > 1"), + "Operator 'greater than' is not compatible with type 'string'" + ); + assert_eq!( + message("[name] < 1"), + "Operator 'less than' is not compatible with type 'string'" + ); + assert_eq!( + message("[name] >= 1"), + "Operator 'greater than or equal' is not compatible with type 'string'" + ); + assert_eq!( + message("[name] <= 1"), + "Operator 'less than or equal' is not compatible with type 'string'" + ); + } + + #[test] + fn compatible_operators_on_string() { + assert!(is_valid("[name] = \"a\"")); + assert!(is_valid("[name] contains \"a\"")); + assert!(is_valid("[name] starts with \"a\"")); + assert!(is_valid("[name] ends with \"a\"")); + } + + #[test] + fn incompatible_operators_on_number() { + assert_eq!( + message("[age] contains \"x\""), + "Operator 'contains' is not compatible with type 'number'" + ); + assert_eq!( + message("[age] starts with \"x\""), + "Operator 'starts with' is not compatible with type 'number'" + ); + assert_eq!( + message("[age] ends with \"x\""), + "Operator 'ends with' is not compatible with type 'number'" + ); + } + + #[test] + fn compatible_operators_on_number_and_date() { + for op in ["=", ">", "<", ">=", "<="] { + assert!(is_valid(&format!("[age] {op} 1")), "age {op}"); + assert!( + is_valid(&format!("[created] {op} \"2024-01-01\"")), + "created {op}" + ); + } + } + + #[test] + fn incompatible_operators_on_boolean() { + assert_eq!( + message("[active] > 1"), + "Operator 'greater than' is not compatible with type 'boolean'" + ); + assert_eq!( + message("[active] contains \"x\""), + "Operator 'contains' is not compatible with type 'boolean'" + ); + assert!(is_valid("[active] = true")); + assert!(is_valid("[active] = false")); + } + + #[test] + fn incompatible_operators_on_enum() { + assert_eq!( + message("[kind] contains \"a\""), + "Operator 'contains' is not compatible with type 'enum'" + ); + assert_eq!( + message("[kind] > 1"), + "Operator 'greater than' is not compatible with type 'enum'" + ); + assert!(is_valid("[kind] = \"a\"")); + } + + #[test] + fn incompatible_operators_on_date() { + assert_eq!( + message("[created] contains \"a\""), + "Operator 'contains' is not compatible with type 'date'" + ); + } + + #[test] + fn blank_operators_are_string_date_and_enum_only() { + assert!(is_valid("[name] is blank")); + assert!(is_valid("[name] is not blank")); + assert!(is_valid("[created] is blank")); + assert!(is_valid("[kind] is blank")); + assert_eq!( + message("[active] is blank"), + "Operator 'is blank' is not compatible with type 'boolean'" + ); + assert_eq!( + message("[active] is not blank"), + "Operator 'is not blank' is not compatible with type 'boolean'" + ); + assert_eq!( + message("[age] is blank"), + "Operator 'is blank' is not compatible with type 'number'" + ); + } + + #[test] + fn wrong_value_types_are_reported() { + assert_eq!( + message("[age] > \"x\""), + "Expected number for column 'age', got string" + ); + assert_eq!( + message("[age] = true"), + "Expected number for column 'age', got boolean" + ); + assert_eq!( + message("[name] = 1"), + "Expected string for column 'name', got number" + ); + assert_eq!( + message("[name] = true"), + "Expected string for column 'name', got boolean" + ); + assert_eq!( + message("[active] = \"x\""), + "Expected boolean for column 'active', got string" + ); + assert_eq!( + message("[active] = 1"), + "Expected boolean for column 'active', got number" + ); + assert_eq!( + message("[created] > 1"), + "Expected date string for column 'created', got number" + ); + assert_eq!( + message("[kind] = 1"), + "Expected string for enum column 'kind', got number" + ); + } + + #[test] + fn date_check_is_shape_only() { + // Nonsense month and day, accepted because the reference accepts them. + assert!(is_valid("[created] > \"2024-13-99\"")); + assert!(is_valid("[created] > \"0000-00-00\"")); + } + + #[test] + fn date_shapes_that_are_accepted() { + for value in [ + "2024-01-01", + "2024-01-01T10:20:30", + "2024-01-01T10:20:30Z", + "2024-01-01T10:20:30.123", + "2024-01-01T10:20:30.123Z", + ] { + assert!(is_iso_date_shape(value), "{value}"); + assert!(is_valid(&format!("[created] > \"{value}\"")), "{value}"); + } + } + + #[test] + fn date_shapes_that_are_rejected() { + for value in [ + "2024-1-1", + "2024-01-01T10:20:30.12", + "2024-01-01T10:20:30.1234", + "2024-01-01T10:20", + "2024-01-01 10:20:30", + "2024-01-01T", + "2024-01-01Z", + "24-01-01", + "", + "x", + "2024-01-01T10:20:30.123ZZ", + "2024-01-01T10:20:30z", + ] { + assert!(!is_iso_date_shape(value), "{value}"); + } + assert_eq!( + message("[created] > \"2024-1-1\""), + "Invalid date format for column 'created'. Expected YYYY-MM-DD or ISO datetime" + ); + } + + #[test] + fn a_comparison_with_no_value_is_reported() { + // The grammar cannot produce this, but a hand-built AST can. + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::GreaterThan, + vec![], + )]); + let errors = validate_filter_group(&group, &columns()); + assert_eq!(errors.len(), 1); + assert_eq!( + errors[0].message, + "Operator 'greater than' requires a value" + ); + } + + #[test] + fn nested_groups_are_validated() { + let result = parse_query("([nope] = 1 OR [age] = 2) AND [name] = 3", &columns()); + let messages: Vec<_> = result.errors().iter().map(|e| e.message.clone()).collect(); + assert_eq!( + messages, + vec![ + "Column 'nope' does not exist".to_string(), + "Expected string for column 'name', got number".to_string(), + ] + ); + } + + #[test] + fn an_empty_group_validates_clean() { + assert!(validate_filter_group(&FilterGroup::default(), &columns()).is_empty()); + } + + #[test] + fn a_filter_with_neither_side_is_ignored() { + let group = FilterGroup::and(vec![Filter::default()]); + assert!(validate_filter_group(&group, &columns()).is_empty()); + } + + #[test] + fn all_args_are_checked() { + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::Equals, + vec![json!(1), json!("x"), json!(true)], + )]); + let errors = validate_filter_group(&group, &columns()); + assert_eq!(errors.len(), 2); + assert_eq!( + errors[0].message, + "Expected number for column 'age', got string" + ); + assert_eq!( + errors[1].message, + "Expected number for column 'age', got boolean" + ); + } + + #[test] + fn a_null_arg_reports_the_javascript_typeof() { + let group = FilterGroup::and(vec![Filter::condition( + "age", + FilterFunction::Equals, + vec![serde_json::Value::Null], + )]); + let errors = validate_filter_group(&group, &columns()); + assert_eq!( + errors[0].message, + "Expected number for column 'age', got object" + ); + } + + #[test] + fn compatibility_helpers_cover_every_pair() { + use FilterFunction::*; + let all = [ + Equals, + GreaterThan, + LessThan, + GreaterThanOrEqual, + LessThanOrEqual, + Contains, + StartsWith, + EndsWith, + ]; + let types = [ + ColumnType::String, + ColumnType::Number, + ColumnType::Boolean, + ColumnType::Date, + ColumnType::Enum, + ]; + // Equality is the one operator every type accepts. + for t in types { + assert!(is_operator_compatible(t, Equals), "{t:?}"); + } + // And every pair has a definite answer, so no combination is unhandled. + for t in types { + for f in all { + let _ = is_operator_compatible(t, f); + } + } + assert!(is_blank_operator_compatible(ColumnType::String)); + assert!(is_blank_operator_compatible(ColumnType::Date)); + assert!(is_blank_operator_compatible(ColumnType::Enum)); + assert!(!is_blank_operator_compatible(ColumnType::Number)); + assert!(!is_blank_operator_compatible(ColumnType::Boolean)); + } + + #[test] + fn normalized_backend_types_validate_like_their_targets() { + let cols = vec![ + ColumnDef::normalized("nAge", "INT32"), + ColumnDef::normalized("tName", "TEXT"), + ColumnDef::normalized("weird", "GUID"), + ]; + assert!(!parse_query("[nAge] > 1", &cols).has_errors()); + assert!(!parse_query("[tName] contains \"a\"", &cols).has_errors()); + // An unknown backend type falls back to string, so `contains` is fine + // and `>` is not. + assert!(!parse_query("[weird] contains \"a\"", &cols).has_errors()); + let result = parse_query("[weird] > 1", &cols); + assert_eq!( + result.errors()[0].message, + "Operator 'greater than' is not compatible with type 'string'" + ); + } +} From 7149f0e545fbbe915758a9f5c860939ade2ba600 Mon Sep 17 00:00:00 2001 From: Mikael Rinne <40919111+rorychatt@users.noreply.github.com> Date: Sun, 2 Aug 2026 12:12:35 +0200 Subject: [PATCH 2/3] [00123] Add differential and round-trip tests for rusty-filter Two integration suites, both measured against the shipped filter-query-editor 2.2.0 bundle rather than written from the grammar. frontend_ast_compat.rs pins 23 expectations that are the verbatim JSON the bundle emits, so serde key *presence* is significant: a comparison omits `negate`, a text operation emits `negate: false`, and existence operations carry empty args. The module doc records the capture recipe (import dist/index.js by absolute file:// URL, call parseQuery). round_trip.rs asserts what a cache key needs: 39 of the 47 corpus queries are exactly AST-stable across print/parse, printing is a fixed point from the second pass on, equivalent spellings collapse onto one key, and 14 differently-meaning queries produce 14 distinct keys. The 8 remaining queries are negated leaves. Printing lifts the negation onto a `NOT (...)` group, so re-parsing moves `negate` from the condition filter to a wrapping group filter. The reference bundle was probed at each of them and reshapes identically, so the plan's claim of universal AST stability does not hold for either implementation. Rather than drop the assertion, NOT_AST_STABLE lists the 8 exactly and the test fails if a ninth query joins them or a listed one starts passing; semantic equivalence over 7 rows is asserted for all 47. Cross-check totals, all with a mutation probe confirming the comparator discriminates: 42 valid queries with 0 AST or formatQuery mismatches, 46 error cases with 0 message or span mismatches, 29 filters x 6 rows with 0 evaluateFilter or countMatches mismatches. 195 tests pass; cargo fmt and clippy --all-targets -D warnings clean. Co-Authored-By: Claude Opus 5 --- rusty-filter/tests/frontend_ast_compat.rs | 189 ++++++++++++++++ rusty-filter/tests/round_trip.rs | 263 ++++++++++++++++++++++ 2 files changed, 452 insertions(+) create mode 100644 rusty-filter/tests/frontend_ast_compat.rs create mode 100644 rusty-filter/tests/round_trip.rs diff --git a/rusty-filter/tests/frontend_ast_compat.rs b/rusty-filter/tests/frontend_ast_compat.rs new file mode 100644 index 0000000..fddda04 --- /dev/null +++ b/rusty-filter/tests/frontend_ast_compat.rs @@ -0,0 +1,189 @@ +//! The AST this crate serializes must be byte-compatible with the JSON the +//! browser's `filter-query-editor` produces, because both halves of a filter can +//! be authored on either side and a `DataTable` query has to mean one thing. +//! +//! Every expected value below is the **verbatim output of the shipped bundle**, +//! captured by importing `dist/index.js` from +//! `src/frontend/node_modules/filter-query-editor` (version 2.2.0) and calling +//! `parseQuery(query, columns)` with the column schema in [`columns`]. Nothing +//! here is hand-written from the grammar: the point of this file is to catch +//! drift away from the reference, and a hand-written expectation would drift with +//! the implementation instead of pinning it. +//! +//! Reproducing the capture: +//! +//! ```js +//! const B = '/src/frontend/node_modules/filter-query-editor/dist/index.js'; +//! const { parseQuery } = await import(B); +//! console.log(JSON.stringify(parseQuery('[age] > 1', cols))); +//! ``` +//! +//! The one thing that is *not* asserted verbatim is a multi-error cascade: on a +//! syntax error this crate reports the first error and stops, which the crate +//! docs call out as a deliberate divergence. Single-error inputs are compared in +//! full, spans included. + +use rusty_filter::{parse_query, ColumnDef, ColumnType}; + +/// The schema the probe ran with. `n` and `x` exist only to keep the captured +/// mixed `AND`/`OR` query short. +fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("when", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ColumnDef::new("n", ColumnType::String), + ColumnDef::new("x", ColumnType::Boolean), + ] +} + +/// Assert that this crate's `ParseResult` serializes to exactly `expected`. +/// +/// Comparing `serde_json::Value` rather than strings keeps key *presence* +/// significant while leaving key order free, which is what matters: the +/// reference omits `negate` on comparisons and emits `negate: false` on text +/// operations, and a round trip through the browser must not change that. +#[track_caller] +fn assert_bundle_json(query: &str, expected: &str) { + let expected: serde_json::Value = + serde_json::from_str(expected).expect("the captured JSON parses"); + let actual = serde_json::to_value(parse_query(query, &columns())).expect("the AST serializes"); + assert_eq!(actual, expected, "query: {query}"); +} + +#[test] +fn a_comparison_omits_the_negate_key() { + assert_bundle_json( + "[age] > 1", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}}]}}"#, + ); + assert_bundle_json( + "[age] = 100", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[100]}}]}}"#, + ); +} + +#[test] +fn a_text_operation_emits_negate_false() { + assert_bundle_json( + r#"[name] contains "ab""#, + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"contains","args":["ab"]},"negate":false}]}}"#, + ); + assert_bundle_json( + r#"[name] starts with "a""#, + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"startsWith","args":["a"]},"negate":false}]}}"#, + ); +} + +#[test] +fn negated_forms_carry_negate_true() { + // `!=` is `equals` with `negate: true` — there is no `notEquals` function. + assert_bundle_json( + "[age] != 5", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[5]},"negate":true}]}}"#, + ); + assert_bundle_json( + r#"[name] not contains "ab""#, + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"contains","args":["ab"]},"negate":true}]}}"#, + ); + // `NOT` lands on the condition filter itself, not on a wrapper. + assert_bundle_json( + "NOT [age] > 1", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]},"negate":true}]}}"#, + ); + // And it *toggles*, so a double `NOT` is `false` rather than absent. + assert_bundle_json( + "not not [age] > 1", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]},"negate":false}]}}"#, + ); +} + +#[test] +fn an_existence_operation_has_empty_args_and_no_negate() { + assert_bundle_json( + "[name] is blank", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"isBlank","args":[]}}]}}"#, + ); + assert_bundle_json( + "[name] is not blank", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"name","function":"isNotBlank","args":[]}}]}}"#, + ); +} + +#[test] +fn mixed_and_or_nests_the_and_arm_as_a_group() { + // `OR` is the root, and the `AND` arm is spliced in as `{group: ...}`. + assert_bundle_json( + r#"[age] > 1 AND [n] = "a" OR [x] = true"#, + r#"{"filters":{"op":"OR","filters":[{"group":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}},{"condition":{"column":"n","function":"equals","args":["a"]}}]}},{"condition":{"column":"x","function":"equals","args":[true]}}]}}"#, + ); +} + +#[test] +fn parentheses_are_never_collapsed() { + assert_bundle_json( + "(([age] > 1))", + r#"{"filters":{"op":"AND","filters":[{"group":{"op":"AND","filters":[{"group":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}}]}}]}}]}}"#, + ); +} + +#[test] +fn an_empty_query_is_an_empty_and_group() { + let empty = r#"{"filters":{"op":"AND","filters":[]}}"#; + assert_bundle_json("", empty); + assert_bundle_json(" ", empty); +} + +#[test] +fn numbers_serialize_as_javascript_writes_them() { + // JavaScript has one number type, so `007` is `7` and an integral fraction + // loses its decimal point. Whitespace inside a number is skipped. + assert_bundle_json( + "[age] = 007", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[7]}}]}}"#, + ); + assert_bundle_json( + "[age] = 1 . 5", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[1.5]}}]}}"#, + ); + assert_bundle_json( + "[age] = 1.000", + r#"{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"equals","args":[1]}}]}}"#, + ); +} + +#[test] +fn a_semantic_error_matches_the_reference_wording_and_zero_span() { + assert_bundle_json( + "[nope] = 1", + r#"{"errors":[{"message":"Column 'nope' does not exist","start":0,"end":0,"severity":"error"}]}"#, + ); + assert_bundle_json( + r#"[age] contains "x""#, + r#"{"errors":[{"message":"Operator 'contains' is not compatible with type 'number'","start":0,"end":0,"severity":"error"}]}"#, + ); + assert_bundle_json( + r#"[age] > "x""#, + r#"{"errors":[{"message":"Expected number for column 'age', got string","start":0,"end":0,"severity":"error"}]}"#, + ); +} + +#[test] +fn a_syntax_error_matches_the_reference_message_and_span() { + // Single-error inputs only: the ANTLR cascade is out of scope, so an input + // the reference reports three errors for would not compare here. + assert_bundle_json( + "[age] > 1 1", + r#"{"errors":[{"message":"extraneous input '1' expecting ","start":10,"end":11,"severity":"error"}]}"#, + ); + assert_bundle_json( + "[age] = 1e5", + r#"{"errors":[{"message":"token recognition error at: 'e5'","start":9,"end":10,"severity":"error"}]}"#, + ); + assert_bundle_json( + "([age] > 1", + r#"{"errors":[{"message":"missing ')' at ''","start":10,"end":10,"severity":"error"}]}"#, + ); +} diff --git a/rusty-filter/tests/round_trip.rs b/rusty-filter/tests/round_trip.rs new file mode 100644 index 0000000..5d1db97 --- /dev/null +++ b/rusty-filter/tests/round_trip.rs @@ -0,0 +1,263 @@ +//! Printing an AST and parsing the result back. +//! +//! This is what makes [`canonical_key`] usable as a cache key: two subscribers +//! who wrote the same filter differently must land on one string, and that string +//! must still mean what they wrote. +//! +//! Assertions here are at the **AST** level, never on string equality. +//! `canonical_key` is not string-idempotent on the first pass — `[name] not +//! contains "a"` prints as `NOT ([name] contains "a")`, which is a different +//! source text — so a string round trip would fail on correct output. +//! +//! # Negated leaves are not AST-stable, in either implementation +//! +//! AST stability does not hold universally, and the reference does not have it +//! either. A negated *leaf* prints as `NOT (...)`, and re-parsing that reads the +//! parentheses as a group, so the negation moves from the condition filter onto a +//! wrapping group filter: +//! +//! ```text +//! [age] != 100 +//! -> {op:AND, filters:[{condition:{...equals...}, negate:true}]} +//! -> prints as "NOT ([age] equals 100)" +//! -> {op:AND, filters:[{group:{op:AND, filters:[{condition:{...}}]}, negate:true}]} +//! ``` +//! +//! Measured against `filter-query-editor` 2.2.0 over the 47 queries in +//! [`QUERIES`]: 39 are AST-stable and the 8 in [`NOT_AST_STABLE`] are not, with +//! the bundle producing the very same reshaped AST this crate does. So the +//! divergence is shared, not introduced here — and the tests below pin both +//! halves: exact AST equality for the stable set, and semantic equivalence plus +//! string idempotence for the rest. + +use rusty_filter::{ + canonical_key, evaluate, parse_query, parse_query_unchecked, to_query_string, ColumnDef, + ColumnType, FilterGroup, +}; + +fn columns() -> Vec { + vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("active", ColumnType::Boolean), + ColumnDef::new("when", ColumnType::Date), + ColumnDef::new("kind", ColumnType::Enum), + ] +} + +/// Rows covering the truth table of every operator the queries below use. +fn rows() -> Vec { + vec![ + serde_json::json!({"age": 100, "name": "ab", "active": true, "when": "2024-01-01", "kind": "a"}), + serde_json::json!({"age": 101, "name": "abc", "active": false, "when": "2025-06-30T00:00:00Z", "kind": ""}), + serde_json::json!({"age": 1, "name": "zab", "active": true, "when": "2023-01-01", "kind": "b"}), + serde_json::json!({"age": -5, "name": "", "active": false, "when": null, "kind": null}), + serde_json::json!({"age": 1.5, "name": "a\"b", "active": true}), + serde_json::json!({"age": 7, "name": "a\\b", "active": false, "when": "2024-01-01"}), + serde_json::json!({}), + ] +} + +/// Every valid query shape the grammar admits, one per feature. +const QUERIES: &[&str] = &[ + // comparisons, symbolic and spelled out + "[age] > 100", + "[age] >= 100", + "[age] < 100", + "[age] <= 100", + "[age] = 100", + "[age] == 100", + "[age] != 100", + "[age] equals 100", + "[age] not equals 100", + "[age] not equal 100", + "[age] greater than 100", + "[age] greater than or equal 100", + "[age] less than 100", + "[age] less than or equal 100", + // text operations, negated and not + r#"[name] contains "ab""#, + r#"[name] not contains "ab""#, + r#"[name] starts with "ab""#, + r#"[name] not starts with "ab""#, + r#"[name] ends with "ab""#, + r#"[name] not ends with "ab""#, + // existence + "[name] is blank", + "[name] is not blank", + "[when] is blank", + "[kind] is not blank", + // literals + "[active] = true", + "[active] = false", + "[age] = -5", + "[age] = 1.5", + "[age] = 007", + r#"[when] = "2024-01-01""#, + r#"[when] > "2024-01-01T10:20:30.123Z""#, + // strings needing escapes + r#"[name] = "a\"b""#, + r#"[name] = "a\\b""#, + // logical structure + r#"[age] > 1 AND [name] = "a""#, + r#"[age] > 1 OR [name] = "a""#, + r#"[age] > 1 AND [name] = "a" AND [active] = true"#, + r#"[age] > 1 OR [name] = "a" OR [active] = true"#, + r#"[age] > 1 AND [name] = "a" OR [active] = true"#, + r#"[age] > 1 OR [name] = "a" AND [active] = true"#, + // negation and parentheses + "NOT [age] > 1", + "not not [age] > 1", + "(([age] > 1))", + r#"NOT ([age] > 1 AND [name] = "a")"#, + r#"([age] > 1 AND [name] = "a") OR [active] = true"#, + r#"([age] > 1 OR [name] = "a") AND [active] = true"#, + // empty + "", + " ", +]; + +/// The queries whose AST is reshaped by a print/parse round trip, because their +/// negation sits on a leaf and printing lifts it onto a `NOT (...)` group. +/// +/// This list is the bundle's, not this crate's: each entry was confirmed to be +/// AST-unstable in `filter-query-editor` 2.2.0 as well. Anything not listed here +/// must be exactly stable, which [`printing_then_reparsing_preserves_the_ast`] +/// enforces — so an implementation change that quietly reshapes one more query +/// fails the suite rather than growing this list. +const NOT_AST_STABLE: &[&str] = &[ + "[age] != 100", + "[age] not equals 100", + "[age] not equal 100", + r#"[name] not contains "ab""#, + r#"[name] not starts with "ab""#, + r#"[name] not ends with "ab""#, + "NOT [age] > 1", + "not not [age] > 1", +]; + +fn parse(query: &str) -> FilterGroup { + parse_query(query, &columns()) + .filters + .unwrap_or_else(|| panic!("{query} should parse")) +} + +#[test] +fn printing_then_reparsing_preserves_the_ast() { + let mut stable = 0; + for query in QUERIES { + let first = parse(query); + let printed = to_query_string(&first); + let second = parse(&printed); + if NOT_AST_STABLE.contains(query) { + assert_ne!( + second, first, + "{query} is listed as AST-unstable but round-tripped cleanly — \ + drop it from NOT_AST_STABLE" + ); + } else { + assert_eq!(second, first, "{query} printed as {printed:?}"); + stable += 1; + } + } + // Pins the split measured against the bundle: 39 of 47 stable. + assert_eq!(stable, QUERIES.len() - NOT_AST_STABLE.len()); + assert_eq!(stable, 39); +} + +#[test] +fn a_reshaped_round_trip_still_means_the_same_thing() { + // The negated-leaf reshaping moves `negate` onto a wrapping group, which is a + // different AST but the same predicate. That is the property that actually + // matters for a cache key, so it is asserted for *every* query rather than + // only the unstable ones. + for query in QUERIES { + let first = parse(query); + let second = parse(&to_query_string(&first)); + let cols = columns(); + for row in rows() { + assert_eq!( + evaluate(&first, &row, &cols), + evaluate(&second, &row, &cols), + "{query} disagreed on row {row}" + ); + } + } +} + +#[test] +fn a_second_round_trip_is_a_fixed_point() { + // Printing is idempotent from the second pass onwards even where the AST is + // reshaped on the first, so a cache key derived from a cache key is the same + // key. Without this, `canonical_key` would not be canonical. + for query in QUERIES { + let once = to_query_string(&parse(query)); + let twice = to_query_string(&parse(&once)); + assert_eq!(twice, once, "{query}"); + let thrice = to_query_string(&parse(&twice)); + assert_eq!(thrice, once, "{query}"); + } +} + +#[test] +fn equivalent_spellings_share_one_cache_key() { + for [a, b] in [ + ["[age] > 100", "[age] greater than 100"], + ["[age] >= 100", "[age] greater than or equal 100"], + ["[age] < 100", "[age] less than 100"], + ["[age] <= 100", "[age] less than or equal 100"], + ["[age] = 100", "[age] == 100"], + ["[age] != 100", "[age] not equals 100"], + ["[age] != 100", "[age] not equal 100"], + ["[age] = 7", "[age] = 007"], + ["[age] = 1.5", "[age] = 1 . 5"], + ["[age] > 1", "not not [age] > 1"], + // Keyword case is folded, so only the keywords differ here. + ["[age] > 1", "[age] GREATER THAN 1"], + [r#"[name] contains "a""#, r#"[name] CONTAINS "a""#], + ] { + let key_a = canonical_key(&parse_query_unchecked(a).expect("valid")); + let key_b = canonical_key(&parse_query_unchecked(b).expect("valid")); + assert_eq!(key_a, key_b, "{a:?} vs {b:?}"); + } +} + +#[test] +fn a_column_name_keeps_its_case_in_the_key() { + // Keyword case is folded but a column name is not, so two differently-cased + // column names are two different cache keys — as they must be, since the + // column lookup at evaluation time is case-sensitive too. + let lower = canonical_key(&parse_query_unchecked("[name] = \"a\"").expect("valid")); + let upper = canonical_key(&parse_query_unchecked("[NAME] = \"a\"").expect("valid")); + assert_ne!(lower, upper); +} + +#[test] +fn differently_meaning_queries_get_different_keys() { + // The other half of a cache key's contract: collapsing two *different* + // filters onto one key would serve one subscriber the other's rows. + let queries = [ + "[age] > 1", + "[age] >= 1", + "[age] > 2", + "[age] < 1", + "NOT [age] > 1", + r#"[age] > 1 AND [name] = "a""#, + r#"[age] > 1 OR [name] = "a""#, + r#"[name] contains "a""#, + r#"[name] not contains "a""#, + r#"[name] starts with "a""#, + r#"[name] ends with "a""#, + "[name] is blank", + "[name] is not blank", + "", + ]; + let keys: Vec = queries + .iter() + .map(|q| canonical_key(&parse_query_unchecked(q).expect("valid"))) + .collect(); + let mut unique = keys.clone(); + unique.sort(); + unique.dedup(); + assert_eq!(unique.len(), keys.len(), "keys collided: {keys:?}"); +} From fda2fdba637f42ec68a150dd3d03062ea58a940c Mon Sep 17 00:00:00 2001 From: Mikael Rinne <40919111+rorychatt@users.noreply.github.com> Date: Sun, 2 Aug 2026 12:22:29 +0200 Subject: [PATCH 3/3] [00123] Wire rusty-filter into DataTable, the query cache and the prelude MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive only: nothing existing changes behaviour, and `to_json`, `register_events` and every field are untouched. `rusty/Cargo.toml` gains the path dependency. `data_table.rs` gains `From for ColumnType` collapsing eight cell types onto the grammar's five, `filter_columns()` offering only `filterable && !hidden` columns, and `apply_filter(query)` returning a table of the matching rows or the parse and validation errors. Excluding non-filterable columns is what `DataTableFilterOption.tsx` does before handing columns to the browser's editor, so naming one gets the same `Column 'x' does not exist` error on both sides instead of working on one. `query_cache.rs` gains one associated function, `filtered_key`, keying on the canonical spelling so `[age] > 1` and `[age] greater than 1` share a cache entry. Associated rather than a method because it reads no cache state, which keeps it clear of the module's rule that the cache lock is never held across an `.await`. The SWR state machine is untouched. The prelude re-exports the flat list plus the crate itself. Docs: a new 02_concepts/07_filters.md covering operators, type compatibility, literals, precedence, evaluation and canonical keys; a "Server-side filtering" section on the DataTable page; and a Filters table in the prelude reference. Three grammar quirks the plan did not mention were measured and documented rather than guessed: bare `equal` is rejected though `not equal` is accepted, the `or equal` tail is singular so `greater than or equals` fails, and an unknown string escape keeps its backslash (`\t` is two characters, not a tab) — that last one re-probed against the bundle, which agrees. Both doc code samples were compiled and run, not just written. The widget's "DataTableConnection is not ported" comment now names what apply_filter does while saying plainly that the rendered filter box still posts to a gRPC DataTableService that does not exist, so nothing here implies typing in the browser reaches Rust. Gate: build, test (663 passed, 0 failed), clippy --workspace --all-targets -D warnings and fmt --check all exit 0. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 1 + rusty-docs/docs/02_concepts/07_filters.md | 179 ++++++++++++++++ rusty-docs/docs/03_widgets/11_data_table.md | 44 +++- .../docs/04_api_reference/01_prelude.md | 28 +++ rusty/Cargo.toml | 1 + rusty/src/core/query_cache.rs | 99 +++++++++ rusty/src/lib.rs | 8 + rusty/src/widgets/data_table.rs | 196 +++++++++++++++++- 8 files changed, 554 insertions(+), 2 deletions(-) create mode 100644 rusty-docs/docs/02_concepts/07_filters.md diff --git a/Cargo.lock b/Cargo.lock index d75c3cf..5bace4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -774,6 +774,7 @@ dependencies = [ "axum", "bytes", "futures", + "rusty-filter", "rusty-macros", "serde", "serde_json", diff --git a/rusty-docs/docs/02_concepts/07_filters.md b/rusty-docs/docs/02_concepts/07_filters.md new file mode 100644 index 0000000..867847c --- /dev/null +++ b/rusty-docs/docs/02_concepts/07_filters.md @@ -0,0 +1,179 @@ +## Filters + +`rusty_filter` parses the filter query language Ivy's filter editor uses, into an +AST you can validate against a column schema, evaluate over rows, and print back +to a canonical string. It is what makes a filter typed in one place mean the same +thing in the other. + +```rust +use rusty::prelude::*; + +let columns = vec![ + ColumnDef::new("age", ColumnType::Number), + ColumnDef::new("name", ColumnType::String), +]; +let result = parse_query("[age] > 30 AND [name] starts with \"A\"", &columns); +let filter = result.filters.expect("the query is valid"); + +let rows = vec![ + serde_json::json!({"age": 41, "name": "Ada"}), + serde_json::json!({"age": 41, "name": "Bob"}), + serde_json::json!({"age": 12, "name": "Ann"}), +]; +assert_eq!(retain_matching(&filter, rows, &columns).len(), 1); +``` + +`parse_query` returns a `ParseResult` carrying **either** `filters` or `errors`, +never both. Empty and whitespace-only input is valid and matches every row, so a +cleared filter box needs no special case. + +### Column references + +A column is named in square brackets: `[age]`. Whatever is between the brackets +is the name, verbatim — spaces and unicode included, so `[first name]` works and +`[ age ]` is a different column from `[age]`. Names are matched case-sensitively; +keywords are not. `[]` is a syntax error, but `[ ]` is the column named `" "`. + +### Operators + +Every operator has a symbolic and a spelled-out form, and the spelled-out forms +are case-insensitive. Both parse to one `FilterFunction`: + +| Function | Spellings | +|----------|-----------| +| `Equals` | `=`, `==`, `equals` | +| `Equals` negated | `!=`, `not equals`, `not equal` | +| `GreaterThan` | `>`, `greater than` | +| `GreaterThanOrEqual` | `>=`, `greater than or equal` | +| `LessThan` | `<`, `less than` | +| `LessThanOrEqual` | `<=`, `less than or equal` | +| `Contains` | `contains`, `not contains` | +| `StartsWith` | `starts with`, `not starts with` | +| `EndsWith` | `ends with`, `not ends with` | +| `IsBlank` | `is blank` | +| `IsNotBlank` | `is not blank` | + +There is no `NotEquals` function: `!=` is `Equals` with `negate: true`. The same +goes for the negated text operators. + +Two spellings that look like they should work and do not: bare `equal` is a +syntax error where `equals` is fine, though `not equal` is accepted; and the +`or equal` tail of `>=` and `<=` is singular, so `greater than or equals` is +rejected. + +### Which operators a column type allows + +`ColumnType` has five variants, and validation rejects a mismatch with +`Operator 'x' is not compatible with type 'y'`: + +| Column type | Allowed | +|-------------|---------| +| `String` | `equals`, `contains`, `starts with`, `ends with`, `is blank`, `is not blank` | +| `Number` | `equals`, `>`, `>=`, `<`, `<=` | +| `Date` | `equals`, `>`, `>=`, `<`, `<=`, `is blank`, `is not blank` | +| `Boolean` | `equals` | +| `Enum` | `equals`, `is blank`, `is not blank` | + +`ColumnType::normalize` maps a backend type name onto one of the five: +`INT32`, `INT64`, `DOUBLE`, `DECIMAL` and `NUMBER` become `Number`; `TEXT`, +`STRING` and `ICON` become `String`; `DATE` and `DATETIME` become `Date`; and any +name it does not recognize becomes `String`. + +### Literals + +| Kind | Examples | Notes | +|------|----------|-------| +| Number | `42`, `-5`, `1.5`, `007` | One JavaScript number type, so `007` is `7` and `1.000` is `1`. `1e5` is a lexer error. | +| String | `"Ada"`, `"a\"b"`, `"a\\b"` | Double quotes only. `\"`, `\'` and `\\` are the only escapes; anything else keeps its backslash, so `\t` stays two characters rather than becoming a tab. A raw newline inside a string is an error, but a raw tab is allowed. | +| Boolean | `true`, `false` | Case-insensitive. | +| Date | `"2024-01-01"`, `"2024-01-01T10:20:30.123Z"` | A quoted string on a `Date` column. Validation is shape-only, so `"2024-13-99"` passes it and then matches nothing. `"2024-1-1"` is rejected — pad to two digits. | + +The value type has to match the column, or validation reports +`Expected number for column 'age', got string`. + +### Combining and precedence + +`AND` binds tighter than `OR`, and both are case-insensitive. Parentheses +override precedence and are **never collapsed**: `(([age] > 1))` keeps both +group levels in the AST, because the frontend renders those groups. + +`NOT` negates the filter that follows and toggles rather than accumulates, so +`not not [age] > 1` is the same filter as `[age] > 1`. + +```rust +"[age] > 1 AND [name] = \"a\" OR [active] = true" +// parses as: ([age] > 1 AND [name] = "a") OR [active] = true +``` + +The AST reflects that: the root is an `OR` group whose first arm is a nested +`AND` group. Same-operator chains stay flat — a three-arm `AND` is one group +with three filters, not two nested ones. + +### Evaluating + +| Function | Use | +|----------|-----| +| `evaluate(&filter, &row, &columns)` | Does one row match? | +| `retain_matching(&filter, rows, &columns)` | Keep the matching rows, in order. | +| `count_matches(&filter, &rows, &columns)` | Count without allocating. | + +Evaluation is strict about types: `equals` on a `String` column does not match +the number `1` against `"1"`. `contains`, `starts with` and `ends with` are +case-sensitive. A `null` or absent column value fails every operator except +`is blank`. An empty `AND` group matches everything — and so does an empty `OR` +group, which is the reference's behaviour rather than the `false` you might +expect. + +### Canonical strings and cache keys + +`to_query_string` prints an AST back to a query, and `canonical_key` is that +same function under the name that says what it is for: two equivalent filters +produce one string, so they can share one cache entry. + +```rust +use rusty::core::QueryService; + +let filter = parse_query_unchecked("[age] greater than 30").unwrap(); +assert_eq!(QueryService::filtered_key("people", Some(&filter)), "people?[age] > 30"); +``` + +The canonical spelling is not simply the input echoed back. It is symbolic for +the orderings and spelled out for equality — `[age] greater than 30` prints as +`[age] > 30` while `[age] = 1` prints as `[age] equals 1` — because that is what +the frontend's `formatQuery` does, and the two sides have to agree. + +Printing is idempotent from the second pass onwards, but the first pass can +reshape the AST: a negated leaf prints as `NOT (...)`, and re-parsing reads those +parentheses as a group, moving the negation onto a wrapping group filter. The +filter still means the same thing, and the reference does the same, so compare +canonical strings rather than ASTs when you need to know whether two filters +match. + +### Compatibility with the frontend editor + +The AST serializes to exactly the JSON `filter-query-editor` produces — +`{"filters":{"op":"AND","filters":[{"condition":{"column":"age","function":"greaterThan","args":[1]}}]}}` +— key presence included, which is finer than it sounds. A comparison omits the +`negate` key altogether; a text operation emits `negate: false`. Behaviour was +matched against version 2.2.0 over roughly a hundred inputs, and +`rusty-filter/tests/frontend_ast_compat.rs` pins the shape against captured +bundle output. + +**The browser's filter box does not reach this crate.** The rendered `DataTable` +sends filter queries to a gRPC `DataTableService` that Rusty does not implement, +so typing in the filter box goes nowhere. What works today is Rust-side +filtering you call yourself, via [`DataTable::apply_filter`](../03_widgets/11_data_table.md) +or the functions above. This crate is what a future `DataTableService` would +parse with. + +### Not implemented + +- The gRPC `DataTableService` (`Query`, `ParseFilter`, `Distinct`). +- `parseInvalidQuery` and its LLM-based query repair. +- Sorting, aggregations, pagination and column selection — the rest of Ivy's + `DataTableQuery`. +- Multiple errors for one input. ANTLR recovers from a syntax error and can + report a cascade; this parser reports the first and stops. Semantic errors are + reported in full, one per offending condition. +- Error spans are byte offsets, not UTF-16 code units. They agree for ASCII and + part company once a multi-byte character appears before the error. diff --git a/rusty-docs/docs/03_widgets/11_data_table.md b/rusty-docs/docs/03_widgets/11_data_table.md index 1ba3d1c..3cee191 100644 --- a/rusty-docs/docs/03_widgets/11_data_table.md +++ b/rusty-docs/docs/03_widgets/11_data_table.md @@ -101,9 +101,51 @@ DataTable::new(vec![ .into() ``` +### Server-side filtering + +`apply_filter` runs a query in the same grammar Ivy's filter editor uses and +returns a table holding only the matching rows. See +[Filters](../02_concepts/07_filters.md) for the grammar itself. + +```rust +let table = DataTable::new(vec![ + DataTableColumn::new("name", "Name", ColType::Text), + DataTableColumn::new("age", "Age", ColType::Number), +]) +.rows(vec![ + json!({"name": "Alice", "age": 30}), + json!({"name": "Bob", "age": 25}), +]); + +let filtered = table.apply_filter("[age] > 28").expect("valid query"); +assert_eq!(filtered.rows.len(), 1); +``` + +| Method | Returns | Description | +|--------|---------|-------------| +| `.filter_columns()` | `Vec` | The columns a query may name, as the grammar sees them | +| `.apply_filter(q)` | `Result>` | Keep the rows matching `q`; `Err` carries the parse and validation errors | + +A column is offered to `filter_columns` only when it is `filterable` and not +`hidden`, which is the same test the frontend applies before handing columns to +its editor. Naming an excluded column therefore gets +`Column 'x' does not exist` on both sides rather than working on one of them. +Each `ColType` maps onto one of the grammar's five types: `Number` to `number`, +`Boolean` to `boolean`, `Date` and `DateTime` to `date`, and `Text`, `Icon`, +`Labels` and `Link` to `string`. + +An empty or whitespace-only query is valid and keeps every row, so clearing a +filter needs no special case. + ### Limitations Rows travel inline in the widget JSON, exactly as `Table` does: the whole set is serialized on every build. Ivy's `DataTableConnection` — the server-side query pipeline that pages, sorts and filters large datasets on demand — is not ported, -so paginate or pre-filter in your own code before handing rows to the widget. +so paginate in your own code before handing rows to the widget. + +Filtering is available, but only when **your code** calls `apply_filter`. The +filter box in the rendered table sends its query to a gRPC `DataTableService` +that Rusty does not implement, so typing there still goes nowhere; the same +applies to sorting and to the search box. Treat `allow_filtering`, +`allow_sorting` and `show_search` as frontend chrome until that service exists. diff --git a/rusty-docs/docs/04_api_reference/01_prelude.md b/rusty-docs/docs/04_api_reference/01_prelude.md index ea0704d..fbca29f 100644 --- a/rusty-docs/docs/04_api_reference/01_prelude.md +++ b/rusty-docs/docs/04_api_reference/01_prelude.md @@ -93,3 +93,31 @@ Note the argument order: wherever a hook takes dependencies, they come **before* | Type | Description | |------|-------------| | `RustyServer` | WebSocket server — `new(port, factory).serve().await` | + +### Filters + +Re-exported from `rusty_filter`. See [Filters](../02_concepts/07_filters.md). + +| Item | Description | +|------|-------------| +| `parse_query(q, &columns)` | Parse and validate, returning `ParseResult` | +| `parse_query_unchecked(q)` | Parse without a column schema | +| `evaluate(&filter, &row, &columns)` | Does one row match? | +| `retain_matching(&filter, rows, &columns)` | Keep the matching rows | +| `count_matches(&filter, &rows, &columns)` | Count the matching rows | +| `validate_filter_group(&filter, &columns)` | Semantic errors only | +| `to_query_string(&filter)` | Print an AST back to a query | +| `canonical_key(&filter)` | The same string, named for cache-key use | +| `ColumnDef` | One filterable column — `new(name, ColumnType)` | +| `ColumnType` | `String`, `Number`, `Boolean`, `Date`, `Enum` | +| `FilterGroup` | `op` plus a list of `Filter` | +| `Filter` | A `Condition` or a nested `FilterGroup`, optionally negated | +| `Condition` | `column`, `function`, `args` | +| `FilterFunction` | `Equals`, `GreaterThan`, `Contains`, `IsBlank`, … | +| `LogicalOp` | `And`, `Or` | +| `ParseResult` | `filters` **or** `errors`, never both | +| `ParseError` | `message`, `start`, `end`, `severity` | +| `ErrorSeverity` | `Error`, `Warning` | + +The crate itself is re-exported too, so `rusty_filter::lexer` and the other +modules are reachable without adding a second dependency. diff --git a/rusty/Cargo.toml b/rusty/Cargo.toml index 43f7531..10957fd 100644 --- a/rusty/Cargo.toml +++ b/rusty/Cargo.toml @@ -16,6 +16,7 @@ tracing.workspace = true tower-http.workspace = true bytes.workspace = true rusty-macros = { path = "../rusty-macros" } +rusty-filter = { path = "../rusty-filter" } [dev-dependencies] tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/rusty/src/core/query_cache.rs b/rusty/src/core/query_cache.rs index 8e15b80..e3ca3a8 100644 --- a/rusty/src/core/query_cache.rs +++ b/rusty/src/core/query_cache.rs @@ -685,6 +685,40 @@ impl QueryService { } } + /// Build a cache key from a base key and a parsed filter, so two subscribers + /// with equivalent filters share one entry. + /// + /// `base` comes back unchanged when there is nothing to filter by — `None`, + /// or a group holding no filters, which is what an empty query parses to. + /// Otherwise the filter follows a `?` separator in the canonical spelling + /// [`rusty_filter::canonical_key`] produces, so `[age] > 1` and + /// `[age] greater than 1` are one key rather than two. + /// + /// An associated function, not a method: it reads no cache state, which also + /// keeps it clear of this module's rule that the cache lock is never held + /// across an `.await`. + /// + /// ``` + /// use rusty::core::QueryService; + /// use rusty_filter::parse_query_unchecked; + /// + /// let a = parse_query_unchecked("[age] > 1").unwrap(); + /// let b = parse_query_unchecked("[age] greater than 1").unwrap(); + /// assert_eq!( + /// QueryService::filtered_key("people", Some(&a)), + /// QueryService::filtered_key("people", Some(&b)), + /// ); + /// assert_eq!(QueryService::filtered_key("people", None), "people"); + /// ``` + pub fn filtered_key(base: &str, filter: Option<&rusty_filter::FilterGroup>) -> String { + match filter { + Some(filter) if !filter.filters.is_empty() => { + format!("{base}?{}", rusty_filter::canonical_key(filter)) + } + _ => base.to_string(), + } + } + /// Transition an entry into `Fetching`/`Revalidating` and return the fetcher /// to spawn, or `None` when a fetch is already running. fn begin_fetch(entry: &mut QueryEntry, revalidating: bool) -> Option { @@ -1556,4 +1590,69 @@ mod tests { assert_eq!(service.entry_state("b"), Some(QueryEntryState::Empty)); assert_eq!(service.peek::("a"), None); } + + fn filter(query: &str) -> rusty_filter::FilterGroup { + rusty_filter::parse_query_unchecked(query).expect("valid query") + } + + #[test] + fn test_filtered_key_returns_the_base_when_there_is_no_filter() { + assert_eq!(QueryService::filtered_key("people", None), "people"); + // An empty query parses to an empty group, so a cleared filter box must + // land back on the unfiltered entry instead of creating `people?`. + assert_eq!( + QueryService::filtered_key("people", Some(&filter(""))), + "people" + ); + assert_eq!( + QueryService::filtered_key("people", Some(&rusty_filter::FilterGroup::default())), + "people" + ); + } + + #[test] + fn test_filtered_key_appends_the_canonical_filter() { + // The canonical spelling is the printer's, which is `>` rather than + // `greater than` — so the key does not echo the query as written. + assert_eq!( + QueryService::filtered_key("people", Some(&filter("[age] greater than 1"))), + "people?[age] > 1" + ); + assert_eq!( + QueryService::filtered_key("people", Some(&filter(r#"[name] CONTAINS "a""#))), + r#"people?[name] contains "a""# + ); + } + + #[test] + fn test_equivalent_filters_share_one_key() { + // The point of keying on the canonical spelling: two subscribers who + // wrote the same filter differently must not each get their own fetch. + for [a, b] in [ + ["[age] > 1", "[age] greater than 1"], + ["[age] = 7", "[age] = 007"], + ["[age] >= 1", "[age] GREATER THAN OR EQUAL 1"], + ] { + assert_eq!( + QueryService::filtered_key("people", Some(&filter(a))), + QueryService::filtered_key("people", Some(&filter(b))), + "{a:?} vs {b:?}" + ); + } + } + + #[test] + fn test_different_filters_get_different_keys() { + let keys = [ + QueryService::filtered_key("people", None), + QueryService::filtered_key("people", Some(&filter("[age] > 1"))), + QueryService::filtered_key("people", Some(&filter("[age] > 2"))), + QueryService::filtered_key("people", Some(&filter("[age] >= 1"))), + QueryService::filtered_key("other", Some(&filter("[age] > 1"))), + ]; + let mut unique = keys.to_vec(); + unique.sort(); + unique.dedup(); + assert_eq!(unique.len(), keys.len(), "keys collided: {keys:?}"); + } } diff --git a/rusty/src/lib.rs b/rusty/src/lib.rs index 30a4ff9..a662165 100644 --- a/rusty/src/lib.rs +++ b/rusty/src/lib.rs @@ -25,6 +25,14 @@ pub mod prelude { SubmitHandler, Validator, View, }; pub use crate::widgets::*; + /// The crate itself, so `rusty_filter::lexer` and the other modules are + /// reachable without a second dependency line in `Cargo.toml`. + pub use rusty_filter; + pub use rusty_filter::{ + canonical_key, count_matches, evaluate, parse_query, parse_query_unchecked, + retain_matching, to_query_string, validate_filter_group, ColumnDef, ColumnType, Condition, + ErrorSeverity, Filter, FilterFunction, FilterGroup, LogicalOp, ParseError, ParseResult, + }; } // Re-export the derive macro diff --git a/rusty/src/widgets/data_table.rs b/rusty/src/widgets/data_table.rs index b1ea796..70f6632 100644 --- a/rusty/src/widgets/data_table.rs +++ b/rusty/src/widgets/data_table.rs @@ -1,6 +1,7 @@ use crate::core::event_registry::EventRegistry; use crate::shared::{Align, Color, Icon, Size}; use crate::views::view::{Element, WidgetData}; +use rusty_filter::{ColumnDef, ColumnType, ParseError}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::sync::Arc; @@ -20,6 +21,23 @@ pub enum ColType { Link, } +/// A `DataTable` has eight cell types but the filter grammar's validator knows +/// only five, so several collapse onto one. The mapping is the one +/// [`ColumnType::normalize`] applies to the backend type names, which is what +/// keeps a Rust-side filter agreeing with the browser's: `Icon` normalizes to +/// `String` there, and `Labels` and `Link` are unknown names that fall back to +/// `String`. +impl From for ColumnType { + fn from(col_type: ColType) -> Self { + match col_type { + ColType::Number => ColumnType::Number, + ColType::Boolean => ColumnType::Boolean, + ColType::Date | ColType::DateTime => ColumnType::Date, + ColType::Text | ColType::Icon | ColType::Labels | ColType::Link => ColumnType::String, + } + } +} + /// Sort direction applied to a column. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -277,7 +295,14 @@ pub struct RowActionArgs { /// A typed data grid with per-column formatting, sorting and cell events. /// /// Rows travel inline in the widget JSON, exactly as [`crate::widgets::Table`] does. -/// Ivy's `DataTableConnection` server-side query pipeline is not ported. +/// +/// Ivy's `DataTableConnection` server-side query pipeline is not ported. What is +/// available is filtering: [`DataTable::apply_filter`] runs a query in the +/// grammar the browser's filter editor uses and keeps the matching rows. It has +/// to be called by the app, though — the filter box in the rendered table talks +/// to a gRPC `DataTableService` that Rusty does not implement, so typing there +/// still goes nowhere. Sorting, aggregation and pagination are likewise not +/// ported. #[derive(Clone, Serialize, Deserialize)] pub struct DataTable { #[serde(skip_serializing_if = "Option::is_none")] @@ -354,6 +379,62 @@ impl DataTable { self } + /// The columns a filter query may name, as the filter grammar sees them. + /// + /// A column is offered only when it is `filterable` and not `hidden`, which + /// is the same test `DataTableFilterOption.tsx` applies before handing + /// columns to the browser's editor. Naming an excluded column is therefore a + /// `Column 'x' does not exist` error on both sides rather than a filter that + /// works on one of them. + pub fn filter_columns(&self) -> Vec { + self.columns + .iter() + .filter(|c| c.filterable && !c.hidden) + .map(|c| ColumnDef::new(c.name.clone(), c.col_type.into())) + .collect() + } + /// Keep only the rows matching `query`, which is parsed and validated against + /// [`DataTable::filter_columns`]. + /// + /// An empty or whitespace-only query is valid and matches every row, so a + /// cleared filter box needs no special case. + /// + /// # Errors + /// + /// Returns the parse and validation errors when `query` is not a valid + /// filter. A syntax error is reported alone; semantic errors come one per + /// offending condition. + /// + /// ``` + /// use rusty::widgets::{ColType, DataTable, DataTableColumn}; + /// + /// let table = DataTable::new(vec![ + /// DataTableColumn::new("name", "Name", ColType::Text), + /// DataTableColumn::new("age", "Age", ColType::Number), + /// ]) + /// .rows(vec![ + /// serde_json::json!({"name": "Ada", "age": 41}), + /// serde_json::json!({"name": "Bob", "age": 12}), + /// ]); + /// + /// let filtered = table.apply_filter("[age] > 18").expect("valid query"); + /// assert_eq!(filtered.rows.len(), 1); + /// assert_eq!(filtered.rows[0]["name"], "Ada"); + /// ``` + pub fn apply_filter(mut self, query: &str) -> Result> { + let columns = self.filter_columns(); + let result = rusty_filter::parse_query(query, &columns); + match result.filters { + Some(filter) => { + self.rows = rusty_filter::retain_matching(&filter, self.rows, &columns); + Ok(self) + } + // `filters` and `errors` are never both populated, so the fallback + // here is unreachable rather than a silently empty error list. + None => Err(result.errors.unwrap_or_default()), + } + } + pub fn into_element(self) -> Element { Element::Widget(Box::new(self)) } @@ -649,4 +730,117 @@ mod tests { let el: Element = DataTable::new(vec![]).into(); assert!(matches!(el, Element::Widget(_))); } + + #[test] + fn test_col_type_maps_onto_a_filter_column_type() { + // All eight variants, so adding a ninth to `ColType` fails to compile + // rather than silently filtering as text. + for (col_type, expected) in [ + (ColType::Number, ColumnType::Number), + (ColType::Boolean, ColumnType::Boolean), + (ColType::Date, ColumnType::Date), + (ColType::DateTime, ColumnType::Date), + (ColType::Text, ColumnType::String), + (ColType::Icon, ColumnType::String), + (ColType::Labels, ColumnType::String), + (ColType::Link, ColumnType::String), + ] { + assert_eq!(ColumnType::from(col_type), expected, "{col_type:?}"); + } + } + + #[test] + fn test_filter_columns_excludes_non_filterable_and_hidden() { + let table = DataTable::new(vec![ + DataTableColumn::new("name", "Name", ColType::Text), + DataTableColumn::new("age", "Age", ColType::Number), + DataTableColumn::new("secret", "Secret", ColType::Text).filterable(false), + DataTableColumn::new("internal", "Internal", ColType::Text).hidden(true), + // Excluded by either test alone, so still excluded by both. + DataTableColumn::new("both", "Both", ColType::Text) + .filterable(false) + .hidden(true), + ]); + + let columns = table.filter_columns(); + assert_eq!( + columns, + vec![ + ColumnDef::new("name", ColumnType::String), + ColumnDef::new("age", ColumnType::Number), + ] + ); + } + + fn people() -> DataTable { + DataTable::new(vec![ + DataTableColumn::new("name", "Name", ColType::Text), + DataTableColumn::new("age", "Age", ColType::Number), + DataTableColumn::new("secret", "Secret", ColType::Text).filterable(false), + ]) + .rows(vec![ + json!({"name": "Ada", "age": 41, "secret": "x"}), + json!({"name": "Bob", "age": 12, "secret": "y"}), + json!({"name": "Ann", "age": 30, "secret": "x"}), + ]) + } + + #[test] + fn test_apply_filter_keeps_only_matching_rows() { + let filtered = people() + .apply_filter(r#"[age] > 18 AND [name] starts with "A""#) + .expect("valid query"); + let names: Vec<&str> = filtered + .rows + .iter() + .map(|r| r["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, ["Ada", "Ann"]); + // Everything else about the table survives. + assert_eq!(filtered.columns.len(), 3); + } + + #[test] + fn test_apply_filter_with_an_empty_query_keeps_every_row() { + assert_eq!(people().apply_filter("").expect("valid").rows.len(), 3); + assert_eq!(people().apply_filter(" ").expect("valid").rows.len(), 3); + } + + #[test] + fn test_apply_filter_on_a_non_filterable_column_says_it_does_not_exist() { + // The column is on the table but not in `filter_columns`, so the query + // gets the same error the browser's editor would give for a name it was + // never offered. + let errors = people() + .apply_filter(r#"[secret] = "x""#) + .expect_err("secret is not filterable"); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].message, "Column 'secret' does not exist"); + } + + #[test] + fn test_apply_filter_reports_a_syntax_error() { + let errors = people().apply_filter("[age] > ").expect_err("no operand"); + assert_eq!(errors.len(), 1); + assert!( + errors[0].message.contains(""), + "unexpected message: {}", + errors[0].message + ); + } + + #[test] + fn test_apply_filter_filters_a_datetime_column_by_iso_order() { + // `DateTime` normalizes to the grammar's `date`, so an ISO string + // comparison works rather than falling back to text comparison. + let table = DataTable::new(vec![DataTableColumn::new("at", "At", ColType::DateTime)]) + .rows(vec![ + json!({"at": "2024-06-01T09:00:00Z"}), + json!({"at": "2023-01-01T09:00:00Z"}), + ]) + .apply_filter(r#"[at] > "2024-01-01""#) + .expect("valid query"); + assert_eq!(table.rows.len(), 1); + assert_eq!(table.rows[0]["at"], "2024-06-01T09:00:00Z"); + } }