feat: opt timezone expressions into codegen dispatch

docs/source/user-guide/latest/expressions.md

@@ -234,7 +234,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci
 | Function | Status | Notes |
 | --- | --- | --- |
 | `add_months` | ✅ |  |
-| `convert_timezone` | ✅ |  |
+| `convert_timezone` | ✅ | Routes through the JVM codegen dispatcher by default (handles all timezone forms); the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) |
 | `curdate` | ✅ | Constant-folded to a literal (alias of `current_date`) |
 | `current_date` | ✅ | Constant-folded to a literal before Comet sees the plan |
 | `current_time` | 🔜 | Blocked on Spark 4.1 TIME type support ([#4288](https://github.com/apache/datafusion-comet/issues/4288)) |
@@ -257,7 +257,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci
 | `dayofyear` | ✅ |  |
 | `extract` | ✅ |  |
 | `from_unixtime` | ✅ |  |
-| `from_utc_timestamp` | ✅ | Legacy zone forms fall back (Incompatible) ([details](compatibility/expressions/datetime.md)) |
+| `from_utc_timestamp` | ✅ | Routes through the JVM codegen dispatcher by default (handles all timezone forms); the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) |
 | `hour` | ✅ |  |
 | `last_day` | ✅ |  |
 | `localtimestamp` | ✅ |  |
@@ -289,7 +289,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci
 | `to_timestamp_ltz` | ✅ | Rewrites to `to_timestamp` (`TimestampType`) |
 | `to_timestamp_ntz` | ✅ | Rewrites to `to_timestamp` (`TimestampNTZType`) |
 | `to_unix_timestamp` | ✅ |  |
-| `to_utc_timestamp` | ✅ | Legacy zone forms fall back (Incompatible) ([details](compatibility/expressions/datetime.md)) |
+| `to_utc_timestamp` | ✅ | Routes through the JVM codegen dispatcher by default (handles all timezone forms); the native path is opt-in via allowIncompatible ([details](compatibility/expressions/datetime.md)) |
 | `trunc` | ✅ |  |
 | `try_make_interval` | 🔜 | Produces legacy CalendarInterval; tracked by [#4540](https://github.com/apache/datafusion-comet/issues/4540) |
 | `try_make_timestamp` | ✅ |  |

spark/src/main/scala/org/apache/comet/serde/datetime.scala

@@ -357,7 +357,9 @@ private object UTCTimestampSerde {
       " execution time. See https://github.com/apache/datafusion-comet/issues/2013."
 }
 
-object CometFromUTCTimestamp extends CometExpressionSerde[FromUTCTimestamp] {
+object CometFromUTCTimestamp
+    extends CometExpressionSerde[FromUTCTimestamp]
+    with CodegenDispatchFallback {
 
   override def getSupportLevel(expr: FromUTCTimestamp): SupportLevel =
     Incompatible(Some(UTCTimestampSerde.tzParseIncompatReason))
@@ -375,7 +377,9 @@ object CometFromUTCTimestamp extends CometExpressionSerde[FromUTCTimestamp] {
   }
 }
 
-object CometToUTCTimestamp extends CometExpressionSerde[ToUTCTimestamp] {
+object CometToUTCTimestamp
+    extends CometExpressionSerde[ToUTCTimestamp]
+    with CodegenDispatchFallback {
 
   override def getSupportLevel(expr: ToUTCTimestamp): SupportLevel =
     Incompatible(Some(UTCTimestampSerde.tzParseIncompatReason))
@@ -393,7 +397,9 @@ object CometToUTCTimestamp extends CometExpressionSerde[ToUTCTimestamp] {
   }
 }
 
-object CometConvertTimezone extends CometExpressionSerde[ConvertTimezone] {
+object CometConvertTimezone
+    extends CometExpressionSerde[ConvertTimezone]
+    with CodegenDispatchFallback {
 
   override def getSupportLevel(expr: ConvertTimezone): SupportLevel =
     Incompatible(Some(UTCTimestampSerde.tzParseIncompatReason))

spark/src/test/resources/sql-tests/expressions/datetime/convert_timezone_dispatch.sql

@@ -0,0 +1,40 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- CometConvertTimezone mixes in CodegenDispatchFallback, so with allowIncompatible unset it routes
+-- through the JVM codegen dispatcher and matches Spark exactly, including the legacy timezone forms
+-- the native parser rejects. Verified across two session zones.
+-- ConfigMatrix: spark.sql.session.timeZone=UTC,America/Los_Angeles
+
+statement
+CREATE TABLE test_ct_dispatch(ts timestamp_ntz, src string, tgt string) USING parquet
+
+statement
+INSERT INTO test_ct_dispatch VALUES
+  (timestamp_ntz'2021-12-06 08:00:00', 'UTC', 'America/Los_Angeles'),
+  (timestamp_ntz'2021-07-01 12:00:00', 'America/New_York', 'Asia/Tokyo'),
+  (NULL, 'UTC', 'Asia/Tokyo')
+
+query
+SELECT convert_timezone(src, tgt, ts) FROM test_ct_dispatch
+
+query
+SELECT convert_timezone('UTC', 'America/Los_Angeles', timestamp_ntz'2021-12-06 08:00:00')
+
+-- legacy forms the native parser rejects but the dispatcher handles
+query
+SELECT convert_timezone('GMT+1', 'PST', timestamp_ntz'2021-12-06 08:00:00')

spark/src/test/resources/sql-tests/expressions/datetime/from_utc_timestamp_dispatch.sql

@@ -0,0 +1,45 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- CometFromUTCTimestamp mixes in CodegenDispatchFallback, so with allowIncompatible unset it
+-- routes through the JVM codegen dispatcher (Spark's own doGenCode) and matches Spark exactly,
+-- including the legacy timezone forms the native parser rejects (e.g. PST, GMT+1). Verified across
+-- two session zones to confirm the resolved timeZoneId survives closure serialization.
+-- ConfigMatrix: spark.sql.session.timeZone=UTC,America/Los_Angeles
+
+statement
+CREATE TABLE test_futc_dispatch(ts timestamp) USING parquet
+
+statement
+INSERT INTO test_futc_dispatch VALUES
+  (timestamp('2015-07-24 00:00:00')),
+  (timestamp('2015-01-24 00:00:00')),
+  (timestamp('1969-12-31 23:59:59')),
+  (NULL)
+
+query
+SELECT from_utc_timestamp(ts, 'America/Los_Angeles') FROM test_futc_dispatch
+
+query
+SELECT from_utc_timestamp(ts, '+02:00') FROM test_futc_dispatch
+
+-- legacy forms the native parser rejects but the dispatcher handles
+query
+SELECT from_utc_timestamp(ts, 'PST') FROM test_futc_dispatch
+
+query
+SELECT from_utc_timestamp(timestamp('2017-07-14 02:40:00'), 'GMT+1'), from_utc_timestamp(timestamp('2017-07-14 02:40:00'), 'Etc/GMT-1')

spark/src/test/resources/sql-tests/expressions/datetime/to_utc_timestamp_dispatch.sql

@@ -0,0 +1,44 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- CometToUTCTimestamp mixes in CodegenDispatchFallback, so with allowIncompatible unset it routes
+-- through the JVM codegen dispatcher and matches Spark exactly, including the legacy timezone forms
+-- the native parser rejects. Verified across two session zones.
+-- ConfigMatrix: spark.sql.session.timeZone=UTC,America/Los_Angeles
+
+statement
+CREATE TABLE test_tutc_dispatch(ts timestamp) USING parquet
+
+statement
+INSERT INTO test_tutc_dispatch VALUES
+  (timestamp('2015-07-24 00:00:00')),
+  (timestamp('2015-01-24 00:00:00')),
+  (timestamp('1969-12-31 23:59:59')),
+  (NULL)
+
+query
+SELECT to_utc_timestamp(ts, 'America/Los_Angeles') FROM test_tutc_dispatch
+
+query
+SELECT to_utc_timestamp(ts, '+02:00') FROM test_tutc_dispatch
+
+-- legacy forms the native parser rejects but the dispatcher handles
+query
+SELECT to_utc_timestamp(ts, 'PST') FROM test_tutc_dispatch
+
+query
+SELECT to_utc_timestamp(timestamp('2017-07-14 02:40:00'), 'GMT+1'), to_utc_timestamp(timestamp('2017-07-14 02:40:00'), 'Etc/GMT-1')