From 8c7ddfca8f4f3fa8c1dda7e4801d1ed57e75ca1e Mon Sep 17 00:00:00 2001 From: Zihan Dai <99155080+PDGGK@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:37:12 +1000 Subject: [PATCH] Return one NULL row for last_by over a never-written device (#16985) A global aggregation without GROUP BY always produces exactly one row; over empty input every aggregate except count() evaluates to NULL. For the last-cache-optimized last/last_by path, a device set that resolves to zero devices (e.g. a filter on a device that was never written) left the operator with nothing to iterate, so it returned zero rows instead of the single NULL row required by SQL semantics (consistent with engines such as Trino). LastQueryAggTableScanOperator now emits one all-NULL row when there are zero devices, no GROUP BY, and the aggregation is at its final/complete stage. GROUP BY over an empty group still returns no rows, and a partial stage still emits nothing. The deleted-device case (the device still exists, so the operator iterates and already emits the NULL row) is unchanged. Scoped to last/last_by; the general aggregation operator (max/min/sum/avg) likely returns empty for a never-written device too and is left as a follow-up. Signed-off-by: Zihan Dai <99155080+PDGGK@users.noreply.github.com> --- .../it/db/it/IoTDBDeletionTableIT.java | 114 ++++++++++++++++++ .../LastQueryAggTableScanOperator.java | 50 +++++++- 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 82b6c1a8ff6b..b31b853f9b82 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -81,6 +81,7 @@ import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -3199,4 +3200,117 @@ private void cleanData(int testNum) throws SQLException { statement.execute(String.format(deleteAllTemplate, testNum)); } } + + // A global aggregation without GROUP BY over zero matching rows must return exactly one row + // whose value is NULL (SQL standard, matching e.g. Trino), not zero rows. These cover the two + // empty-input shapes: a device that was never written, and a device whose data was deleted. + + @Test + public void testLastByOverNeverWrittenDeviceReturnsSingleNullRow() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute("create table last_empty0(deviceId string tag, s0 int32 field)"); + statement.execute("insert into last_empty0(time, deviceId, s0) values (1, 'd0', 1)"); + statement.execute("flush"); + + // 'nope' was never written, so its device set resolves to zero devices. + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, time) from last_empty0 where deviceId = 'nope'")) { + assertSingleAllNullRow(resultSet); + } + } + } + + @Test + public void testThreeArgLastByOverNeverWrittenDeviceReturnsSingleNullRow() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "create table last_empty4(deviceId string tag, s0 int32 field, s1 int32 field)"); + statement.execute("insert into last_empty4(time, deviceId, s0, s1) values (1, 'd0', 1, 2)"); + statement.execute("flush"); + + // 3-arg last_by(target, ordering, time) over a never-written device -> one NULL row. + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, s1, time) from last_empty4 where deviceId = 'nope'")) { + assertSingleAllNullRow(resultSet); + } + } + } + + @Test + public void testMultipleLastAggregatesOverNeverWrittenDeviceReturnSingleAllNullRow() + throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "create table last_empty1(deviceId string tag, s0 int32 field, s1 int64 field)"); + statement.execute("insert into last_empty1(time, deviceId, s0, s1) values (1, 'd0', 1, 2)"); + statement.execute("flush"); + + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, time), last_by(s1, time) from last_empty1 where deviceId = 'nope'")) { + assertSingleAllNullRow(resultSet); + } + } + } + + @Test + public void testLastByOverNeverWrittenDeviceWithGroupByReturnsNoRows() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute("create table last_empty2(deviceId string tag, s0 int32 field)"); + statement.execute("insert into last_empty2(time, deviceId, s0) values (1, 'd0', 1)"); + statement.execute("flush"); + + // The one-NULL-row rule is for no-GROUP-BY only; an empty group produces no rows. + try (ResultSet resultSet = + statement.executeQuery( + "select deviceId, last_by(s0, time) from last_empty2 where deviceId = 'nope' group by deviceId")) { + assertFalse("GROUP BY over an empty group must return no rows", resultSet.next()); + } + } + } + + @Test + public void testLastByOverDeletedDeviceStillReturnsSingleNullRow() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute("create table last_empty3(deviceId string tag, s0 int32 field)"); + statement.execute("insert into last_empty3(time, deviceId, s0) values (1, 'd0', 1)"); + statement.execute("flush"); + statement.execute("delete from last_empty3 where deviceId = 'd0'"); + + // The deleted device still exists in the schema; last_by must return one NULL row. Query + // twice to cover both the last-value-cache path and the recomputed path. + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, time) from last_empty3 where deviceId = 'd0'")) { + assertSingleAllNullRow(resultSet); + } + try (ResultSet resultSet = + statement.executeQuery( + "select last_by(s0, time) from last_empty3 where deviceId = 'd0'")) { + assertSingleAllNullRow(resultSet); + } + } + } + + private void assertSingleAllNullRow(ResultSet resultSet) throws SQLException { + assertTrue("Expected exactly one result row, but got none", resultSet.next()); + int columnCount = resultSet.getMetaData().getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + Object value = resultSet.getObject(i); + assertNull("Expected column " + i + " to be NULL, but got: " + value, value); + } + assertFalse("Expected exactly one result row, but got more than one", resultSet.next()); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java index dc95221f0a95..cf2682bb9903 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java @@ -75,6 +75,13 @@ public class LastQueryAggTableScanOperator extends AbstractAggTableScanOperator private final List cachedDeviceEntries; private final int allDeviceCount; + // A no-GROUP-BY global aggregation whose device set resolves to zero devices + // (e.g. last_by on a device that was never written) has nothing for the device + // loop to iterate, so without this it would emit zero rows. Per SQL semantics a + // global aggregate over empty input must return exactly one row whose value is + // NULL (except count()); this one-shot flag drives that single row. + private boolean emptyGlobalResultEmitted = false; + private final boolean needUpdateCache; private final boolean needUpdateNullEntry; private final List hitCachesIndexes; @@ -129,8 +136,10 @@ public boolean hasNext() throws Exception { if (retainedTsBlock != null) { return true; } - - return outputDeviceIndex < allDeviceCount; + if (outputDeviceIndex < allDeviceCount) { + return true; + } + return shouldEmitEmptyGlobalResult() && !emptyGlobalResultEmitted; } @Override @@ -147,6 +156,13 @@ public TsBlock next() throws Exception { processCurrentDevice(); } + if (resultTsBlockBuilder.isEmpty() + && !emptyGlobalResultEmitted + && shouldEmitEmptyGlobalResult()) { + appendEmptyGlobalAggregationResult(); + emptyGlobalResultEmitted = true; + } + if (resultTsBlockBuilder.isEmpty()) { return null; } @@ -155,6 +171,36 @@ public TsBlock next() throws Exception { return checkTsBlockSizeAndGetResult(); } + /** + * A no-GROUP-BY global aggregation whose device set resolves to zero devices must still return + * exactly one row (each aggregate NULL, except count()) — the same result the deleted-device case + * already produces once its device is iterated. Limited to the final/complete stage: a partial + * stage must not emit an intermediate row, and a GROUP BY stays empty (empty groups produce no + * rows). + */ + private boolean shouldEmitEmptyGlobalResult() { + return allDeviceCount == 0 + && groupingKeySize == 0 + && !tableAggregators.isEmpty() + && !tableAggregators.get(0).getStep().isOutputPartial(); + } + + private void appendEmptyGlobalAggregationResult() { + // One row, each aggregator's no-input final value. shouldEmitEmptyGlobalResult() + // guarantees groupingKeySize == 0, and a last-cache-optimized scan never carries a + // date-bin window (dateBinSize == 0), so the value columns begin at + // groupingKeySize + dateBinSize; using the full offset mirrors appendAggregationResult. + // The accumulators were never fed input, so evaluate() (final step) writes NULL for + // last / last_by. + for (int i = 0; i < tableAggregators.size(); i++) { + tableAggregators + .get(i) + .evaluate( + resultTsBlockBuilder.getValueColumnBuilders()[groupingKeySize + dateBinSize + i]); + } + resultTsBlockBuilder.declarePosition(); + } + /** Main process logic, calc the last aggregation results of current device. */ private void processCurrentDevice() throws Exception { if (currentHitCacheIndex < hitCachesIndexes.size()