From 89c23b385c5d5ebaf6a559d472a51fefa6ae9de3 Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 15 Apr 2026 10:25:37 +0800 Subject: [PATCH 1/7] Limit REST write request size and return 413 on overflow --- .../RequestLimitExceededException.java | 25 ++++ .../filter/RequestSizeLimitFilter.java | 98 +++++++++++++++ .../protocol/handler/RequestLimitChecker.java | 56 +++++++++ .../table/v1/handler/ExceptionHandler.java | 10 +- .../v1/handler/RequestValidationHandler.java | 8 ++ .../table/v1/impl/RestApiServiceImpl.java | 16 ++- .../protocol/v1/handler/ExceptionHandler.java | 10 +- .../v1/handler/RequestValidationHandler.java | 23 ++++ .../v1/impl/GrafanaApiServiceImpl.java | 12 +- .../protocol/v1/impl/RestApiServiceImpl.java | 12 +- .../protocol/v2/handler/ExceptionHandler.java | 10 +- .../v2/handler/RequestValidationHandler.java | 37 ++++++ .../v2/impl/GrafanaApiServiceImpl.java | 12 +- .../protocol/v2/impl/RestApiServiceImpl.java | 20 +++- .../handler/RequestValidationLimitTest.java | 112 ++++++++++++++++++ .../db/conf/rest/IoTDBRestServiceConfig.java | 44 +++++++ .../conf/rest/IoTDBRestServiceDescriptor.java | 17 +++ .../test/resources/iotdb-common.properties | 12 ++ .../test/resources/iotdb-system.properties | 12 ++ .../conf/iotdb-system.properties.template | 20 ++++ 20 files changed, 545 insertions(+), 21 deletions(-) create mode 100644 external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java create mode 100644 external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java create mode 100644 external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java create mode 100644 external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java new file mode 100644 index 0000000000000..ab56ec4bb0344 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.exception; + +public class RequestLimitExceededException extends IllegalArgumentException { + + public RequestLimitExceededException(String message) { + super(message); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java new file mode 100644 index 0000000000000..c04c7a0c4b365 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.PreMatching; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.Provider; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +@Provider +@PreMatching +public class RequestSizeLimitFilter implements ContainerRequestFilter { + + @Override + public void filter(ContainerRequestContext requestContext) { + long maxBodySize = + IoTDBRestServiceDescriptor.getInstance().getConfig().getRestMaxRequestBodySizeInBytes(); + if (maxBodySize <= 0) { + return; + } + + int contentLength = requestContext.getLength(); + if (contentLength > maxBodySize) { + requestContext.abortWith(buildPayloadTooLargeResponse(maxBodySize)); + return; + } + + requestContext.setEntityStream( + new LimitedInputStream(requestContext.getEntityStream(), maxBodySize)); + } + + private static Response buildPayloadTooLargeResponse(long maxBodySize) { + return Response.status(413) + .type(MediaType.TEXT_PLAIN_TYPE) + .entity("REST request body exceeds limit " + maxBodySize + " bytes") + .build(); + } + + private static class LimitedInputStream extends FilterInputStream { + + private final long maxBodySize; + private long bytesRead; + + private LimitedInputStream(InputStream in, long maxBodySize) { + super(in); + this.maxBodySize = maxBodySize; + } + + @Override + public int read() throws IOException { + int result = super.read(); + if (result != -1) { + incrementBytesRead(1); + } + return result; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int result = super.read(b, off, len); + if (result > 0) { + incrementBytesRead(result); + } + return result; + } + + private void incrementBytesRead(int increment) { + bytesRead += increment; + if (bytesRead > maxBodySize) { + throw new WebApplicationException(buildPayloadTooLargeResponse(maxBodySize)); + } + } + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java new file mode 100644 index 0000000000000..085b017684343 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.handler; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; + +public class RequestLimitChecker { + + private RequestLimitChecker() {} + + public static void checkRowCount(String requestName, int rowCount) { + int maxRows = getConfig().getRestMaxInsertRows(); + if (maxRows > 0 && rowCount > maxRows) { + throw new RequestLimitExceededException( + String.format("%s row count %d exceeds limit %d", requestName, rowCount, maxRows)); + } + } + + public static void checkColumnCount(String requestName, int columnCount) { + int maxColumns = getConfig().getRestMaxInsertColumns(); + if (maxColumns > 0 && columnCount > maxColumns) { + throw new RequestLimitExceededException( + String.format( + "%s column count %d exceeds limit %d", requestName, columnCount, maxColumns)); + } + } + + public static void checkValueCount(String requestName, long valueCount) { + long maxValues = getConfig().getRestMaxInsertValues(); + if (maxValues > 0 && valueCount > maxValues) { + throw new RequestLimitExceededException( + String.format("%s value count %d exceeds limit %d", requestName, valueCount, maxValues)); + } + } + + private static IoTDBRestServiceConfig getConfig() { + return IoTDBRestServiceDescriptor.getInstance().getConfig(); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java index 61e64465a0678..5d9b78a6cfe3c 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -46,7 +47,10 @@ private ExceptionHandler() {} public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -92,4 +96,8 @@ public static ExecutionStatus tryCatchException(Exception e) { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java index fc546212caf43..ae127460bf8a6 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java @@ -17,6 +17,7 @@ package org.apache.iotdb.rest.protocol.table.v1.handler; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest; import org.apache.iotdb.rest.protocol.table.v1.model.SQL; @@ -62,6 +63,13 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR errorMessages.add("values and timestamps should have the same size"); } + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getColumnNames().size(); + RequestLimitChecker.checkRowCount("table insertTablet request", rowCount); + RequestLimitChecker.checkColumnCount("table insertTablet request", columnCount); + RequestLimitChecker.checkValueCount( + "table insertTablet request", (long) rowCount * columnCount); + for (int i = 0; i < insertTabletRequest.getDataTypes().size(); i++) { String dataType = insertTabletRequest.getDataTypes().get(i); if (isDataType(dataType)) { diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java index b35e9d27f595c..753ef436c6664 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java @@ -109,7 +109,9 @@ public Response executeQueryInternal( return res; } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -132,7 +134,9 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext) return executeQueryInternal(sql, statement, clientSession, relationSqlParser); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -174,7 +178,9 @@ public Response insertTablet( return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) @@ -231,7 +237,9 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex } return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java index 6edcb57ab4d9a..22301ff28ecce 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.v1.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -45,7 +46,10 @@ private ExceptionHandler() {} public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -88,4 +92,8 @@ public static ExecutionStatus tryCatchException(Exception e) { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java index dc81322cc7880..518217070c486 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java @@ -17,12 +17,14 @@ package org.apache.iotdb.rest.protocol.v1.handler; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.v1.model.ExpressionRequest; import org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest; import org.apache.iotdb.rest.protocol.v1.model.SQL; import org.apache.tsfile.external.commons.lang3.Validate; +import java.util.List; import java.util.Objects; public class RequestValidationHandler { @@ -40,8 +42,29 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR Objects.requireNonNull(insertTabletRequest.getTimestamps(), "timestamps should not be null"); Objects.requireNonNull(insertTabletRequest.getIsAligned(), "isAligned should not be null"); Objects.requireNonNull(insertTabletRequest.getDeviceId(), "deviceId should not be null"); + Objects.requireNonNull(insertTabletRequest.getMeasurements(), "measurements should not be null"); Objects.requireNonNull(insertTabletRequest.getDataTypes(), "dataTypes should not be null"); Objects.requireNonNull(insertTabletRequest.getValues(), "values should not be null"); + + if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("measurements and dataTypes should have the same size"); + } + if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("values and dataTypes should have the same size"); + } + + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getMeasurements().size(); + RequestLimitChecker.checkRowCount("insertTablet request", rowCount); + RequestLimitChecker.checkColumnCount("insertTablet request", columnCount); + RequestLimitChecker.checkValueCount("insertTablet request", (long) rowCount * columnCount); + + for (List column : insertTabletRequest.getValues()) { + if (column.size() != rowCount) { + throw new IllegalArgumentException( + "Each value column should have the same size as timestamps"); + } + } } public static void validateExpressionRequest(ExpressionRequest expressionRequest) { diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java index 0db4cd06c669f..8322599fc2625 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java @@ -133,7 +133,9 @@ public Response variables(SQL sql, SecurityContext securityContext) { return QueryDataSetHandler.fillGrafanaVariablesResult(queryExecution, statement); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -206,7 +208,9 @@ public Response expression(ExpressionRequest expressionRequest, SecurityContext } } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -269,7 +273,9 @@ public Response node(List requestBody, SecurityContext securityContext) return QueryDataSetHandler.fillGrafanaNodesResult(null); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java index 329ac47034bdb..55c48a4bae9b3 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java @@ -149,7 +149,9 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex .build(); } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -230,7 +232,9 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext) } } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -301,7 +305,9 @@ public Response insertTablet( .message(result.status.getMessage())) .build(); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java index 931ec0cec002b..23155368d0d5d 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -45,7 +46,10 @@ private ExceptionHandler() {} public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -88,4 +92,8 @@ public static ExecutionStatus tryCatchException(Exception e) { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java index 5f999b3ddccdf..76cd07085ad6b 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java @@ -17,6 +17,7 @@ package org.apache.iotdb.rest.protocol.v2.handler; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.v2.model.ExpressionRequest; import org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest; import org.apache.iotdb.rest.protocol.v2.model.InsertTabletRequest; @@ -56,6 +57,23 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR Objects.requireNonNull( insertTabletRequest.getMeasurements(), "measurements should not be null"); Objects.requireNonNull(insertTabletRequest.getValues(), "values should not be null"); + if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("measurements and data_types should have the same size"); + } + if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException("values and data_types should have the same size"); + } + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getMeasurements().size(); + RequestLimitChecker.checkRowCount("insertTablet request", rowCount); + RequestLimitChecker.checkColumnCount("insertTablet request", columnCount); + RequestLimitChecker.checkValueCount("insertTablet request", (long) rowCount * columnCount); + for (List column : insertTabletRequest.getValues()) { + if (column.size() != rowCount) { + throw new IllegalArgumentException( + "Each value column should have the same size as timestamps"); + } + } List errorMessages = new ArrayList<>(); String device = insertTabletRequest.getDevice(); for (int i = 0; i < insertTabletRequest.getMeasurements().size(); i++) { @@ -80,10 +98,28 @@ public static void validateInsertRecordsRequest(InsertRecordsRequest insertRecor Objects.requireNonNull(insertRecordsRequest.getValuesList(), "values_list should not be null"); Objects.requireNonNull( insertRecordsRequest.getMeasurementsList(), "measurements_list should not be null"); + int rowCount = insertRecordsRequest.getDevices().size(); + if (insertRecordsRequest.getTimestamps().size() != rowCount + || insertRecordsRequest.getMeasurementsList().size() != rowCount + || insertRecordsRequest.getDataTypesList().size() != rowCount + || insertRecordsRequest.getValuesList().size() != rowCount) { + throw new IllegalArgumentException( + "devices, timestamps, measurements_list, data_types_list and values_list should have the same size"); + } + RequestLimitChecker.checkRowCount("insertRecords request", rowCount); List errorMessages = new ArrayList<>(); + long valueCount = 0; for (int i = 0; i < insertRecordsRequest.getDataTypesList().size(); i++) { String device = insertRecordsRequest.getDevices().get(i); List measurements = insertRecordsRequest.getMeasurementsList().get(i); + List dataTypes = insertRecordsRequest.getDataTypesList().get(i); + List values = insertRecordsRequest.getValuesList().get(i); + if (measurements.size() != dataTypes.size() || values.size() != dataTypes.size()) { + throw new IllegalArgumentException( + "Each insertRecords row should have the same number of measurements, data types and values"); + } + RequestLimitChecker.checkColumnCount("insertRecords request", measurements.size()); + valueCount += values.size(); for (int c = 0; c < insertRecordsRequest.getDataTypesList().get(i).size(); c++) { String dataType = insertRecordsRequest.getDataTypesList().get(i).get(c); String measurement = measurements.get(c); @@ -93,6 +129,7 @@ public static void validateInsertRecordsRequest(InsertRecordsRequest insertRecor } } } + RequestLimitChecker.checkValueCount("insertRecords request", valueCount); if (!errorMessages.isEmpty()) { throw new RuntimeException(String.join(",", errorMessages)); } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java index 5b120b9c1d7a3..882d049f1dd76 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java @@ -133,7 +133,9 @@ public Response variables(SQL sql, SecurityContext securityContext) { return QueryDataSetHandler.fillGrafanaVariablesResult(queryExecution, statement); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -206,7 +208,9 @@ public Response expression(ExpressionRequest expressionRequest, SecurityContext } } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -269,7 +273,9 @@ public Response node(List requestBody, SecurityContext securityContext) return QueryDataSetHandler.fillGrafanaNodesResult(null); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java index f6c42533a6231..3ecce38e7c5f6 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java @@ -210,7 +210,9 @@ public Response executeFastLastQueryStatement( } catch (Exception e) { finish = true; t = e; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long endTime = System.nanoTime(); long costTime = endTime - startTime; @@ -307,7 +309,9 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex return responseGenerateHelper(result); } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; if (statement != null) @@ -388,7 +392,9 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext) } } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -438,7 +444,9 @@ public Response insertRecords( return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertRowsStatement) @@ -492,7 +500,9 @@ public Response insertTablet( false); return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java new file mode 100644 index 0000000000000..9da02ecfa983f --- /dev/null +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.handler; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class RequestValidationLimitTest { + + private IoTDBRestServiceConfig config; + private int originalMaxInsertRows; + private int originalMaxInsertColumns; + private long originalMaxInsertValues; + + @Before + public void setUp() { + config = IoTDBRestServiceDescriptor.getInstance().getConfig(); + originalMaxInsertRows = config.getRestMaxInsertRows(); + originalMaxInsertColumns = config.getRestMaxInsertColumns(); + originalMaxInsertValues = config.getRestMaxInsertValues(); + } + + @After + public void tearDown() { + config.setRestMaxInsertRows(originalMaxInsertRows); + config.setRestMaxInsertColumns(originalMaxInsertColumns); + config.setRestMaxInsertValues(originalMaxInsertValues); + } + + @Test(expected = RequestLimitExceededException.class) + public void testV1InsertTabletRejectsTooManyRows() { + config.setRestMaxInsertRows(2); + + org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest request = + new org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest(); + request.setDeviceId("root.sg.d1"); + request.setIsAligned(false); + request.setMeasurements(Collections.singletonList("s1")); + request.setDataTypes(Collections.singletonList("INT64")); + request.setTimestamps(Arrays.asList(1L, 2L, 3L)); + request.setValues(Collections.singletonList(Arrays.asList(1L, 2L, 3L))); + + org.apache.iotdb.rest.protocol.v1.handler.RequestValidationHandler.validateInsertTabletRequest( + request); + } + + @Test(expected = RequestLimitExceededException.class) + public void testV2InsertRecordsRejectsTooManyValues() { + config.setRestMaxInsertRows(10); + config.setRestMaxInsertColumns(10); + config.setRestMaxInsertValues(2); + + org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest request = + new org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest(); + request.setIsAligned(false); + request.setDevices(Arrays.asList("root.sg.d1", "root.sg.d2")); + request.setTimestamps(Arrays.asList(1L, 2L)); + request.setMeasurementsList( + Arrays.asList( + Arrays.asList("s1", "s2"), + Collections.singletonList("s1"))); + request.setDataTypesList( + Arrays.asList( + Arrays.asList("INT64", "INT64"), + Collections.singletonList("INT64"))); + request.setValuesList(Arrays.asList(Arrays.asList(1L, 2L), Collections.singletonList(3L))); + + org.apache.iotdb.rest.protocol.v2.handler.RequestValidationHandler.validateInsertRecordsRequest( + request); + } + + @Test(expected = RequestLimitExceededException.class) + public void testTableInsertTabletRejectsTooManyColumns() { + config.setRestMaxInsertColumns(1); + + org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest request = + new org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest(); + request.setDatabase("db"); + request.setTable("t1"); + request.setColumnNames(Arrays.asList("tag1", "s1")); + request.setColumnCategories(Arrays.asList("TAG", "FIELD")); + request.setDataTypes(Arrays.asList("STRING", "INT64")); + request.setTimestamps(Collections.singletonList(1L)); + request.setValues(Collections.singletonList(Arrays.asList("a", 1L))); + + org.apache.iotdb.rest.protocol.table.v1.handler.RequestValidationHandler + .validateInsertTabletRequest(request); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index 64c0f65fe3034..2e2acd2e4d9bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -59,6 +59,18 @@ public class IoTDBRestServiceConfig { private int restQueryDefaultRowSizeLimit = 10000; + /** Maximum accepted REST request body size in bytes. */ + private long restMaxRequestBodySizeInBytes = 16 * 1024 * 1024L; + + /** Maximum row count accepted by a single REST write request. */ + private int restMaxInsertRows = 100000; + + /** Maximum column count accepted by a single REST write request. */ + private int restMaxInsertColumns = 1024; + + /** Maximum value cell count accepted by a single REST write request. */ + private long restMaxInsertValues = 1000000L; + /** Is client authentication required. */ private boolean clientAuth = false; @@ -173,4 +185,36 @@ public int getRestQueryDefaultRowSizeLimit() { public void setRestQueryDefaultRowSizeLimit(int restQueryDefaultRowSizeLimit) { this.restQueryDefaultRowSizeLimit = restQueryDefaultRowSizeLimit; } + + public long getRestMaxRequestBodySizeInBytes() { + return restMaxRequestBodySizeInBytes; + } + + public void setRestMaxRequestBodySizeInBytes(long restMaxRequestBodySizeInBytes) { + this.restMaxRequestBodySizeInBytes = restMaxRequestBodySizeInBytes; + } + + public int getRestMaxInsertRows() { + return restMaxInsertRows; + } + + public void setRestMaxInsertRows(int restMaxInsertRows) { + this.restMaxInsertRows = restMaxInsertRows; + } + + public int getRestMaxInsertColumns() { + return restMaxInsertColumns; + } + + public void setRestMaxInsertColumns(int restMaxInsertColumns) { + this.restMaxInsertColumns = restMaxInsertColumns; + } + + public long getRestMaxInsertValues() { + return restMaxInsertValues; + } + + public void setRestMaxInsertValues(long restMaxInsertValues) { + this.restMaxInsertValues = restMaxInsertValues; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index c4f9d131de958..124c0fbbb7c50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -91,6 +91,23 @@ private void loadProps(TrimProperties trimProperties) { trimProperties.getProperty( "rest_query_default_row_size_limit", Integer.toString(conf.getRestQueryDefaultRowSizeLimit())))); + conf.setRestMaxRequestBodySizeInBytes( + Long.parseLong( + trimProperties.getProperty( + "rest_max_request_body_size_in_bytes", + Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); + conf.setRestMaxInsertRows( + Integer.parseInt( + trimProperties.getProperty( + "rest_max_insert_rows", Integer.toString(conf.getRestMaxInsertRows())))); + conf.setRestMaxInsertColumns( + Integer.parseInt( + trimProperties.getProperty( + "rest_max_insert_columns", Integer.toString(conf.getRestMaxInsertColumns())))); + conf.setRestMaxInsertValues( + Long.parseLong( + trimProperties.getProperty( + "rest_max_insert_values", Long.toString(conf.getRestMaxInsertValues())))); conf.setEnableSwagger( Boolean.parseBoolean( trimProperties.getProperty( diff --git a/iotdb-core/datanode/src/test/resources/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/iotdb-common.properties index 1053eaafa2dad..74c5a2c70aa8d 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-common.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-common.properties @@ -33,6 +33,18 @@ enable_rest_service=true # the default row limit to a REST query response when the rowSize parameter is not given in request # rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes +# rest_max_request_body_size_in_bytes=16777216 + +# Maximum rows accepted by a single REST write request +# rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request +# rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request +# rest_max_insert_values=1000000 + # is SSL enabled # enable_https=false diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties index ce0ecff34f35f..4e542beba4f46 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-system.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -50,6 +50,18 @@ enable_rest_service=true # the default row limit to a REST query response when the rowSize parameter is not given in request # rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes +# rest_max_request_body_size_in_bytes=16777216 + +# Maximum rows accepted by a single REST write request +# rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request +# rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request +# rest_max_insert_values=1000000 + # is SSL enabled # enable_https=false diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 378a6226cbffd..18bcc038b8966 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -584,6 +584,26 @@ enable_swagger=false # Datatype: int rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: long +rest_max_request_body_size_in_bytes=16777216 + +# Maximum rows accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: int +rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: int +rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: long +rest_max_insert_values=1000000 + # Is client authentication required # effectiveMode: restart # Datatype: boolean From 39b16b80deac1e2b1969fa9b159357823740e3b9 Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 15 Apr 2026 10:33:05 +0800 Subject: [PATCH 2/7] style: apply spotless formatting --- .../protocol/v1/handler/RequestValidationHandler.java | 3 ++- .../rest/protocol/handler/RequestValidationLimitTest.java | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java index 518217070c486..55843bbbc0923 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java @@ -42,7 +42,8 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR Objects.requireNonNull(insertTabletRequest.getTimestamps(), "timestamps should not be null"); Objects.requireNonNull(insertTabletRequest.getIsAligned(), "isAligned should not be null"); Objects.requireNonNull(insertTabletRequest.getDeviceId(), "deviceId should not be null"); - Objects.requireNonNull(insertTabletRequest.getMeasurements(), "measurements should not be null"); + Objects.requireNonNull( + insertTabletRequest.getMeasurements(), "measurements should not be null"); Objects.requireNonNull(insertTabletRequest.getDataTypes(), "dataTypes should not be null"); Objects.requireNonNull(insertTabletRequest.getValues(), "values should not be null"); diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java index 9da02ecfa983f..e061e4d0c12eb 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java @@ -79,13 +79,9 @@ public void testV2InsertRecordsRejectsTooManyValues() { request.setDevices(Arrays.asList("root.sg.d1", "root.sg.d2")); request.setTimestamps(Arrays.asList(1L, 2L)); request.setMeasurementsList( - Arrays.asList( - Arrays.asList("s1", "s2"), - Collections.singletonList("s1"))); + Arrays.asList(Arrays.asList("s1", "s2"), Collections.singletonList("s1"))); request.setDataTypesList( - Arrays.asList( - Arrays.asList("INT64", "INT64"), - Collections.singletonList("INT64"))); + Arrays.asList(Arrays.asList("INT64", "INT64"), Collections.singletonList("INT64"))); request.setValuesList(Arrays.asList(Arrays.asList(1L, 2L), Collections.singletonList(3L))); org.apache.iotdb.rest.protocol.v2.handler.RequestValidationHandler.validateInsertRecordsRequest( From be75042f480bc5d4e2b653432f308e2eead644de Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 22 Jul 2026 11:32:02 +0800 Subject: [PATCH 3/7] Fix REST payload limit response format --- .../apache/iotdb/rest/i18n/RestMessages.java | 4 + .../apache/iotdb/rest/i18n/RestMessages.java | 4 + .../filter/RequestSizeLimitFilter.java | 16 +- .../filter/RequestSizeLimitFilterTest.java | 149 ++++++++++++++++++ .../handler/RequestValidationLimitTest.java | 17 +- 5 files changed, 179 insertions(+), 11 deletions(-) create mode 100644 external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java diff --git a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java index 1b4bf36432cac..374df3036ebdd 100644 --- a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java @@ -74,6 +74,10 @@ public final class RestMessages { "The number of values in the %dth row is not equal to the data_types size"; public static final String ERROR_MESSAGE_SEPARATOR = ","; + // --- RequestSizeLimitFilter --- + public static final String MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412 = + "REST request body exceeds limit %d bytes"; + private RestMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java index 23f3475cdc4cc..e43b16a652ffd 100644 --- a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java @@ -72,6 +72,10 @@ public final class RestMessages { "第 %d 行的 values 数量与 data_types 数量不相等"; public static final String ERROR_MESSAGE_SEPARATOR = "，"; + // --- RequestSizeLimitFilter --- + public static final String MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412 = + "REST 请求体超过限制 %d 字节"; + private RestMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java index 7ef8cc0af7488..01337b04c9579 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java @@ -18,6 +18,8 @@ package org.apache.iotdb.rest.protocol.filter; import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.container.ContainerRequestContext; @@ -35,6 +37,8 @@ @PreMatching public class RequestSizeLimitFilter implements ContainerRequestFilter { + private static final int PAYLOAD_TOO_LARGE_STATUS_CODE = 413; + @Override public void filter(ContainerRequestContext requestContext) { long maxBodySize = @@ -54,9 +58,15 @@ public void filter(ContainerRequestContext requestContext) { } private static Response buildPayloadTooLargeResponse(long maxBodySize) { - return Response.status(413) - .type(MediaType.TEXT_PLAIN_TYPE) - .entity("REST request body exceeds limit " + maxBodySize + " bytes") + return Response.status(PAYLOAD_TOO_LARGE_STATUS_CODE) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity( + new ExecutionStatus() + .code(PAYLOAD_TOO_LARGE_STATUS_CODE) + .message( + String.format( + RestMessages.MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412, + maxBodySize))) .build(); } diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java new file mode 100644 index 0000000000000..86e37b07aeab1 --- /dev/null +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.model.ExecutionStatus; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class RequestSizeLimitFilterTest { + + private IoTDBRestServiceConfig config; + private long originalMaxBodySize; + + @Before + public void setUp() { + config = IoTDBRestServiceDescriptor.getInstance().getConfig(); + originalMaxBodySize = config.getRestMaxRequestBodySizeInBytes(); + } + + @After + public void tearDown() { + config.setRestMaxRequestBodySizeInBytes(originalMaxBodySize); + } + + @Test + public void testAbortContentLengthOverLimit() { + config.setRestMaxRequestBodySizeInBytes(4); + TestRequestContext context = TestRequestContext.withLength(5); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertPayloadTooLarge(context.abortedResponse(), 4); + } + + @Test + public void testRejectStreamOverLimit() throws IOException { + config.setRestMaxRequestBodySizeInBytes(4); + TestRequestContext context = + TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertNull(context.abortedResponse()); + WebApplicationException exception = + assertThrows(WebApplicationException.class, () -> context.entityStream().readAllBytes()); + assertPayloadTooLarge(exception.getResponse(), 4); + } + + private static void assertPayloadTooLarge(Response response, long maxBodySize) { + assertEquals(413, response.getStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + assertTrue(response.getEntity() instanceof ExecutionStatus); + + ExecutionStatus status = (ExecutionStatus) response.getEntity(); + assertEquals(Integer.valueOf(413), status.getCode()); + assertEquals( + String.format( + RestMessages.MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412, maxBodySize), + status.getMessage()); + } + + private static class TestRequestContext { + + private final int contentLength; + private final AtomicReference entityStream; + private final AtomicReference abortedResponse = new AtomicReference<>(); + + private TestRequestContext(int contentLength, InputStream entityStream) { + this.contentLength = contentLength; + this.entityStream = new AtomicReference<>(entityStream); + } + + private static TestRequestContext withLength(int contentLength) { + return new TestRequestContext(contentLength, InputStream.nullInputStream()); + } + + private static TestRequestContext withStream(byte[] body) { + return new TestRequestContext(-1, new ByteArrayInputStream(body)); + } + + private ContainerRequestContext proxy() { + return (ContainerRequestContext) + Proxy.newProxyInstance( + ContainerRequestContext.class.getClassLoader(), + new Class[] {ContainerRequestContext.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "getLength": + return contentLength; + case "getEntityStream": + return entityStream.get(); + case "setEntityStream": + entityStream.set((InputStream) args[0]); + return null; + case "abortWith": + abortedResponse.set((Response) args[0]); + return null; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } + + private InputStream entityStream() { + return entityStream.get(); + } + + private Response abortedResponse() { + return abortedResponse.get(); + } + } +} diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java index e061e4d0c12eb..810de95c191eb 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java @@ -20,6 +20,8 @@ import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; +import org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest; +import org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest; import org.junit.After; import org.junit.Before; @@ -28,6 +30,9 @@ import java.util.Arrays; import java.util.Collections; +import static org.apache.iotdb.rest.protocol.v1.handler.RequestValidationHandler.validateInsertTabletRequest; +import static org.apache.iotdb.rest.protocol.v2.handler.RequestValidationHandler.validateInsertRecordsRequest; + public class RequestValidationLimitTest { private IoTDBRestServiceConfig config; @@ -54,8 +59,7 @@ public void tearDown() { public void testV1InsertTabletRejectsTooManyRows() { config.setRestMaxInsertRows(2); - org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest request = - new org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest(); + InsertTabletRequest request = new InsertTabletRequest(); request.setDeviceId("root.sg.d1"); request.setIsAligned(false); request.setMeasurements(Collections.singletonList("s1")); @@ -63,8 +67,7 @@ public void testV1InsertTabletRejectsTooManyRows() { request.setTimestamps(Arrays.asList(1L, 2L, 3L)); request.setValues(Collections.singletonList(Arrays.asList(1L, 2L, 3L))); - org.apache.iotdb.rest.protocol.v1.handler.RequestValidationHandler.validateInsertTabletRequest( - request); + validateInsertTabletRequest(request); } @Test(expected = RequestLimitExceededException.class) @@ -73,8 +76,7 @@ public void testV2InsertRecordsRejectsTooManyValues() { config.setRestMaxInsertColumns(10); config.setRestMaxInsertValues(2); - org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest request = - new org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest(); + InsertRecordsRequest request = new InsertRecordsRequest(); request.setIsAligned(false); request.setDevices(Arrays.asList("root.sg.d1", "root.sg.d2")); request.setTimestamps(Arrays.asList(1L, 2L)); @@ -84,8 +86,7 @@ public void testV2InsertRecordsRejectsTooManyValues() { Arrays.asList(Arrays.asList("INT64", "INT64"), Collections.singletonList("INT64"))); request.setValuesList(Arrays.asList(Arrays.asList(1L, 2L), Collections.singletonList(3L))); - org.apache.iotdb.rest.protocol.v2.handler.RequestValidationHandler.validateInsertRecordsRequest( - request); + validateInsertRecordsRequest(request); } @Test(expected = RequestLimitExceededException.class) From a91e99f63613a6ccefee7d4185b72f3a447b0087 Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 22 Jul 2026 12:07:29 +0800 Subject: [PATCH 4/7] Add REST request body memory quota --- .../apache/iotdb/rest/i18n/RestMessages.java | 36 ++++++ .../apache/iotdb/rest/i18n/RestMessages.java | 36 ++++++ .../RequestBodyMemoryReleaseFilter.java | 33 +++++ .../filter/RequestSizeLimitFilter.java | 82 ++++++++++++- .../filter/RestRequestBodyMemoryManager.java | 116 ++++++++++++++++++ .../protocol/handler/RequestLimitChecker.java | 18 ++- .../v1/handler/RequestValidationHandler.java | 8 +- .../v1/handler/RequestValidationHandler.java | 17 ++- .../v2/handler/RequestValidationHandler.java | 32 +++-- .../filter/RequestSizeLimitFilterTest.java | 104 ++++++++++++++++ .../db/conf/rest/IoTDBRestServiceConfig.java | 11 ++ .../conf/rest/IoTDBRestServiceDescriptor.java | 5 + .../test/resources/iotdb-common.properties | 3 + .../test/resources/iotdb-system.properties | 3 + .../conf/iotdb-system.properties.template | 5 + 15 files changed, 481 insertions(+), 28 deletions(-) create mode 100644 external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestBodyMemoryReleaseFilter.java create mode 100644 external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RestRequestBodyMemoryManager.java diff --git a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java index 374df3036ebdd..ac96b14e4c6b4 100644 --- a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java @@ -77,6 +77,42 @@ public final class RestMessages { // --- RequestSizeLimitFilter --- public static final String MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412 = "REST request body exceeds limit %d bytes"; + public static final String + MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9 = + "REST request body memory quota exceeds limit %d bytes"; + + // --- RequestLimitChecker --- + public static final String MESSAGE_INSERTTABLET_REQUEST_8647CA58 = "insertTablet request"; + public static final String MESSAGE_INSERTRECORDS_REQUEST_93E12369 = "insertRecords request"; + public static final String MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C = + "table insertTablet request"; + public static final String EXCEPTION_ARG_ROW_COUNT_ARG_EXCEEDS_LIMIT_ARG_EE427E4B = + "%s row count %d exceeds limit %d"; + public static final String EXCEPTION_ARG_COLUMN_COUNT_ARG_EXCEEDS_LIMIT_ARG_DEE9637E = + "%s column count %d exceeds limit %d"; + public static final String EXCEPTION_ARG_VALUE_COUNT_ARG_EXCEEDS_LIMIT_ARG_77F95703 = + "%s value count %d exceeds limit %d"; + + // --- RequestValidationHandler --- + public static final String + EXCEPTION_MEASUREMENTS_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_FF715FA9 = + "measurements and dataTypes should have the same size"; + public static final String EXCEPTION_VALUES_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_5BC1D604 = + "values and dataTypes should have the same size"; + public static final String + EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD = + "Each value column should have the same size as timestamps"; + public static final String + EXCEPTION_MEASUREMENTS_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_8526F19A = + "measurements and data_types should have the same size"; + public static final String EXCEPTION_VALUES_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_0BAE701D = + "values and data_types should have the same size"; + public static final String + EXCEPTION_DEVICES_TIMESTAMPS_MEASUREMENTS_LIST_DATA_TYPES_LIST_AND_VALUES_LIST_SHOULD_HAVE_THE_SAME_SIZE_5983AAC2 = + "devices, timestamps, measurements_list, data_types_list and values_list should have the same size"; + public static final String + EXCEPTION_EACH_INSERTRECORDS_ROW_SHOULD_HAVE_THE_SAME_NUMBER_OF_MEASUREMENTS_DATA_TYPES_AND_VALUES_AD58AEF2 = + "Each insertRecords row should have the same number of measurements, data types and values"; private RestMessages() {} // --------------------------------------------------------------------------- diff --git a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java index e43b16a652ffd..01037b2020419 100644 --- a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java @@ -75,6 +75,42 @@ public final class RestMessages { // --- RequestSizeLimitFilter --- public static final String MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412 = "REST 请求体超过限制 %d 字节"; + public static final String + MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9 = + "REST 请求体内存配额超过限制 %d 字节"; + + // --- RequestLimitChecker --- + public static final String MESSAGE_INSERTTABLET_REQUEST_8647CA58 = "insertTablet 请求"; + public static final String MESSAGE_INSERTRECORDS_REQUEST_93E12369 = "insertRecords 请求"; + public static final String MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C = + "table insertTablet 请求"; + public static final String EXCEPTION_ARG_ROW_COUNT_ARG_EXCEEDS_LIMIT_ARG_EE427E4B = + "%s 行数 %d 超过限制 %d"; + public static final String EXCEPTION_ARG_COLUMN_COUNT_ARG_EXCEEDS_LIMIT_ARG_DEE9637E = + "%s 列数 %d 超过限制 %d"; + public static final String EXCEPTION_ARG_VALUE_COUNT_ARG_EXCEEDS_LIMIT_ARG_77F95703 = + "%s value 数 %d 超过限制 %d"; + + // --- RequestValidationHandler --- + public static final String + EXCEPTION_MEASUREMENTS_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_FF715FA9 = + "measurements 和 dataTypes 的数量应相同"; + public static final String EXCEPTION_VALUES_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_5BC1D604 = + "values 和 dataTypes 的数量应相同"; + public static final String + EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD = + "每个 value 列的数量应与 timestamps 数量相同"; + public static final String + EXCEPTION_MEASUREMENTS_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_8526F19A = + "measurements 和 data_types 的数量应相同"; + public static final String EXCEPTION_VALUES_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_0BAE701D = + "values 和 data_types 的数量应相同"; + public static final String + EXCEPTION_DEVICES_TIMESTAMPS_MEASUREMENTS_LIST_DATA_TYPES_LIST_AND_VALUES_LIST_SHOULD_HAVE_THE_SAME_SIZE_5983AAC2 = + "devices、timestamps、measurements_list、data_types_list 和 values_list 的数量应相同"; + public static final String + EXCEPTION_EACH_INSERTRECORDS_ROW_SHOULD_HAVE_THE_SAME_NUMBER_OF_MEASUREMENTS_DATA_TYPES_AND_VALUES_AD58AEF2 = + "每个 insertRecords 行中的 measurements、data types 和 values 数量应相同"; private RestMessages() {} // --------------------------------------------------------------------------- diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestBodyMemoryReleaseFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestBodyMemoryReleaseFilter.java new file mode 100644 index 0000000000000..301de4f3b1c63 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestBodyMemoryReleaseFilter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.container.ContainerResponseFilter; +import jakarta.ws.rs.ext.Provider; + +@Provider +public class RequestBodyMemoryReleaseFilter implements ContainerResponseFilter { + + @Override + public void filter( + ContainerRequestContext requestContext, ContainerResponseContext responseContext) { + RestRequestBodyMemoryManager.releaseReservation(requestContext); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java index 01337b04c9579..71dd92858453e 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java @@ -38,23 +38,47 @@ public class RequestSizeLimitFilter implements ContainerRequestFilter { private static final int PAYLOAD_TOO_LARGE_STATUS_CODE = 413; + private static final int SERVICE_UNAVAILABLE_STATUS_CODE = 503; @Override public void filter(ContainerRequestContext requestContext) { long maxBodySize = IoTDBRestServiceDescriptor.getInstance().getConfig().getRestMaxRequestBodySizeInBytes(); - if (maxBodySize <= 0) { + long memoryLimit = + IoTDBRestServiceDescriptor.getInstance().getConfig().getRestRequestBodyMemoryLimitInBytes(); + if (maxBodySize <= 0 && memoryLimit <= 0) { return; } int contentLength = requestContext.getLength(); - if (contentLength > maxBodySize) { + if (maxBodySize > 0 && contentLength > maxBodySize) { requestContext.abortWith(buildPayloadTooLargeResponse(maxBodySize)); return; } + RestRequestBodyMemoryManager.Reservation memoryReservation = + RestRequestBodyMemoryManager.newReservation(memoryLimit); + long memoryReservedByContentLength = 0; + if (contentLength > 0 && memoryReservation.isEnabled()) { + if (!memoryReservation.reserve(contentLength)) { + memoryReservation.close(); + requestContext.abortWith(buildMemoryQuotaExceededResponse(memoryLimit)); + return; + } + memoryReservedByContentLength = contentLength; + RestRequestBodyMemoryManager.registerReservation(requestContext, memoryReservation); + } + requestContext.setEntityStream( - new LimitedInputStream(requestContext.getEntityStream(), maxBodySize)); + new LimitedInputStream( + requestContext.getEntityStream(), + maxBodySize, + memoryLimit, + memoryReservation, + memoryReservedByContentLength)); + if (memoryReservation.isEnabled() && memoryReservedByContentLength == 0) { + RestRequestBodyMemoryManager.registerReservation(requestContext, memoryReservation); + } } private static Response buildPayloadTooLargeResponse(long maxBodySize) { @@ -70,14 +94,39 @@ private static Response buildPayloadTooLargeResponse(long maxBodySize) { .build(); } + private static Response buildMemoryQuotaExceededResponse(long memoryLimit) { + return Response.status(SERVICE_UNAVAILABLE_STATUS_CODE) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity( + new ExecutionStatus() + .code(SERVICE_UNAVAILABLE_STATUS_CODE) + .message( + String.format( + RestMessages + .MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9, + memoryLimit))) + .build(); + } + private static class LimitedInputStream extends FilterInputStream { private final long maxBodySize; + private final long memoryLimit; + private final RestRequestBodyMemoryManager.Reservation memoryReservation; + private long memoryCoveredBytes; private long bytesRead; - private LimitedInputStream(InputStream in, long maxBodySize) { + private LimitedInputStream( + InputStream in, + long maxBodySize, + long memoryLimit, + RestRequestBodyMemoryManager.Reservation memoryReservation, + long memoryCoveredBytes) { super(in); this.maxBodySize = maxBodySize; + this.memoryLimit = memoryLimit; + this.memoryReservation = memoryReservation; + this.memoryCoveredBytes = memoryCoveredBytes; } @Override @@ -100,9 +149,32 @@ public int read(byte[] b, int off, int len) throws IOException { private void incrementBytesRead(int increment) { bytesRead += increment; - if (bytesRead > maxBodySize) { + if (maxBodySize > 0 && bytesRead > maxBodySize) { + memoryReservation.close(); throw new WebApplicationException(buildPayloadTooLargeResponse(maxBodySize)); } + reserveMemoryIfNecessary(); + } + + private void reserveMemoryIfNecessary() { + if (bytesRead <= memoryCoveredBytes) { + return; + } + long sizeToReserve = bytesRead - memoryCoveredBytes; + if (!memoryReservation.reserve(sizeToReserve)) { + memoryReservation.close(); + throw new WebApplicationException(buildMemoryQuotaExceededResponse(memoryLimit)); + } + memoryCoveredBytes = bytesRead; + } + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + memoryReservation.close(); + } } } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RestRequestBodyMemoryManager.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RestRequestBodyMemoryManager.java new file mode 100644 index 0000000000000..3b9ea6bb4b5e9 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RestRequestBodyMemoryManager.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import jakarta.ws.rs.container.ContainerRequestContext; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +final class RestRequestBodyMemoryManager { + + static final String REQUEST_BODY_MEMORY_RESERVATION_PROPERTY = + RestRequestBodyMemoryManager.class.getName() + ".requestBodyMemoryReservation"; + private static final AtomicLong RESERVED_MEMORY_IN_BYTES = new AtomicLong(); + + private RestRequestBodyMemoryManager() {} + + static Reservation newReservation(long memoryLimitInBytes) { + return new Reservation(memoryLimitInBytes); + } + + static long getReservedMemoryInBytes() { + return RESERVED_MEMORY_IN_BYTES.get(); + } + + static void resetForTest() { + RESERVED_MEMORY_IN_BYTES.set(0); + } + + static void registerReservation( + ContainerRequestContext requestContext, Reservation memoryReservation) { + if (memoryReservation.isEnabled()) { + requestContext.setProperty(REQUEST_BODY_MEMORY_RESERVATION_PROPERTY, memoryReservation); + } + } + + static void releaseReservation(ContainerRequestContext requestContext) { + Object reservation = requestContext.getProperty(REQUEST_BODY_MEMORY_RESERVATION_PROPERTY); + if (reservation instanceof Reservation) { + ((Reservation) reservation).close(); + requestContext.removeProperty(REQUEST_BODY_MEMORY_RESERVATION_PROPERTY); + } + } + + private static boolean tryReserve(long sizeInBytes, long memoryLimitInBytes) { + if (sizeInBytes <= 0 || memoryLimitInBytes <= 0) { + return true; + } + + while (true) { + long currentReservedBytes = RESERVED_MEMORY_IN_BYTES.get(); + if (currentReservedBytes > memoryLimitInBytes - sizeInBytes) { + return false; + } + if (RESERVED_MEMORY_IN_BYTES.compareAndSet( + currentReservedBytes, currentReservedBytes + sizeInBytes)) { + return true; + } + } + } + + static final class Reservation implements AutoCloseable { + + private final long memoryLimitInBytes; + private final AtomicBoolean released = new AtomicBoolean(); + private long reservedMemoryInBytes; + + private Reservation(long memoryLimitInBytes) { + this.memoryLimitInBytes = memoryLimitInBytes; + } + + boolean isEnabled() { + return memoryLimitInBytes > 0; + } + + synchronized boolean reserve(long sizeInBytes) { + if (sizeInBytes <= 0 || !isEnabled()) { + return true; + } + if (released.get()) { + return false; + } + if (!tryReserve(sizeInBytes, memoryLimitInBytes)) { + return false; + } + reservedMemoryInBytes += sizeInBytes; + return true; + } + + @Override + public synchronized void close() { + if (released.compareAndSet(false, true)) { + long reservedBytes = reservedMemoryInBytes; + reservedMemoryInBytes = 0; + if (reservedBytes > 0) { + RESERVED_MEMORY_IN_BYTES.addAndGet(-reservedBytes); + } + } + } + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java index 085b017684343..0ec64b9b125c5 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java @@ -19,6 +19,7 @@ import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.i18n.RestMessages; import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; public class RequestLimitChecker { @@ -29,7 +30,11 @@ public static void checkRowCount(String requestName, int rowCount) { int maxRows = getConfig().getRestMaxInsertRows(); if (maxRows > 0 && rowCount > maxRows) { throw new RequestLimitExceededException( - String.format("%s row count %d exceeds limit %d", requestName, rowCount, maxRows)); + String.format( + RestMessages.EXCEPTION_ARG_ROW_COUNT_ARG_EXCEEDS_LIMIT_ARG_EE427E4B, + requestName, + rowCount, + maxRows)); } } @@ -38,7 +43,10 @@ public static void checkColumnCount(String requestName, int columnCount) { if (maxColumns > 0 && columnCount > maxColumns) { throw new RequestLimitExceededException( String.format( - "%s column count %d exceeds limit %d", requestName, columnCount, maxColumns)); + RestMessages.EXCEPTION_ARG_COLUMN_COUNT_ARG_EXCEEDS_LIMIT_ARG_DEE9637E, + requestName, + columnCount, + maxColumns)); } } @@ -46,7 +54,11 @@ public static void checkValueCount(String requestName, long valueCount) { long maxValues = getConfig().getRestMaxInsertValues(); if (maxValues > 0 && valueCount > maxValues) { throw new RequestLimitExceededException( - String.format("%s value count %d exceeds limit %d", requestName, valueCount, maxValues)); + String.format( + RestMessages.EXCEPTION_ARG_VALUE_COUNT_ARG_EXCEEDS_LIMIT_ARG_77F95703, + requestName, + valueCount, + maxValues)); } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java index 858ce2569ef05..3e6edabdeffad 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java @@ -67,10 +67,12 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR int rowCount = insertTabletRequest.getTimestamps().size(); int columnCount = insertTabletRequest.getColumnNames().size(); - RequestLimitChecker.checkRowCount("table insertTablet request", rowCount); - RequestLimitChecker.checkColumnCount("table insertTablet request", columnCount); + RequestLimitChecker.checkRowCount( + RestMessages.MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C, rowCount); + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C, columnCount); RequestLimitChecker.checkValueCount( - "table insertTablet request", (long) rowCount * columnCount); + RestMessages.MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C, (long) rowCount * columnCount); for (int i = 0; i < insertTabletRequest.getDataTypes().size(); i++) { String dataType = insertTabletRequest.getDataTypes().get(i); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java index 2cba6cf0438b7..a001a06e929b8 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java @@ -51,22 +51,27 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR Objects.requireNonNull(insertTabletRequest.getValues(), RestMessages.VALUES_NOT_NULL); if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { - throw new IllegalArgumentException("measurements and dataTypes should have the same size"); + throw new IllegalArgumentException( + RestMessages.EXCEPTION_MEASUREMENTS_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_FF715FA9); } if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { - throw new IllegalArgumentException("values and dataTypes should have the same size"); + throw new IllegalArgumentException( + RestMessages.EXCEPTION_VALUES_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_5BC1D604); } int rowCount = insertTabletRequest.getTimestamps().size(); int columnCount = insertTabletRequest.getMeasurements().size(); - RequestLimitChecker.checkRowCount("insertTablet request", rowCount); - RequestLimitChecker.checkColumnCount("insertTablet request", columnCount); - RequestLimitChecker.checkValueCount("insertTablet request", (long) rowCount * columnCount); + RequestLimitChecker.checkRowCount(RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, rowCount); + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, columnCount); + RequestLimitChecker.checkValueCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, (long) rowCount * columnCount); for (List column : insertTabletRequest.getValues()) { if (column.size() != rowCount) { throw new IllegalArgumentException( - "Each value column should have the same size as timestamps"); + RestMessages + .EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD); } } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java index 5c6ae2d0f2d6d..d0fd4218ee7a4 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java @@ -59,20 +59,25 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR insertTabletRequest.getMeasurements(), RestMessages.MEASUREMENTS_NOT_NULL); Objects.requireNonNull(insertTabletRequest.getValues(), RestMessages.VALUES_NOT_NULL); if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { - throw new IllegalArgumentException("measurements and data_types should have the same size"); + throw new IllegalArgumentException( + RestMessages.EXCEPTION_MEASUREMENTS_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_8526F19A); } if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { - throw new IllegalArgumentException("values and data_types should have the same size"); + throw new IllegalArgumentException( + RestMessages.EXCEPTION_VALUES_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_0BAE701D); } int rowCount = insertTabletRequest.getTimestamps().size(); int columnCount = insertTabletRequest.getMeasurements().size(); - RequestLimitChecker.checkRowCount("insertTablet request", rowCount); - RequestLimitChecker.checkColumnCount("insertTablet request", columnCount); - RequestLimitChecker.checkValueCount("insertTablet request", (long) rowCount * columnCount); + RequestLimitChecker.checkRowCount(RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, rowCount); + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, columnCount); + RequestLimitChecker.checkValueCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, (long) rowCount * columnCount); for (List column : insertTabletRequest.getValues()) { if (column.size() != rowCount) { throw new IllegalArgumentException( - "Each value column should have the same size as timestamps"); + RestMessages + .EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD); } } List errorMessages = new ArrayList<>(); @@ -106,9 +111,11 @@ public static void validateInsertRecordsRequest(InsertRecordsRequest insertRecor || insertRecordsRequest.getDataTypesList().size() != rowCount || insertRecordsRequest.getValuesList().size() != rowCount) { throw new IllegalArgumentException( - "devices, timestamps, measurements_list, data_types_list and values_list should have the same size"); + RestMessages + .EXCEPTION_DEVICES_TIMESTAMPS_MEASUREMENTS_LIST_DATA_TYPES_LIST_AND_VALUES_LIST_SHOULD_HAVE_THE_SAME_SIZE_5983AAC2); } - RequestLimitChecker.checkRowCount("insertRecords request", rowCount); + RequestLimitChecker.checkRowCount( + RestMessages.MESSAGE_INSERTRECORDS_REQUEST_93E12369, rowCount); List errorMessages = new ArrayList<>(); long valueCount = 0; for (int i = 0; i < insertRecordsRequest.getDataTypesList().size(); i++) { @@ -118,9 +125,11 @@ public static void validateInsertRecordsRequest(InsertRecordsRequest insertRecor List values = insertRecordsRequest.getValuesList().get(i); if (measurements.size() != dataTypes.size() || values.size() != dataTypes.size()) { throw new IllegalArgumentException( - "Each insertRecords row should have the same number of measurements, data types and values"); + RestMessages + .EXCEPTION_EACH_INSERTRECORDS_ROW_SHOULD_HAVE_THE_SAME_NUMBER_OF_MEASUREMENTS_DATA_TYPES_AND_VALUES_AD58AEF2); } - RequestLimitChecker.checkColumnCount("insertRecords request", measurements.size()); + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_INSERTRECORDS_REQUEST_93E12369, measurements.size()); valueCount += values.size(); for (int c = 0; c < insertRecordsRequest.getDataTypesList().get(i).size(); c++) { String dataType = insertRecordsRequest.getDataTypesList().get(i).get(c); @@ -135,7 +144,8 @@ public static void validateInsertRecordsRequest(InsertRecordsRequest insertRecor } } } - RequestLimitChecker.checkValueCount("insertRecords request", valueCount); + RequestLimitChecker.checkValueCount( + RestMessages.MESSAGE_INSERTRECORDS_REQUEST_93E12369, valueCount); if (!errorMessages.isEmpty()) { throw new RuntimeException(String.join(RestMessages.ERROR_MESSAGE_SEPARATOR, errorMessages)); } diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java index 86e37b07aeab1..48a78b9f866cd 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java @@ -28,6 +28,7 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; @@ -36,6 +37,8 @@ import java.io.InputStream; import java.lang.reflect.Proxy; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; @@ -47,21 +50,27 @@ public class RequestSizeLimitFilterTest { private IoTDBRestServiceConfig config; private long originalMaxBodySize; + private long originalMemoryLimit; @Before public void setUp() { config = IoTDBRestServiceDescriptor.getInstance().getConfig(); originalMaxBodySize = config.getRestMaxRequestBodySizeInBytes(); + originalMemoryLimit = config.getRestRequestBodyMemoryLimitInBytes(); + RestRequestBodyMemoryManager.resetForTest(); } @After public void tearDown() { config.setRestMaxRequestBodySizeInBytes(originalMaxBodySize); + config.setRestRequestBodyMemoryLimitInBytes(originalMemoryLimit); + RestRequestBodyMemoryManager.resetForTest(); } @Test public void testAbortContentLengthOverLimit() { config.setRestMaxRequestBodySizeInBytes(4); + config.setRestRequestBodyMemoryLimitInBytes(10); TestRequestContext context = TestRequestContext.withLength(5); new RequestSizeLimitFilter().filter(context.proxy()); @@ -72,6 +81,7 @@ public void testAbortContentLengthOverLimit() { @Test public void testRejectStreamOverLimit() throws IOException { config.setRestMaxRequestBodySizeInBytes(4); + config.setRestRequestBodyMemoryLimitInBytes(10); TestRequestContext context = TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); @@ -81,6 +91,67 @@ public void testRejectStreamOverLimit() throws IOException { WebApplicationException exception = assertThrows(WebApplicationException.class, () -> context.entityStream().readAllBytes()); assertPayloadTooLarge(exception.getResponse(), 4); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testAbortContentLengthOverMemoryLimit() { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestRequestBodyMemoryLimitInBytes(4); + TestRequestContext context = TestRequestContext.withLength(5); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertMemoryQuotaExceeded(context.abortedResponse(), 4); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testRejectStreamOverMemoryLimit() throws IOException { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestRequestBodyMemoryLimitInBytes(4); + TestRequestContext context = + TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertNull(context.abortedResponse()); + WebApplicationException exception = + assertThrows(WebApplicationException.class, () -> context.entityStream().readAllBytes()); + assertMemoryQuotaExceeded(exception.getResponse(), 4); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testDisabledMemoryLimitDoesNotReserveMemory() throws IOException { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestRequestBodyMemoryLimitInBytes(0); + TestRequestContext context = + TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertNull(context.abortedResponse()); + assertEquals( + "12345", new String(context.entityStream().readAllBytes(), StandardCharsets.UTF_8)); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testReleaseMemoryOnResponse() { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestRequestBodyMemoryLimitInBytes(5); + TestRequestContext context = TestRequestContext.withLength(4); + RequestSizeLimitFilter filter = new RequestSizeLimitFilter(); + + filter.filter(context.proxy()); + + assertNull(context.abortedResponse()); + assertEquals(4, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + + new RequestBodyMemoryReleaseFilter().filter(context.proxy(), responseContext()); + + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); } private static void assertPayloadTooLarge(Response response, long maxBodySize) { @@ -96,11 +167,36 @@ private static void assertPayloadTooLarge(Response response, long maxBodySize) { status.getMessage()); } + private static void assertMemoryQuotaExceeded(Response response, long memoryLimit) { + assertEquals(503, response.getStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + assertTrue(response.getEntity() instanceof ExecutionStatus); + + ExecutionStatus status = (ExecutionStatus) response.getEntity(); + assertEquals(Integer.valueOf(503), status.getCode()); + assertEquals( + String.format( + RestMessages.MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9, + memoryLimit), + status.getMessage()); + } + + private static ContainerResponseContext responseContext() { + return (ContainerResponseContext) + Proxy.newProxyInstance( + ContainerResponseContext.class.getClassLoader(), + new Class[] {ContainerResponseContext.class}, + (proxy, method, args) -> { + throw new UnsupportedOperationException(method.getName()); + }); + } + private static class TestRequestContext { private final int contentLength; private final AtomicReference entityStream; private final AtomicReference abortedResponse = new AtomicReference<>(); + private final Map properties = new HashMap<>(); private TestRequestContext(int contentLength, InputStream entityStream) { this.contentLength = contentLength; @@ -132,6 +228,14 @@ private ContainerRequestContext proxy() { case "abortWith": abortedResponse.set((Response) args[0]); return null; + case "getProperty": + return properties.get((String) args[0]); + case "setProperty": + properties.put((String) args[0], args[1]); + return null; + case "removeProperty": + properties.remove((String) args[0]); + return null; default: throw new UnsupportedOperationException(method.getName()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index 3cb24a5a2c32e..a8ad0d7298eec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -65,6 +65,9 @@ public class IoTDBRestServiceConfig { /** Maximum accepted REST request body size in bytes. */ private long restMaxRequestBodySizeInBytes = 16 * 1024 * 1024L; + /** Maximum REST request body bytes reserved across concurrent requests. */ + private long restRequestBodyMemoryLimitInBytes = 64 * 1024 * 1024L; + /** Maximum row count accepted by a single REST write request. */ private int restMaxInsertRows = 100000; @@ -205,6 +208,14 @@ public void setRestMaxRequestBodySizeInBytes(long restMaxRequestBodySizeInBytes) this.restMaxRequestBodySizeInBytes = restMaxRequestBodySizeInBytes; } + public long getRestRequestBodyMemoryLimitInBytes() { + return restRequestBodyMemoryLimitInBytes; + } + + public void setRestRequestBodyMemoryLimitInBytes(long restRequestBodyMemoryLimitInBytes) { + this.restRequestBodyMemoryLimitInBytes = restRequestBodyMemoryLimitInBytes; + } + public int getRestMaxInsertRows() { return restMaxInsertRows; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index 769887a92d68b..176db7d112db5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -98,6 +98,11 @@ private void loadProps(TrimProperties trimProperties) { trimProperties.getProperty( "rest_max_request_body_size_in_bytes", Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); + conf.setRestRequestBodyMemoryLimitInBytes( + Long.parseLong( + trimProperties.getProperty( + "rest_request_body_memory_limit_in_bytes", + Long.toString(conf.getRestRequestBodyMemoryLimitInBytes())))); conf.setRestMaxInsertRows( Integer.parseInt( trimProperties.getProperty( diff --git a/iotdb-core/datanode/src/test/resources/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/iotdb-common.properties index 77989acc6e572..19b4a81e185df 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-common.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-common.properties @@ -36,6 +36,9 @@ enable_rest_service=true # Maximum REST request body size in bytes # rest_max_request_body_size_in_bytes=16777216 +# Maximum total REST request body bytes reserved across concurrent requests +# rest_request_body_memory_limit_in_bytes=67108864 + # Maximum rows accepted by a single REST write request # rest_max_insert_rows=100000 diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties index 9ee85d4d993fd..961911d010e22 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-system.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -54,6 +54,9 @@ enable_rest_service=true # Maximum REST request body size in bytes # rest_max_request_body_size_in_bytes=16777216 +# Maximum total REST request body bytes reserved across concurrent requests +# rest_request_body_memory_limit_in_bytes=67108864 + # Maximum rows accepted by a single REST write request # rest_max_insert_rows=100000 diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index c6530ca8356bd..df002cd7b4692 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -617,6 +617,11 @@ rest_query_default_row_size_limit=10000 # Datatype: long rest_max_request_body_size_in_bytes=16777216 +# Maximum total REST request body bytes reserved across concurrent requests. Set to 0 or a negative value to disable the limit. +# effectiveMode: restart +# Datatype: long +rest_request_body_memory_limit_in_bytes=67108864 + # Maximum rows accepted by a single REST write request. Set to 0 or a negative value to disable the limit. # effectiveMode: restart # Datatype: int From 8e126adbe53bc5f85b90bbea5ce9f5605bed708f Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 22 Jul 2026 12:14:53 +0800 Subject: [PATCH 5/7] Rename REST concurrent body size config --- .../filter/RequestSizeLimitFilter.java | 4 +++- .../filter/RequestSizeLimitFilterTest.java | 20 ++++++++++--------- .../db/conf/rest/IoTDBRestServiceConfig.java | 14 +++++++------ .../conf/rest/IoTDBRestServiceDescriptor.java | 6 +++--- .../test/resources/iotdb-common.properties | 4 ++-- .../test/resources/iotdb-system.properties | 4 ++-- .../conf/iotdb-system.properties.template | 4 ++-- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java index 71dd92858453e..782ef57b710eb 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java @@ -45,7 +45,9 @@ public void filter(ContainerRequestContext requestContext) { long maxBodySize = IoTDBRestServiceDescriptor.getInstance().getConfig().getRestMaxRequestBodySizeInBytes(); long memoryLimit = - IoTDBRestServiceDescriptor.getInstance().getConfig().getRestRequestBodyMemoryLimitInBytes(); + IoTDBRestServiceDescriptor.getInstance() + .getConfig() + .getRestMaxTotalConcurrentRequestBodySizeInBytes(); if (maxBodySize <= 0 && memoryLimit <= 0) { return; } diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java index 48a78b9f866cd..8b76820335fc5 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java @@ -50,27 +50,29 @@ public class RequestSizeLimitFilterTest { private IoTDBRestServiceConfig config; private long originalMaxBodySize; - private long originalMemoryLimit; + private long originalMaxTotalConcurrentRequestBodySize; @Before public void setUp() { config = IoTDBRestServiceDescriptor.getInstance().getConfig(); originalMaxBodySize = config.getRestMaxRequestBodySizeInBytes(); - originalMemoryLimit = config.getRestRequestBodyMemoryLimitInBytes(); + originalMaxTotalConcurrentRequestBodySize = + config.getRestMaxTotalConcurrentRequestBodySizeInBytes(); RestRequestBodyMemoryManager.resetForTest(); } @After public void tearDown() { config.setRestMaxRequestBodySizeInBytes(originalMaxBodySize); - config.setRestRequestBodyMemoryLimitInBytes(originalMemoryLimit); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes( + originalMaxTotalConcurrentRequestBodySize); RestRequestBodyMemoryManager.resetForTest(); } @Test public void testAbortContentLengthOverLimit() { config.setRestMaxRequestBodySizeInBytes(4); - config.setRestRequestBodyMemoryLimitInBytes(10); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(10); TestRequestContext context = TestRequestContext.withLength(5); new RequestSizeLimitFilter().filter(context.proxy()); @@ -81,7 +83,7 @@ public void testAbortContentLengthOverLimit() { @Test public void testRejectStreamOverLimit() throws IOException { config.setRestMaxRequestBodySizeInBytes(4); - config.setRestRequestBodyMemoryLimitInBytes(10); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(10); TestRequestContext context = TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); @@ -97,7 +99,7 @@ public void testRejectStreamOverLimit() throws IOException { @Test public void testAbortContentLengthOverMemoryLimit() { config.setRestMaxRequestBodySizeInBytes(10); - config.setRestRequestBodyMemoryLimitInBytes(4); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(4); TestRequestContext context = TestRequestContext.withLength(5); new RequestSizeLimitFilter().filter(context.proxy()); @@ -109,7 +111,7 @@ public void testAbortContentLengthOverMemoryLimit() { @Test public void testRejectStreamOverMemoryLimit() throws IOException { config.setRestMaxRequestBodySizeInBytes(10); - config.setRestRequestBodyMemoryLimitInBytes(4); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(4); TestRequestContext context = TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); @@ -125,7 +127,7 @@ public void testRejectStreamOverMemoryLimit() throws IOException { @Test public void testDisabledMemoryLimitDoesNotReserveMemory() throws IOException { config.setRestMaxRequestBodySizeInBytes(10); - config.setRestRequestBodyMemoryLimitInBytes(0); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(0); TestRequestContext context = TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); @@ -140,7 +142,7 @@ public void testDisabledMemoryLimitDoesNotReserveMemory() throws IOException { @Test public void testReleaseMemoryOnResponse() { config.setRestMaxRequestBodySizeInBytes(10); - config.setRestRequestBodyMemoryLimitInBytes(5); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(5); TestRequestContext context = TestRequestContext.withLength(4); RequestSizeLimitFilter filter = new RequestSizeLimitFilter(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index a8ad0d7298eec..276756a141cd7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -65,8 +65,8 @@ public class IoTDBRestServiceConfig { /** Maximum accepted REST request body size in bytes. */ private long restMaxRequestBodySizeInBytes = 16 * 1024 * 1024L; - /** Maximum REST request body bytes reserved across concurrent requests. */ - private long restRequestBodyMemoryLimitInBytes = 64 * 1024 * 1024L; + /** Maximum total in-flight REST request body size in bytes across concurrent requests. */ + private long restMaxTotalConcurrentRequestBodySizeInBytes = 64 * 1024 * 1024L; /** Maximum row count accepted by a single REST write request. */ private int restMaxInsertRows = 100000; @@ -208,12 +208,14 @@ public void setRestMaxRequestBodySizeInBytes(long restMaxRequestBodySizeInBytes) this.restMaxRequestBodySizeInBytes = restMaxRequestBodySizeInBytes; } - public long getRestRequestBodyMemoryLimitInBytes() { - return restRequestBodyMemoryLimitInBytes; + public long getRestMaxTotalConcurrentRequestBodySizeInBytes() { + return restMaxTotalConcurrentRequestBodySizeInBytes; } - public void setRestRequestBodyMemoryLimitInBytes(long restRequestBodyMemoryLimitInBytes) { - this.restRequestBodyMemoryLimitInBytes = restRequestBodyMemoryLimitInBytes; + public void setRestMaxTotalConcurrentRequestBodySizeInBytes( + long restMaxTotalConcurrentRequestBodySizeInBytes) { + this.restMaxTotalConcurrentRequestBodySizeInBytes = + restMaxTotalConcurrentRequestBodySizeInBytes; } public int getRestMaxInsertRows() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index 176db7d112db5..6ab0377373e34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -98,11 +98,11 @@ private void loadProps(TrimProperties trimProperties) { trimProperties.getProperty( "rest_max_request_body_size_in_bytes", Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); - conf.setRestRequestBodyMemoryLimitInBytes( + conf.setRestMaxTotalConcurrentRequestBodySizeInBytes( Long.parseLong( trimProperties.getProperty( - "rest_request_body_memory_limit_in_bytes", - Long.toString(conf.getRestRequestBodyMemoryLimitInBytes())))); + "rest_max_total_concurrent_request_body_size_in_bytes", + Long.toString(conf.getRestMaxTotalConcurrentRequestBodySizeInBytes())))); conf.setRestMaxInsertRows( Integer.parseInt( trimProperties.getProperty( diff --git a/iotdb-core/datanode/src/test/resources/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/iotdb-common.properties index 19b4a81e185df..bfdb5a5da81e0 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-common.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-common.properties @@ -36,8 +36,8 @@ enable_rest_service=true # Maximum REST request body size in bytes # rest_max_request_body_size_in_bytes=16777216 -# Maximum total REST request body bytes reserved across concurrent requests -# rest_request_body_memory_limit_in_bytes=67108864 +# Maximum total in-flight REST request body size in bytes across concurrent requests +# rest_max_total_concurrent_request_body_size_in_bytes=67108864 # Maximum rows accepted by a single REST write request # rest_max_insert_rows=100000 diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties index 961911d010e22..952bea33095a5 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-system.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -54,8 +54,8 @@ enable_rest_service=true # Maximum REST request body size in bytes # rest_max_request_body_size_in_bytes=16777216 -# Maximum total REST request body bytes reserved across concurrent requests -# rest_request_body_memory_limit_in_bytes=67108864 +# Maximum total in-flight REST request body size in bytes across concurrent requests +# rest_max_total_concurrent_request_body_size_in_bytes=67108864 # Maximum rows accepted by a single REST write request # rest_max_insert_rows=100000 diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index df002cd7b4692..6512ef6f8076c 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -617,10 +617,10 @@ rest_query_default_row_size_limit=10000 # Datatype: long rest_max_request_body_size_in_bytes=16777216 -# Maximum total REST request body bytes reserved across concurrent requests. Set to 0 or a negative value to disable the limit. +# Maximum total in-flight REST request body size in bytes across concurrent requests. Set to 0 or a negative value to disable the limit. # effectiveMode: restart # Datatype: long -rest_request_body_memory_limit_in_bytes=67108864 +rest_max_total_concurrent_request_body_size_in_bytes=67108864 # Maximum rows accepted by a single REST write request. Set to 0 or a negative value to disable the limit. # effectiveMode: restart From ba3ab8be4c699ba1585d10fd6dabd0b6e0725b53 Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 22 Jul 2026 12:37:23 +0800 Subject: [PATCH 6/7] Align REST body budget with thrift buffer budget --- .../filter/RequestSizeLimitFilterTest.java | 2 +- .../iotdb/db/conf/DataNodeMemoryConfig.java | 67 ++++++++++++++----- .../db/conf/rest/IoTDBRestServiceConfig.java | 5 +- .../conf/rest/IoTDBRestServiceDescriptor.java | 11 ++- .../db/conf/DataNodeMemoryConfigTest.java | 56 ++++++++++++++++ .../test/resources/iotdb-common.properties | 4 +- .../test/resources/iotdb-system.properties | 4 +- .../conf/iotdb-system.properties.template | 4 +- 8 files changed, 127 insertions(+), 26 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java index 8b76820335fc5..4af98197de834 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java @@ -127,7 +127,7 @@ public void testRejectStreamOverMemoryLimit() throws IOException { @Test public void testDisabledMemoryLimitDoesNotReserveMemory() throws IOException { config.setRestMaxRequestBodySizeInBytes(10); - config.setRestMaxTotalConcurrentRequestBodySizeInBytes(0); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(-1); TestRequestContext context = TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java index 6612732441e66..c6a3f6146399c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java @@ -36,6 +36,9 @@ public class DataNodeMemoryConfig { public static final String SCHEMA_CACHE = "SchemaCache"; public static final String SCHEMA_REGION = "SchemaRegion"; public static final String PARTITION_CACHE = "PartitionCache"; + private static final String DATANODE_MEMORY_PROPORTION = "datanode_memory_proportion"; + private static final String STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_PROPORTION = + "storage_query_schema_consensus_free_memory_proportion"; /** Reject proportion for system */ private double rejectProportion = 0.8; @@ -149,26 +152,27 @@ public class DataNodeMemoryConfig { /** The memory manager of direct Buffer */ private MemoryManager directBufferMemoryManager; + public static long getDefaultAutoResizingBufferMemorySizeInBytes() { + return Runtime.getRuntime().maxMemory() / 20; + } + + public static long calculateAutoResizingBufferMemorySizeInBytes(TrimProperties properties) { + return calculateAutoResizingBufferMemorySizeInBytes( + getMemoryAllocateProportion(properties, false)); + } + public void init(TrimProperties properties) { // on heap memory - String memoryAllocateProportion = properties.getProperty("datanode_memory_proportion", null); + String memoryAllocateProportion = getMemoryAllocateProportion(properties, true); // Get global memory manager here - if (memoryAllocateProportion == null) { - memoryAllocateProportion = - properties.getProperty("storage_query_schema_consensus_free_memory_proportion"); - if (memoryAllocateProportion != null) { - LOGGER.warn( - DataNodeMiscMessages - .MISC_LOG_THE_PARAMETER_STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_51C9A377); - } - } long storageEngineMemorySize = Runtime.getRuntime().maxMemory() * 3 / 10; long queryEngineMemorySize = Runtime.getRuntime().maxMemory() * 3 / 10; long schemaEngineMemorySize = Runtime.getRuntime().maxMemory() / 10; long consensusMemorySize = Runtime.getRuntime().maxMemory() / 10; long pipeMemorySize = Runtime.getRuntime().maxMemory() / 10; - long autoResizingBufferMemorySize = Runtime.getRuntime().maxMemory() / 20; + long autoResizingBufferMemorySize = + calculateAutoResizingBufferMemorySizeInBytes(memoryAllocateProportion); if (memoryAllocateProportion != null) { String[] proportions = memoryAllocateProportion.split(":"); int proportionSum = 0; @@ -190,11 +194,6 @@ public void init(TrimProperties properties) { if (proportions.length >= 6) { pipeMemorySize = maxMemoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum; - autoResizingBufferMemorySize = - maxMemoryAvailable - * Integer.parseInt(proportions[proportions.length - 1].trim()) - / proportionSum - / 2; } else { pipeMemorySize = (maxMemoryAvailable @@ -260,6 +259,42 @@ public void init(TrimProperties properties) { "DirectBuffer", totalDirectBufferMemorySizeLimit); } + private static String getMemoryAllocateProportion( + TrimProperties properties, boolean warnDeprecatedProperty) { + String memoryAllocateProportion = properties.getProperty(DATANODE_MEMORY_PROPORTION, null); + if (memoryAllocateProportion == null) { + memoryAllocateProportion = + properties.getProperty(STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_PROPORTION); + if (memoryAllocateProportion != null && warnDeprecatedProperty) { + LOGGER.warn( + DataNodeMiscMessages + .MISC_LOG_THE_PARAMETER_STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_51C9A377); + } + } + return memoryAllocateProportion; + } + + private static long calculateAutoResizingBufferMemorySizeInBytes( + String memoryAllocateProportion) { + long autoResizingBufferMemorySize = getDefaultAutoResizingBufferMemorySizeInBytes(); + if (memoryAllocateProportion != null) { + String[] proportions = memoryAllocateProportion.split(":"); + int proportionSum = 0; + for (String proportion : proportions) { + proportionSum += Integer.parseInt(proportion.trim()); + } + long maxMemoryAvailable = Runtime.getRuntime().maxMemory(); + if (proportionSum != 0 && proportions.length >= 6) { + autoResizingBufferMemorySize = + maxMemoryAvailable + * Integer.parseInt(proportions[proportions.length - 1].trim()) + / proportionSum + / 2; + } + } + return autoResizingBufferMemorySize; + } + @SuppressWarnings("squid:S3518") private void initSchemaMemoryAllocate( MemoryManager schemaEngineMemoryManager, TrimProperties properties) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index 276756a141cd7..2f202ceaa4ea2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.conf.rest; +import org.apache.iotdb.db.conf.DataNodeMemoryConfig; + public class IoTDBRestServiceConfig { /** If the enableRestService is true, we will start REST Service. */ @@ -66,7 +68,8 @@ public class IoTDBRestServiceConfig { private long restMaxRequestBodySizeInBytes = 16 * 1024 * 1024L; /** Maximum total in-flight REST request body size in bytes across concurrent requests. */ - private long restMaxTotalConcurrentRequestBodySizeInBytes = 64 * 1024 * 1024L; + private long restMaxTotalConcurrentRequestBodySizeInBytes = + DataNodeMemoryConfig.getDefaultAutoResizingBufferMemorySizeInBytes(); /** Maximum row count accepted by a single REST write request. */ private int restMaxInsertRows = 100000; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index 6ab0377373e34..9129b96198fa6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.conf.TrimProperties; +import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.rpc.RpcSslUtils; @@ -98,11 +99,17 @@ private void loadProps(TrimProperties trimProperties) { trimProperties.getProperty( "rest_max_request_body_size_in_bytes", Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); - conf.setRestMaxTotalConcurrentRequestBodySizeInBytes( + long defaultMaxTotalConcurrentRequestBodySizeInBytes = + DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(trimProperties); + long maxTotalConcurrentRequestBodySizeInBytes = Long.parseLong( trimProperties.getProperty( "rest_max_total_concurrent_request_body_size_in_bytes", - Long.toString(conf.getRestMaxTotalConcurrentRequestBodySizeInBytes())))); + Long.toString(defaultMaxTotalConcurrentRequestBodySizeInBytes))); + conf.setRestMaxTotalConcurrentRequestBodySizeInBytes( + maxTotalConcurrentRequestBodySizeInBytes == 0 + ? defaultMaxTotalConcurrentRequestBodySizeInBytes + : maxTotalConcurrentRequestBodySizeInBytes); conf.setRestMaxInsertRows( Integer.parseInt( trimProperties.getProperty( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java new file mode 100644 index 0000000000000..464e3c52e8c6f --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.conf; + +import org.apache.iotdb.commons.conf.TrimProperties; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class DataNodeMemoryConfigTest { + + @Test + public void testDefaultAutoResizingBufferMemorySize() { + assertEquals( + Runtime.getRuntime().maxMemory() / 20, + DataNodeMemoryConfig.getDefaultAutoResizingBufferMemorySizeInBytes()); + } + + @Test + public void testCalculateAutoResizingBufferMemorySizeWithDataNodeMemoryProportion() { + TrimProperties properties = new TrimProperties(); + properties.setProperty("datanode_memory_proportion", "1:1:1:1:1:5"); + + assertEquals( + Runtime.getRuntime().maxMemory() / 4, + DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(properties)); + } + + @Test + public void testCalculateAutoResizingBufferMemorySizeWithDeprecatedMemoryProportion() { + TrimProperties properties = new TrimProperties(); + properties.setProperty("storage_query_schema_consensus_free_memory_proportion", "1:1:1:1:1:2"); + + assertEquals( + Runtime.getRuntime().maxMemory() / 7, + DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(properties)); + } +} diff --git a/iotdb-core/datanode/src/test/resources/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/iotdb-common.properties index bfdb5a5da81e0..8cf1da6bb7c9d 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-common.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-common.properties @@ -36,8 +36,8 @@ enable_rest_service=true # Maximum REST request body size in bytes # rest_max_request_body_size_in_bytes=16777216 -# Maximum total in-flight REST request body size in bytes across concurrent requests -# rest_max_total_concurrent_request_body_size_in_bytes=67108864 +# Maximum total in-flight REST request body size in bytes across concurrent requests. When set to 0, use the same budget as AutoResizingBuffer memory control. Set to a negative value to disable the limit. +# rest_max_total_concurrent_request_body_size_in_bytes=0 # Maximum rows accepted by a single REST write request # rest_max_insert_rows=100000 diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties index 952bea33095a5..d83782023118f 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-system.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -54,8 +54,8 @@ enable_rest_service=true # Maximum REST request body size in bytes # rest_max_request_body_size_in_bytes=16777216 -# Maximum total in-flight REST request body size in bytes across concurrent requests -# rest_max_total_concurrent_request_body_size_in_bytes=67108864 +# Maximum total in-flight REST request body size in bytes across concurrent requests. When set to 0, use the same budget as AutoResizingBuffer memory control. Set to a negative value to disable the limit. +# rest_max_total_concurrent_request_body_size_in_bytes=0 # Maximum rows accepted by a single REST write request # rest_max_insert_rows=100000 diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 6512ef6f8076c..a8957866ddd25 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -617,10 +617,10 @@ rest_query_default_row_size_limit=10000 # Datatype: long rest_max_request_body_size_in_bytes=16777216 -# Maximum total in-flight REST request body size in bytes across concurrent requests. Set to 0 or a negative value to disable the limit. +# Maximum total in-flight REST request body size in bytes across concurrent requests. When set to 0, use the same budget as AutoResizingBuffer memory control. Set to a negative value to disable the limit. # effectiveMode: restart # Datatype: long -rest_max_total_concurrent_request_body_size_in_bytes=67108864 +rest_max_total_concurrent_request_body_size_in_bytes=0 # Maximum rows accepted by a single REST write request. Set to 0 or a negative value to disable the limit. # effectiveMode: restart From 2ee9e3325861941ccc62a1a47ef88e8061ee7e28 Mon Sep 17 00:00:00 2001 From: HTHou Date: Wed, 22 Jul 2026 16:33:05 +0800 Subject: [PATCH 7/7] Address REST request limit review comments --- .../apache/iotdb/rest/i18n/RestMessages.java | 11 +- .../apache/iotdb/rest/i18n/RestMessages.java | 11 +- .../filter/RequestSizeLimitFilter.java | 5 +- .../filter/RequestSizeLimitFilterTest.java | 7 +- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 5 + .../conf/rest/IoTDBRestServiceDescriptor.java | 110 ++++++++++++------ .../apache/iotdb/db/conf/PropertiesTest.java | 63 ++++++++++ .../conf/iotdb-system.properties.template | 12 +- 8 files changed, 173 insertions(+), 51 deletions(-) diff --git a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java index c570dd6660d7f..5de9a55267932 100644 --- a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java @@ -78,11 +78,14 @@ public final class RestMessages { public static final String ERROR_MESSAGE_SEPARATOR = ","; // --- RequestSizeLimitFilter --- - public static final String MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412 = - "REST request body exceeds limit %d bytes"; public static final String - MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9 = - "REST request body memory quota exceeds limit %d bytes"; + MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6 = + "REST request body exceeds limit %d bytes. Use SET CONFIGURATION" + + " 'rest_max_request_body_size_in_bytes'='' to increase it."; + public static final String + MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD = + "REST request body memory quota exceeds limit %d bytes. Use SET CONFIGURATION" + + " 'rest_max_total_concurrent_request_body_size_in_bytes'='' to increase it."; // --- RequestLimitChecker --- public static final String MESSAGE_INSERTTABLET_REQUEST_8647CA58 = "insertTablet request"; diff --git a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java index 2ca782f52bb15..c69db29571f89 100644 --- a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java @@ -76,11 +76,14 @@ public final class RestMessages { public static final String ERROR_MESSAGE_SEPARATOR = "，"; // --- RequestSizeLimitFilter --- - public static final String MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412 = - "REST 请求体超过限制 %d 字节"; public static final String - MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9 = - "REST 请求体内存配额超过限制 %d 字节"; + MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6 = + "REST 请求体超过限制 %d 字节。请执行 SET CONFIGURATION" + + " 'rest_max_request_body_size_in_bytes'='' 增大限制。"; + public static final String + MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD = + "REST 请求体内存配额超过限制 %d 字节。请执行 SET CONFIGURATION" + + " 'rest_max_total_concurrent_request_body_size_in_bytes'='' 增大配额。"; // --- RequestLimitChecker --- public static final String MESSAGE_INSERTTABLET_REQUEST_8647CA58 = "insertTablet 请求"; diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java index 782ef57b710eb..e16d8ab0c6ba4 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java @@ -91,7 +91,8 @@ private static Response buildPayloadTooLargeResponse(long maxBodySize) { .code(PAYLOAD_TOO_LARGE_STATUS_CODE) .message( String.format( - RestMessages.MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412, + RestMessages + .MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6, maxBodySize))) .build(); } @@ -105,7 +106,7 @@ private static Response buildMemoryQuotaExceededResponse(long memoryLimit) { .message( String.format( RestMessages - .MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9, + .MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD, memoryLimit))) .build(); } diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java index 4af98197de834..1311b8f1045a7 100644 --- a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java @@ -165,7 +165,9 @@ private static void assertPayloadTooLarge(Response response, long maxBodySize) { assertEquals(Integer.valueOf(413), status.getCode()); assertEquals( String.format( - RestMessages.MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_D9F9B412, maxBodySize), + RestMessages + .MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6, + maxBodySize), status.getMessage()); } @@ -178,7 +180,8 @@ private static void assertMemoryQuotaExceeded(Response response, long memoryLimi assertEquals(Integer.valueOf(503), status.getCode()); assertEquals( String.format( - RestMessages.MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_7F2994D9, + RestMessages + .MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD, memoryLimit), status.getMessage()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 1ab0c22bbf622..ecd3d73a3e30d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -39,6 +39,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig; import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig; import org.apache.iotdb.consensus.config.IoTConsensusV2Config; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheLoadStrategy; @@ -2319,6 +2320,9 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) // update trusted_uri_pattern loadTrustedUriPattern(properties); + // update REST runtime limit config + IoTDBRestServiceDescriptor.getInstance().loadHotModifiedProps(properties); + // update cache_eviction_memory_computation_threshold conf.setCacheEvictionMemoryComputationThreshold( Integer.parseInt( @@ -2380,6 +2384,7 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) ConfigurationFileUtils.updateAppliedProperties(properties, true); // Overwrite the keys whose setters above may have rewritten, so `show configuration` // displays the effective values rather than the raw file values. + IoTDBRestServiceDescriptor.getInstance().overwriteAppliedRuntimeLimitProperties(); overlayEffectiveConfigurationValues(); } catch (Exception e) { if (e instanceof InterruptedException) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index 9129b96198fa6..7cfdc0dd471f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.conf.rest; import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.conf.TrimProperties; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; @@ -41,6 +42,16 @@ public class IoTDBRestServiceDescriptor { private static final Logger logger = LoggerFactory.getLogger(IoTDBRestServiceDescriptor.class); + private static final String REST_QUERY_DEFAULT_ROW_SIZE_LIMIT = + "rest_query_default_row_size_limit"; + private static final String REST_MAX_REQUEST_BODY_SIZE_IN_BYTES = + "rest_max_request_body_size_in_bytes"; + private static final String REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES = + "rest_max_total_concurrent_request_body_size_in_bytes"; + private static final String REST_MAX_INSERT_ROWS = "rest_max_insert_rows"; + private static final String REST_MAX_INSERT_COLUMNS = "rest_max_insert_columns"; + private static final String REST_MAX_INSERT_VALUES = "rest_max_insert_values"; + private final IoTDBRestServiceConfig conf = new IoTDBRestServiceConfig(); protected IoTDBRestServiceDescriptor() { @@ -89,39 +100,7 @@ private void loadProps(TrimProperties trimProperties) { Integer.parseInt( trimProperties.getProperty( "rest_service_port", Integer.toString(conf.getRestServicePort())))); - conf.setRestQueryDefaultRowSizeLimit( - Integer.parseInt( - trimProperties.getProperty( - "rest_query_default_row_size_limit", - Integer.toString(conf.getRestQueryDefaultRowSizeLimit())))); - conf.setRestMaxRequestBodySizeInBytes( - Long.parseLong( - trimProperties.getProperty( - "rest_max_request_body_size_in_bytes", - Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); - long defaultMaxTotalConcurrentRequestBodySizeInBytes = - DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(trimProperties); - long maxTotalConcurrentRequestBodySizeInBytes = - Long.parseLong( - trimProperties.getProperty( - "rest_max_total_concurrent_request_body_size_in_bytes", - Long.toString(defaultMaxTotalConcurrentRequestBodySizeInBytes))); - conf.setRestMaxTotalConcurrentRequestBodySizeInBytes( - maxTotalConcurrentRequestBodySizeInBytes == 0 - ? defaultMaxTotalConcurrentRequestBodySizeInBytes - : maxTotalConcurrentRequestBodySizeInBytes); - conf.setRestMaxInsertRows( - Integer.parseInt( - trimProperties.getProperty( - "rest_max_insert_rows", Integer.toString(conf.getRestMaxInsertRows())))); - conf.setRestMaxInsertColumns( - Integer.parseInt( - trimProperties.getProperty( - "rest_max_insert_columns", Integer.toString(conf.getRestMaxInsertColumns())))); - conf.setRestMaxInsertValues( - Long.parseLong( - trimProperties.getProperty( - "rest_max_insert_values", Long.toString(conf.getRestMaxInsertValues())))); + loadRuntimeLimitProps(trimProperties); conf.setEnableSwagger( Boolean.parseBoolean( trimProperties.getProperty( @@ -147,6 +126,71 @@ private void loadProps(TrimProperties trimProperties) { "idle_timeout_in_seconds", Integer.toString(conf.getIdleTimeoutInSeconds())))); } + public synchronized void loadHotModifiedProps(TrimProperties trimProperties) { + loadRuntimeLimitProps(trimProperties); + } + + public synchronized void overwriteAppliedRuntimeLimitProperties() { + overlayRuntimeLimitProperties(); + } + + private void loadRuntimeLimitProps(TrimProperties trimProperties) { + conf.setRestQueryDefaultRowSizeLimit( + Integer.parseInt( + trimProperties.getProperty( + REST_QUERY_DEFAULT_ROW_SIZE_LIMIT, + Integer.toString(conf.getRestQueryDefaultRowSizeLimit())))); + conf.setRestMaxRequestBodySizeInBytes( + Long.parseLong( + trimProperties.getProperty( + REST_MAX_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); + conf.setRestMaxTotalConcurrentRequestBodySizeInBytes( + parseMaxTotalConcurrentRequestBodySizeInBytes(trimProperties)); + conf.setRestMaxInsertRows( + Integer.parseInt( + trimProperties.getProperty( + REST_MAX_INSERT_ROWS, Integer.toString(conf.getRestMaxInsertRows())))); + conf.setRestMaxInsertColumns( + Integer.parseInt( + trimProperties.getProperty( + REST_MAX_INSERT_COLUMNS, Integer.toString(conf.getRestMaxInsertColumns())))); + conf.setRestMaxInsertValues( + Long.parseLong( + trimProperties.getProperty( + REST_MAX_INSERT_VALUES, Long.toString(conf.getRestMaxInsertValues())))); + overlayRuntimeLimitProperties(); + } + + private long parseMaxTotalConcurrentRequestBodySizeInBytes(TrimProperties trimProperties) { + long maxTotalConcurrentRequestBodySizeInBytes = + Long.parseLong( + trimProperties.getProperty( + REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxTotalConcurrentRequestBodySizeInBytes()))); + return maxTotalConcurrentRequestBodySizeInBytes == 0 + ? DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(trimProperties) + : maxTotalConcurrentRequestBodySizeInBytes; + } + + private void overlayRuntimeLimitProperties() { + ConfigurationFileUtils.updateAppliedProperties( + REST_QUERY_DEFAULT_ROW_SIZE_LIMIT, + Integer.toString(conf.getRestQueryDefaultRowSizeLimit())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxRequestBodySizeInBytes())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxTotalConcurrentRequestBodySizeInBytes())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_INSERT_ROWS, Integer.toString(conf.getRestMaxInsertRows())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_INSERT_COLUMNS, Integer.toString(conf.getRestMaxInsertColumns())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_INSERT_VALUES, Long.toString(conf.getRestMaxInsertValues())); + } + /** * get props url location * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java index f4daffbf303b9..24cb6a1b8c522 100755 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java @@ -19,8 +19,11 @@ package org.apache.iotdb.db.conf; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.TrimProperties; import org.apache.iotdb.commons.utils.RegionMigrationFileRemoveRateLimiter; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import com.tngtech.archunit.core.domain.JavaClasses; import com.tngtech.archunit.core.importer.ClassFileImporter; @@ -65,6 +68,66 @@ public void testHotReloadRegionMigrationFileRemoveSpeedLimit() throws Exception } } + @Test + public void testHotReloadRestRuntimeLimitProperties() throws Exception { + IoTDBRestServiceConfig restConfig = IoTDBRestServiceDescriptor.getInstance().getConfig(); + int originalQueryDefaultRowSizeLimit = restConfig.getRestQueryDefaultRowSizeLimit(); + long originalMaxRequestBodySizeInBytes = restConfig.getRestMaxRequestBodySizeInBytes(); + long originalMaxTotalConcurrentRequestBodySizeInBytes = + restConfig.getRestMaxTotalConcurrentRequestBodySizeInBytes(); + int originalMaxInsertRows = restConfig.getRestMaxInsertRows(); + int originalMaxInsertColumns = restConfig.getRestMaxInsertColumns(); + long originalMaxInsertValues = restConfig.getRestMaxInsertValues(); + int originalRestServicePort = restConfig.getRestServicePort(); + boolean originalEnableHttps = restConfig.isEnableHttps(); + try { + TrimProperties properties = new TrimProperties(); + properties.setProperty("rest_query_default_row_size_limit", "123"); + properties.setProperty("rest_max_request_body_size_in_bytes", "456"); + properties.setProperty("rest_max_total_concurrent_request_body_size_in_bytes", "789"); + properties.setProperty("rest_max_insert_rows", "11"); + properties.setProperty("rest_max_insert_columns", "12"); + properties.setProperty("rest_max_insert_values", "13"); + properties.setProperty("rest_service_port", Integer.toString(originalRestServicePort + 1)); + properties.setProperty("enable_https", Boolean.toString(!originalEnableHttps)); + + IoTDBDescriptor.getInstance().loadHotModifiedProps(properties); + + Assert.assertEquals(123, restConfig.getRestQueryDefaultRowSizeLimit()); + Assert.assertEquals(456, restConfig.getRestMaxRequestBodySizeInBytes()); + Assert.assertEquals(789, restConfig.getRestMaxTotalConcurrentRequestBodySizeInBytes()); + Assert.assertEquals(11, restConfig.getRestMaxInsertRows()); + Assert.assertEquals(12, restConfig.getRestMaxInsertColumns()); + Assert.assertEquals(13, restConfig.getRestMaxInsertValues()); + Assert.assertEquals(originalRestServicePort, restConfig.getRestServicePort()); + Assert.assertEquals(originalEnableHttps, restConfig.isEnableHttps()); + + properties.setProperty("rest_max_total_concurrent_request_body_size_in_bytes", "0"); + properties.setProperty("datanode_memory_proportion", "1:1:1:1:1:5"); + + IoTDBDescriptor.getInstance().loadHotModifiedProps(properties); + + Assert.assertEquals( + Runtime.getRuntime().maxMemory() / 4, + restConfig.getRestMaxTotalConcurrentRequestBodySizeInBytes()); + Assert.assertEquals( + Long.toString(Runtime.getRuntime().maxMemory() / 4), + ConfigurationFileUtils.getAppliedProperties() + .get("rest_max_total_concurrent_request_body_size_in_bytes")); + } finally { + restConfig.setRestQueryDefaultRowSizeLimit(originalQueryDefaultRowSizeLimit); + restConfig.setRestMaxRequestBodySizeInBytes(originalMaxRequestBodySizeInBytes); + restConfig.setRestMaxTotalConcurrentRequestBodySizeInBytes( + originalMaxTotalConcurrentRequestBodySizeInBytes); + restConfig.setRestMaxInsertRows(originalMaxInsertRows); + restConfig.setRestMaxInsertColumns(originalMaxInsertColumns); + restConfig.setRestMaxInsertValues(originalMaxInsertValues); + restConfig.setRestServicePort(originalRestServicePort); + restConfig.setEnableHttps(originalEnableHttps); + IoTDBRestServiceDescriptor.getInstance().overwriteAppliedRuntimeLimitProperties(); + } + } + @Test public void PropertiesWithSpace() { IoTDBDescriptor descriptor = IoTDBDescriptor.getInstance(); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index d9a50bea52f82..dd3ba2d295ecb 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -610,32 +610,32 @@ enable_swagger=false # The maximum row limit for REST and Grafana query responses. # The request rowLimit/row_limit value cannot exceed this limit. # A non-positive value is invalid and falls back to the default (10000); it no longer means unlimited. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: int rest_query_default_row_size_limit=10000 # Maximum REST request body size in bytes. Set to 0 or a negative value to disable the limit. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: long rest_max_request_body_size_in_bytes=16777216 # Maximum total in-flight REST request body size in bytes across concurrent requests. When set to 0, use the same budget as AutoResizingBuffer memory control. Set to a negative value to disable the limit. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: long rest_max_total_concurrent_request_body_size_in_bytes=0 # Maximum rows accepted by a single REST write request. Set to 0 or a negative value to disable the limit. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: int rest_max_insert_rows=100000 # Maximum columns accepted by a single REST write request. Set to 0 or a negative value to disable the limit. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: int rest_max_insert_columns=1024 # Maximum values accepted by a single REST write request. Set to 0 or a negative value to disable the limit. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: long rest_max_insert_values=1000000