diff --git a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java index 882bb16c55808..5de9a55267932 100644 --- a/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/en/org/apache/iotdb/rest/i18n/RestMessages.java @@ -77,6 +77,49 @@ public final class RestMessages { "The number of values in the %dth row is not equal to the data_types size"; public static final String ERROR_MESSAGE_SEPARATOR = ","; + // --- RequestSizeLimitFilter --- + public static final String + MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6 = + "REST request body exceeds limit %d bytes. Use SET CONFIGURATION" + + " 'rest_max_request_body_size_in_bytes'='' to increase it."; + public static final String + MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD = + "REST request body memory quota exceeds limit %d bytes. Use SET CONFIGURATION" + + " 'rest_max_total_concurrent_request_body_size_in_bytes'='' to increase it."; + + // --- RequestLimitChecker --- + public static final String MESSAGE_INSERTTABLET_REQUEST_8647CA58 = "insertTablet request"; + public static final String MESSAGE_INSERTRECORDS_REQUEST_93E12369 = "insertRecords request"; + public static final String MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C = + "table insertTablet request"; + public static final String EXCEPTION_ARG_ROW_COUNT_ARG_EXCEEDS_LIMIT_ARG_EE427E4B = + "%s row count %d exceeds limit %d"; + public static final String EXCEPTION_ARG_COLUMN_COUNT_ARG_EXCEEDS_LIMIT_ARG_DEE9637E = + "%s column count %d exceeds limit %d"; + public static final String EXCEPTION_ARG_VALUE_COUNT_ARG_EXCEEDS_LIMIT_ARG_77F95703 = + "%s value count %d exceeds limit %d"; + + // --- RequestValidationHandler --- + public static final String + EXCEPTION_MEASUREMENTS_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_FF715FA9 = + "measurements and dataTypes should have the same size"; + public static final String EXCEPTION_VALUES_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_5BC1D604 = + "values and dataTypes should have the same size"; + public static final String + EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD = + "Each value column should have the same size as timestamps"; + public static final String + EXCEPTION_MEASUREMENTS_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_8526F19A = + "measurements and data_types should have the same size"; + public static final String EXCEPTION_VALUES_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_0BAE701D = + "values and data_types should have the same size"; + public static final String + EXCEPTION_DEVICES_TIMESTAMPS_MEASUREMENTS_LIST_DATA_TYPES_LIST_AND_VALUES_LIST_SHOULD_HAVE_THE_SAME_SIZE_5983AAC2 = + "devices, timestamps, measurements_list, data_types_list and values_list should have the same size"; + public static final String + EXCEPTION_EACH_INSERTRECORDS_ROW_SHOULD_HAVE_THE_SAME_NUMBER_OF_MEASUREMENTS_DATA_TYPES_AND_VALUES_AD58AEF2 = + "Each insertRecords row should have the same number of measurements, data types and values"; + private RestMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java index cb443b6ec974f..c69db29571f89 100644 --- a/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java +++ b/external-service-impl/rest/src/main/i18n/zh/org/apache/iotdb/rest/i18n/RestMessages.java @@ -75,6 +75,49 @@ public final class RestMessages { "第 %d 行的 values 数量与 data_types 数量不相等"; public static final String ERROR_MESSAGE_SEPARATOR = "，"; + // --- RequestSizeLimitFilter --- + public static final String + MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6 = + "REST 请求体超过限制 %d 字节。请执行 SET CONFIGURATION" + + " 'rest_max_request_body_size_in_bytes'='' 增大限制。"; + public static final String + MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD = + "REST 请求体内存配额超过限制 %d 字节。请执行 SET CONFIGURATION" + + " 'rest_max_total_concurrent_request_body_size_in_bytes'='' 增大配额。"; + + // --- RequestLimitChecker --- + public static final String MESSAGE_INSERTTABLET_REQUEST_8647CA58 = "insertTablet 请求"; + public static final String MESSAGE_INSERTRECORDS_REQUEST_93E12369 = "insertRecords 请求"; + public static final String MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C = + "table insertTablet 请求"; + public static final String EXCEPTION_ARG_ROW_COUNT_ARG_EXCEEDS_LIMIT_ARG_EE427E4B = + "%s 行数 %d 超过限制 %d"; + public static final String EXCEPTION_ARG_COLUMN_COUNT_ARG_EXCEEDS_LIMIT_ARG_DEE9637E = + "%s 列数 %d 超过限制 %d"; + public static final String EXCEPTION_ARG_VALUE_COUNT_ARG_EXCEEDS_LIMIT_ARG_77F95703 = + "%s value 数 %d 超过限制 %d"; + + // --- RequestValidationHandler --- + public static final String + EXCEPTION_MEASUREMENTS_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_FF715FA9 = + "measurements 和 dataTypes 的数量应相同"; + public static final String EXCEPTION_VALUES_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_5BC1D604 = + "values 和 dataTypes 的数量应相同"; + public static final String + EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD = + "每个 value 列的数量应与 timestamps 数量相同"; + public static final String + EXCEPTION_MEASUREMENTS_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_8526F19A = + "measurements 和 data_types 的数量应相同"; + public static final String EXCEPTION_VALUES_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_0BAE701D = + "values 和 data_types 的数量应相同"; + public static final String + EXCEPTION_DEVICES_TIMESTAMPS_MEASUREMENTS_LIST_DATA_TYPES_LIST_AND_VALUES_LIST_SHOULD_HAVE_THE_SAME_SIZE_5983AAC2 = + "devices、timestamps、measurements_list、data_types_list 和 values_list 的数量应相同"; + public static final String + EXCEPTION_EACH_INSERTRECORDS_ROW_SHOULD_HAVE_THE_SAME_NUMBER_OF_MEASUREMENTS_DATA_TYPES_AND_VALUES_AD58AEF2 = + "每个 insertRecords 行中的 measurements、data types 和 values 数量应相同"; + private RestMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java new file mode 100644 index 0000000000000..ab56ec4bb0344 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/exception/RequestLimitExceededException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.exception; + +public class RequestLimitExceededException extends IllegalArgumentException { + + public RequestLimitExceededException(String message) { + super(message); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestBodyMemoryReleaseFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestBodyMemoryReleaseFilter.java new file mode 100644 index 0000000000000..301de4f3b1c63 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestBodyMemoryReleaseFilter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.container.ContainerResponseFilter; +import jakarta.ws.rs.ext.Provider; + +@Provider +public class RequestBodyMemoryReleaseFilter implements ContainerResponseFilter { + + @Override + public void filter( + ContainerRequestContext requestContext, ContainerResponseContext responseContext) { + RestRequestBodyMemoryManager.releaseReservation(requestContext); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java new file mode 100644 index 0000000000000..e16d8ab0c6ba4 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilter.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.model.ExecutionStatus; + +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.container.PreMatching; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.ext.Provider; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +@Provider +@PreMatching +public class RequestSizeLimitFilter implements ContainerRequestFilter { + + private static final int PAYLOAD_TOO_LARGE_STATUS_CODE = 413; + private static final int SERVICE_UNAVAILABLE_STATUS_CODE = 503; + + @Override + public void filter(ContainerRequestContext requestContext) { + long maxBodySize = + IoTDBRestServiceDescriptor.getInstance().getConfig().getRestMaxRequestBodySizeInBytes(); + long memoryLimit = + IoTDBRestServiceDescriptor.getInstance() + .getConfig() + .getRestMaxTotalConcurrentRequestBodySizeInBytes(); + if (maxBodySize <= 0 && memoryLimit <= 0) { + return; + } + + int contentLength = requestContext.getLength(); + if (maxBodySize > 0 && contentLength > maxBodySize) { + requestContext.abortWith(buildPayloadTooLargeResponse(maxBodySize)); + return; + } + + RestRequestBodyMemoryManager.Reservation memoryReservation = + RestRequestBodyMemoryManager.newReservation(memoryLimit); + long memoryReservedByContentLength = 0; + if (contentLength > 0 && memoryReservation.isEnabled()) { + if (!memoryReservation.reserve(contentLength)) { + memoryReservation.close(); + requestContext.abortWith(buildMemoryQuotaExceededResponse(memoryLimit)); + return; + } + memoryReservedByContentLength = contentLength; + RestRequestBodyMemoryManager.registerReservation(requestContext, memoryReservation); + } + + requestContext.setEntityStream( + new LimitedInputStream( + requestContext.getEntityStream(), + maxBodySize, + memoryLimit, + memoryReservation, + memoryReservedByContentLength)); + if (memoryReservation.isEnabled() && memoryReservedByContentLength == 0) { + RestRequestBodyMemoryManager.registerReservation(requestContext, memoryReservation); + } + } + + private static Response buildPayloadTooLargeResponse(long maxBodySize) { + return Response.status(PAYLOAD_TOO_LARGE_STATUS_CODE) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity( + new ExecutionStatus() + .code(PAYLOAD_TOO_LARGE_STATUS_CODE) + .message( + String.format( + RestMessages + .MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6, + maxBodySize))) + .build(); + } + + private static Response buildMemoryQuotaExceededResponse(long memoryLimit) { + return Response.status(SERVICE_UNAVAILABLE_STATUS_CODE) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity( + new ExecutionStatus() + .code(SERVICE_UNAVAILABLE_STATUS_CODE) + .message( + String.format( + RestMessages + .MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD, + memoryLimit))) + .build(); + } + + private static class LimitedInputStream extends FilterInputStream { + + private final long maxBodySize; + private final long memoryLimit; + private final RestRequestBodyMemoryManager.Reservation memoryReservation; + private long memoryCoveredBytes; + private long bytesRead; + + private LimitedInputStream( + InputStream in, + long maxBodySize, + long memoryLimit, + RestRequestBodyMemoryManager.Reservation memoryReservation, + long memoryCoveredBytes) { + super(in); + this.maxBodySize = maxBodySize; + this.memoryLimit = memoryLimit; + this.memoryReservation = memoryReservation; + this.memoryCoveredBytes = memoryCoveredBytes; + } + + @Override + public int read() throws IOException { + int result = super.read(); + if (result != -1) { + incrementBytesRead(1); + } + return result; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int result = super.read(b, off, len); + if (result > 0) { + incrementBytesRead(result); + } + return result; + } + + private void incrementBytesRead(int increment) { + bytesRead += increment; + if (maxBodySize > 0 && bytesRead > maxBodySize) { + memoryReservation.close(); + throw new WebApplicationException(buildPayloadTooLargeResponse(maxBodySize)); + } + reserveMemoryIfNecessary(); + } + + private void reserveMemoryIfNecessary() { + if (bytesRead <= memoryCoveredBytes) { + return; + } + long sizeToReserve = bytesRead - memoryCoveredBytes; + if (!memoryReservation.reserve(sizeToReserve)) { + memoryReservation.close(); + throw new WebApplicationException(buildMemoryQuotaExceededResponse(memoryLimit)); + } + memoryCoveredBytes = bytesRead; + } + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + memoryReservation.close(); + } + } + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RestRequestBodyMemoryManager.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RestRequestBodyMemoryManager.java new file mode 100644 index 0000000000000..3b9ea6bb4b5e9 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/filter/RestRequestBodyMemoryManager.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import jakarta.ws.rs.container.ContainerRequestContext; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +final class RestRequestBodyMemoryManager { + + static final String REQUEST_BODY_MEMORY_RESERVATION_PROPERTY = + RestRequestBodyMemoryManager.class.getName() + ".requestBodyMemoryReservation"; + private static final AtomicLong RESERVED_MEMORY_IN_BYTES = new AtomicLong(); + + private RestRequestBodyMemoryManager() {} + + static Reservation newReservation(long memoryLimitInBytes) { + return new Reservation(memoryLimitInBytes); + } + + static long getReservedMemoryInBytes() { + return RESERVED_MEMORY_IN_BYTES.get(); + } + + static void resetForTest() { + RESERVED_MEMORY_IN_BYTES.set(0); + } + + static void registerReservation( + ContainerRequestContext requestContext, Reservation memoryReservation) { + if (memoryReservation.isEnabled()) { + requestContext.setProperty(REQUEST_BODY_MEMORY_RESERVATION_PROPERTY, memoryReservation); + } + } + + static void releaseReservation(ContainerRequestContext requestContext) { + Object reservation = requestContext.getProperty(REQUEST_BODY_MEMORY_RESERVATION_PROPERTY); + if (reservation instanceof Reservation) { + ((Reservation) reservation).close(); + requestContext.removeProperty(REQUEST_BODY_MEMORY_RESERVATION_PROPERTY); + } + } + + private static boolean tryReserve(long sizeInBytes, long memoryLimitInBytes) { + if (sizeInBytes <= 0 || memoryLimitInBytes <= 0) { + return true; + } + + while (true) { + long currentReservedBytes = RESERVED_MEMORY_IN_BYTES.get(); + if (currentReservedBytes > memoryLimitInBytes - sizeInBytes) { + return false; + } + if (RESERVED_MEMORY_IN_BYTES.compareAndSet( + currentReservedBytes, currentReservedBytes + sizeInBytes)) { + return true; + } + } + } + + static final class Reservation implements AutoCloseable { + + private final long memoryLimitInBytes; + private final AtomicBoolean released = new AtomicBoolean(); + private long reservedMemoryInBytes; + + private Reservation(long memoryLimitInBytes) { + this.memoryLimitInBytes = memoryLimitInBytes; + } + + boolean isEnabled() { + return memoryLimitInBytes > 0; + } + + synchronized boolean reserve(long sizeInBytes) { + if (sizeInBytes <= 0 || !isEnabled()) { + return true; + } + if (released.get()) { + return false; + } + if (!tryReserve(sizeInBytes, memoryLimitInBytes)) { + return false; + } + reservedMemoryInBytes += sizeInBytes; + return true; + } + + @Override + public synchronized void close() { + if (released.compareAndSet(false, true)) { + long reservedBytes = reservedMemoryInBytes; + reservedMemoryInBytes = 0; + if (reservedBytes > 0) { + RESERVED_MEMORY_IN_BYTES.addAndGet(-reservedBytes); + } + } + } + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java new file mode 100644 index 0000000000000..0ec64b9b125c5 --- /dev/null +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/handler/RequestLimitChecker.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.handler; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; + +public class RequestLimitChecker { + + private RequestLimitChecker() {} + + public static void checkRowCount(String requestName, int rowCount) { + int maxRows = getConfig().getRestMaxInsertRows(); + if (maxRows > 0 && rowCount > maxRows) { + throw new RequestLimitExceededException( + String.format( + RestMessages.EXCEPTION_ARG_ROW_COUNT_ARG_EXCEEDS_LIMIT_ARG_EE427E4B, + requestName, + rowCount, + maxRows)); + } + } + + public static void checkColumnCount(String requestName, int columnCount) { + int maxColumns = getConfig().getRestMaxInsertColumns(); + if (maxColumns > 0 && columnCount > maxColumns) { + throw new RequestLimitExceededException( + String.format( + RestMessages.EXCEPTION_ARG_COLUMN_COUNT_ARG_EXCEEDS_LIMIT_ARG_DEE9637E, + requestName, + columnCount, + maxColumns)); + } + } + + public static void checkValueCount(String requestName, long valueCount) { + long maxValues = getConfig().getRestMaxInsertValues(); + if (maxValues > 0 && valueCount > maxValues) { + throw new RequestLimitExceededException( + String.format( + RestMessages.EXCEPTION_ARG_VALUE_COUNT_ARG_EXCEEDS_LIMIT_ARG_77F95703, + requestName, + valueCount, + maxValues)); + } + } + + private static IoTDBRestServiceConfig getConfig() { + return IoTDBRestServiceDescriptor.getInstance().getConfig(); + } +} diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java index e3b83f51b7d96..65af32084ab69 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/ExceptionHandler.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -46,7 +47,10 @@ private ExceptionHandler() {} public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -92,4 +96,8 @@ public static ExecutionStatus tryCatchException(Exception e) { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java index 8bab22426fe7e..3e6edabdeffad 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/handler/RequestValidationHandler.java @@ -18,6 +18,7 @@ package org.apache.iotdb.rest.protocol.table.v1.handler; import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest; import org.apache.iotdb.rest.protocol.table.v1.model.SQL; @@ -64,6 +65,15 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR errorMessages.add(RestMessages.VALUES_AND_TIMESTAMPS_SIZE_MISMATCH); } + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getColumnNames().size(); + RequestLimitChecker.checkRowCount( + RestMessages.MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C, rowCount); + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C, columnCount); + RequestLimitChecker.checkValueCount( + RestMessages.MESSAGE_TABLE_INSERTTABLET_REQUEST_573D371C, (long) rowCount * columnCount); + for (int i = 0; i < insertTabletRequest.getDataTypes().size(); i++) { String dataType = insertTabletRequest.getDataTypes().get(i); if (isDataType(dataType)) { diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java index a04ae7a16b4c3..20249f4f682c9 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java @@ -113,7 +113,9 @@ public Response executeQueryInternal( return res; } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -136,7 +138,9 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext) return executeQueryInternal(sql, statement, clientSession, relationSqlParser); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -178,7 +182,9 @@ public Response insertTablet( return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) @@ -234,7 +240,9 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex } return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java index 1f43f2fc25ffa..62f3da75f639e 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/ExceptionHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.v1.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -45,7 +46,10 @@ private ExceptionHandler() {} public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -88,4 +92,8 @@ public static ExecutionStatus tryCatchException(Exception e) { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java index 9a01bffb249dd..a001a06e929b8 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/handler/RequestValidationHandler.java @@ -18,12 +18,14 @@ package org.apache.iotdb.rest.protocol.v1.handler; import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.v1.model.ExpressionRequest; import org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest; import org.apache.iotdb.rest.protocol.v1.model.SQL; import org.apache.tsfile.external.commons.lang3.Validate; +import java.util.List; import java.util.Objects; public class RequestValidationHandler { @@ -42,9 +44,36 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR Objects.requireNonNull( insertTabletRequest.getIsAligned(), RestMessages.IS_ALIGNED_CAMEL_NOT_NULL); Objects.requireNonNull(insertTabletRequest.getDeviceId(), RestMessages.DEVICE_ID_NOT_NULL); + Objects.requireNonNull( + insertTabletRequest.getMeasurements(), RestMessages.MEASUREMENTS_NOT_NULL); Objects.requireNonNull( insertTabletRequest.getDataTypes(), RestMessages.DATA_TYPES_CAMEL_NOT_NULL); Objects.requireNonNull(insertTabletRequest.getValues(), RestMessages.VALUES_NOT_NULL); + + if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException( + RestMessages.EXCEPTION_MEASUREMENTS_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_FF715FA9); + } + if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException( + RestMessages.EXCEPTION_VALUES_AND_DATATYPES_SHOULD_HAVE_THE_SAME_SIZE_5BC1D604); + } + + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getMeasurements().size(); + RequestLimitChecker.checkRowCount(RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, rowCount); + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, columnCount); + RequestLimitChecker.checkValueCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, (long) rowCount * columnCount); + + for (List column : insertTabletRequest.getValues()) { + if (column.size() != rowCount) { + throw new IllegalArgumentException( + RestMessages + .EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD); + } + } } public static void validateExpressionRequest(ExpressionRequest expressionRequest) { diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java index dcc17c43a59d9..94ecdc61e4d61 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/GrafanaApiServiceImpl.java @@ -142,7 +142,9 @@ public Response variables(SQL sql, SecurityContext securityContext) { queryExecution, statement, defaultQueryRowLimit); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -219,7 +221,9 @@ public Response expression(ExpressionRequest expressionRequest, SecurityContext } } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -284,7 +288,9 @@ public Response node(List requestBody, SecurityContext securityContext) return QueryDataSetHandler.fillGrafanaNodesResult(null, defaultQueryRowLimit); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java index 4939e10a41246..a5d7302920d21 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v1/impl/RestApiServiceImpl.java @@ -154,7 +154,9 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex .build(); } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -237,7 +239,9 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext) } } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -308,7 +312,9 @@ public Response insertTablet( .message(result.status.getMessage())) .build(); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java index 39780877239e3..9ed5111c7e7ce 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/ExceptionHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; import org.apache.iotdb.rest.protocol.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; @@ -45,7 +46,10 @@ private ExceptionHandler() {} public static ExecutionStatus tryCatchException(Exception e) { ExecutionStatus responseResult = new ExecutionStatus(); - if (e instanceof QueryProcessException) { + if (e instanceof RequestLimitExceededException) { + responseResult.setMessage(e.getMessage()); + responseResult.setCode(413); + } else if (e instanceof QueryProcessException) { responseResult.setMessage(e.getMessage()); responseResult.setCode(((QueryProcessException) e).getErrorCode()); } else if (e instanceof DatabaseNotSetException) { @@ -88,4 +92,8 @@ public static ExecutionStatus tryCatchException(Exception e) { LOGGER.warn(e.getMessage(), e); return responseResult; } + + public static int getHttpStatus(Exception e) { + return e instanceof RequestLimitExceededException ? 413 : Status.OK.getStatusCode(); + } } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java index 01cc4ea7ae216..d0fd4218ee7a4 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/handler/RequestValidationHandler.java @@ -18,6 +18,7 @@ package org.apache.iotdb.rest.protocol.v2.handler; import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.handler.RequestLimitChecker; import org.apache.iotdb.rest.protocol.v2.model.ExpressionRequest; import org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest; import org.apache.iotdb.rest.protocol.v2.model.InsertTabletRequest; @@ -57,6 +58,28 @@ public static void validateInsertTabletRequest(InsertTabletRequest insertTabletR Objects.requireNonNull( insertTabletRequest.getMeasurements(), RestMessages.MEASUREMENTS_NOT_NULL); Objects.requireNonNull(insertTabletRequest.getValues(), RestMessages.VALUES_NOT_NULL); + if (insertTabletRequest.getMeasurements().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException( + RestMessages.EXCEPTION_MEASUREMENTS_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_8526F19A); + } + if (insertTabletRequest.getValues().size() != insertTabletRequest.getDataTypes().size()) { + throw new IllegalArgumentException( + RestMessages.EXCEPTION_VALUES_AND_DATA_TYPES_SHOULD_HAVE_THE_SAME_SIZE_0BAE701D); + } + int rowCount = insertTabletRequest.getTimestamps().size(); + int columnCount = insertTabletRequest.getMeasurements().size(); + RequestLimitChecker.checkRowCount(RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, rowCount); + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, columnCount); + RequestLimitChecker.checkValueCount( + RestMessages.MESSAGE_INSERTTABLET_REQUEST_8647CA58, (long) rowCount * columnCount); + for (List column : insertTabletRequest.getValues()) { + if (column.size() != rowCount) { + throw new IllegalArgumentException( + RestMessages + .EXCEPTION_EACH_VALUE_COLUMN_SHOULD_HAVE_THE_SAME_SIZE_AS_TIMESTAMPS_523598BD); + } + } List errorMessages = new ArrayList<>(); String device = insertTabletRequest.getDevice(); for (int i = 0; i < insertTabletRequest.getMeasurements().size(); i++) { @@ -82,10 +105,32 @@ public static void validateInsertRecordsRequest(InsertRecordsRequest insertRecor Objects.requireNonNull(insertRecordsRequest.getValuesList(), RestMessages.VALUES_LIST_NOT_NULL); Objects.requireNonNull( insertRecordsRequest.getMeasurementsList(), RestMessages.MEASUREMENTS_LIST_NOT_NULL); + int rowCount = insertRecordsRequest.getDevices().size(); + if (insertRecordsRequest.getTimestamps().size() != rowCount + || insertRecordsRequest.getMeasurementsList().size() != rowCount + || insertRecordsRequest.getDataTypesList().size() != rowCount + || insertRecordsRequest.getValuesList().size() != rowCount) { + throw new IllegalArgumentException( + RestMessages + .EXCEPTION_DEVICES_TIMESTAMPS_MEASUREMENTS_LIST_DATA_TYPES_LIST_AND_VALUES_LIST_SHOULD_HAVE_THE_SAME_SIZE_5983AAC2); + } + RequestLimitChecker.checkRowCount( + RestMessages.MESSAGE_INSERTRECORDS_REQUEST_93E12369, rowCount); List errorMessages = new ArrayList<>(); + long valueCount = 0; for (int i = 0; i < insertRecordsRequest.getDataTypesList().size(); i++) { String device = insertRecordsRequest.getDevices().get(i); List measurements = insertRecordsRequest.getMeasurementsList().get(i); + List dataTypes = insertRecordsRequest.getDataTypesList().get(i); + List values = insertRecordsRequest.getValuesList().get(i); + if (measurements.size() != dataTypes.size() || values.size() != dataTypes.size()) { + throw new IllegalArgumentException( + RestMessages + .EXCEPTION_EACH_INSERTRECORDS_ROW_SHOULD_HAVE_THE_SAME_NUMBER_OF_MEASUREMENTS_DATA_TYPES_AND_VALUES_AD58AEF2); + } + RequestLimitChecker.checkColumnCount( + RestMessages.MESSAGE_INSERTRECORDS_REQUEST_93E12369, measurements.size()); + valueCount += values.size(); for (int c = 0; c < insertRecordsRequest.getDataTypesList().get(i).size(); c++) { String dataType = insertRecordsRequest.getDataTypesList().get(i).get(c); String measurement = measurements.get(c); @@ -99,6 +144,8 @@ public static void validateInsertRecordsRequest(InsertRecordsRequest insertRecor } } } + RequestLimitChecker.checkValueCount( + RestMessages.MESSAGE_INSERTRECORDS_REQUEST_93E12369, valueCount); if (!errorMessages.isEmpty()) { throw new RuntimeException(String.join(RestMessages.ERROR_MESSAGE_SEPARATOR, errorMessages)); } diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java index 32834a5bced0d..e6e7cdf3a5dc2 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/GrafanaApiServiceImpl.java @@ -142,7 +142,9 @@ public Response variables(SQL sql, SecurityContext securityContext) { queryExecution, statement, defaultQueryRowLimit); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -219,7 +221,9 @@ public Response expression(ExpressionRequest expressionRequest, SecurityContext } } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); @@ -284,7 +288,9 @@ public Response node(List requestBody, SecurityContext securityContext) return QueryDataSetHandler.fillGrafanaNodesResult(null, defaultQueryRowLimit); } } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { if (queryId != null) { COORDINATOR.cleanupQueryExecution(queryId); diff --git a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java index cfe572ce9e046..f6a2ef7b54c03 100644 --- a/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java +++ b/external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java @@ -185,7 +185,9 @@ public Response executeFastLastQueryStatement( } catch (Exception e) { finish = true; t = e; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long endTime = System.nanoTime(); long costTime = endTime - startTime; @@ -284,7 +286,9 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex return responseGenerateHelper(result); } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; if (statement != null) { @@ -368,7 +372,9 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext) } } catch (Exception e) { finish = true; - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(statement) @@ -418,7 +424,9 @@ public Response insertRecords( return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertRowsStatement) @@ -472,7 +480,9 @@ public Response insertTablet( false); return responseGenerateHelper(result); } catch (Exception e) { - return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build(); + return Response.status(ExceptionHandler.getHttpStatus(e)) + .entity(ExceptionHandler.tryCatchException(e)) + .build(); } finally { long costTime = System.nanoTime() - startTime; Optional.ofNullable(insertTabletStatement) diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java new file mode 100644 index 0000000000000..1311b8f1045a7 --- /dev/null +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/filter/RequestSizeLimitFilterTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.filter; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.i18n.RestMessages; +import org.apache.iotdb.rest.protocol.model.ExecutionStatus; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class RequestSizeLimitFilterTest { + + private IoTDBRestServiceConfig config; + private long originalMaxBodySize; + private long originalMaxTotalConcurrentRequestBodySize; + + @Before + public void setUp() { + config = IoTDBRestServiceDescriptor.getInstance().getConfig(); + originalMaxBodySize = config.getRestMaxRequestBodySizeInBytes(); + originalMaxTotalConcurrentRequestBodySize = + config.getRestMaxTotalConcurrentRequestBodySizeInBytes(); + RestRequestBodyMemoryManager.resetForTest(); + } + + @After + public void tearDown() { + config.setRestMaxRequestBodySizeInBytes(originalMaxBodySize); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes( + originalMaxTotalConcurrentRequestBodySize); + RestRequestBodyMemoryManager.resetForTest(); + } + + @Test + public void testAbortContentLengthOverLimit() { + config.setRestMaxRequestBodySizeInBytes(4); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(10); + TestRequestContext context = TestRequestContext.withLength(5); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertPayloadTooLarge(context.abortedResponse(), 4); + } + + @Test + public void testRejectStreamOverLimit() throws IOException { + config.setRestMaxRequestBodySizeInBytes(4); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(10); + TestRequestContext context = + TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertNull(context.abortedResponse()); + WebApplicationException exception = + assertThrows(WebApplicationException.class, () -> context.entityStream().readAllBytes()); + assertPayloadTooLarge(exception.getResponse(), 4); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testAbortContentLengthOverMemoryLimit() { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(4); + TestRequestContext context = TestRequestContext.withLength(5); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertMemoryQuotaExceeded(context.abortedResponse(), 4); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testRejectStreamOverMemoryLimit() throws IOException { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(4); + TestRequestContext context = + TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertNull(context.abortedResponse()); + WebApplicationException exception = + assertThrows(WebApplicationException.class, () -> context.entityStream().readAllBytes()); + assertMemoryQuotaExceeded(exception.getResponse(), 4); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testDisabledMemoryLimitDoesNotReserveMemory() throws IOException { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(-1); + TestRequestContext context = + TestRequestContext.withStream("12345".getBytes(StandardCharsets.UTF_8)); + + new RequestSizeLimitFilter().filter(context.proxy()); + + assertNull(context.abortedResponse()); + assertEquals( + "12345", new String(context.entityStream().readAllBytes(), StandardCharsets.UTF_8)); + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + @Test + public void testReleaseMemoryOnResponse() { + config.setRestMaxRequestBodySizeInBytes(10); + config.setRestMaxTotalConcurrentRequestBodySizeInBytes(5); + TestRequestContext context = TestRequestContext.withLength(4); + RequestSizeLimitFilter filter = new RequestSizeLimitFilter(); + + filter.filter(context.proxy()); + + assertNull(context.abortedResponse()); + assertEquals(4, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + + new RequestBodyMemoryReleaseFilter().filter(context.proxy(), responseContext()); + + assertEquals(0, RestRequestBodyMemoryManager.getReservedMemoryInBytes()); + } + + private static void assertPayloadTooLarge(Response response, long maxBodySize) { + assertEquals(413, response.getStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + assertTrue(response.getEntity() instanceof ExecutionStatus); + + ExecutionStatus status = (ExecutionStatus) response.getEntity(); + assertEquals(Integer.valueOf(413), status.getCode()); + assertEquals( + String.format( + RestMessages + .MESSAGE_REST_REQUEST_BODY_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_424392C6, + maxBodySize), + status.getMessage()); + } + + private static void assertMemoryQuotaExceeded(Response response, long memoryLimit) { + assertEquals(503, response.getStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + assertTrue(response.getEntity() instanceof ExecutionStatus); + + ExecutionStatus status = (ExecutionStatus) response.getEntity(); + assertEquals(Integer.valueOf(503), status.getCode()); + assertEquals( + String.format( + RestMessages + .MESSAGE_REST_REQUEST_BODY_MEMORY_QUOTA_EXCEEDS_LIMIT_ARG_BYTES_USE_SET_CONFIGURATION_REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES_BYTES_TO_INCREASE_IT_F07B9DDD, + memoryLimit), + status.getMessage()); + } + + private static ContainerResponseContext responseContext() { + return (ContainerResponseContext) + Proxy.newProxyInstance( + ContainerResponseContext.class.getClassLoader(), + new Class[] {ContainerResponseContext.class}, + (proxy, method, args) -> { + throw new UnsupportedOperationException(method.getName()); + }); + } + + private static class TestRequestContext { + + private final int contentLength; + private final AtomicReference entityStream; + private final AtomicReference abortedResponse = new AtomicReference<>(); + private final Map properties = new HashMap<>(); + + private TestRequestContext(int contentLength, InputStream entityStream) { + this.contentLength = contentLength; + this.entityStream = new AtomicReference<>(entityStream); + } + + private static TestRequestContext withLength(int contentLength) { + return new TestRequestContext(contentLength, InputStream.nullInputStream()); + } + + private static TestRequestContext withStream(byte[] body) { + return new TestRequestContext(-1, new ByteArrayInputStream(body)); + } + + private ContainerRequestContext proxy() { + return (ContainerRequestContext) + Proxy.newProxyInstance( + ContainerRequestContext.class.getClassLoader(), + new Class[] {ContainerRequestContext.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "getLength": + return contentLength; + case "getEntityStream": + return entityStream.get(); + case "setEntityStream": + entityStream.set((InputStream) args[0]); + return null; + case "abortWith": + abortedResponse.set((Response) args[0]); + return null; + case "getProperty": + return properties.get((String) args[0]); + case "setProperty": + properties.put((String) args[0], args[1]); + return null; + case "removeProperty": + properties.remove((String) args[0]); + return null; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } + + private InputStream entityStream() { + return entityStream.get(); + } + + private Response abortedResponse() { + return abortedResponse.get(); + } + } +} diff --git a/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java new file mode 100644 index 0000000000000..810de95c191eb --- /dev/null +++ b/external-service-impl/rest/src/test/java/org/apache/iotdb/rest/protocol/handler/RequestValidationLimitTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.rest.protocol.handler; + +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; +import org.apache.iotdb.rest.protocol.exception.RequestLimitExceededException; +import org.apache.iotdb.rest.protocol.v1.model.InsertTabletRequest; +import org.apache.iotdb.rest.protocol.v2.model.InsertRecordsRequest; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.iotdb.rest.protocol.v1.handler.RequestValidationHandler.validateInsertTabletRequest; +import static org.apache.iotdb.rest.protocol.v2.handler.RequestValidationHandler.validateInsertRecordsRequest; + +public class RequestValidationLimitTest { + + private IoTDBRestServiceConfig config; + private int originalMaxInsertRows; + private int originalMaxInsertColumns; + private long originalMaxInsertValues; + + @Before + public void setUp() { + config = IoTDBRestServiceDescriptor.getInstance().getConfig(); + originalMaxInsertRows = config.getRestMaxInsertRows(); + originalMaxInsertColumns = config.getRestMaxInsertColumns(); + originalMaxInsertValues = config.getRestMaxInsertValues(); + } + + @After + public void tearDown() { + config.setRestMaxInsertRows(originalMaxInsertRows); + config.setRestMaxInsertColumns(originalMaxInsertColumns); + config.setRestMaxInsertValues(originalMaxInsertValues); + } + + @Test(expected = RequestLimitExceededException.class) + public void testV1InsertTabletRejectsTooManyRows() { + config.setRestMaxInsertRows(2); + + InsertTabletRequest request = new InsertTabletRequest(); + request.setDeviceId("root.sg.d1"); + request.setIsAligned(false); + request.setMeasurements(Collections.singletonList("s1")); + request.setDataTypes(Collections.singletonList("INT64")); + request.setTimestamps(Arrays.asList(1L, 2L, 3L)); + request.setValues(Collections.singletonList(Arrays.asList(1L, 2L, 3L))); + + validateInsertTabletRequest(request); + } + + @Test(expected = RequestLimitExceededException.class) + public void testV2InsertRecordsRejectsTooManyValues() { + config.setRestMaxInsertRows(10); + config.setRestMaxInsertColumns(10); + config.setRestMaxInsertValues(2); + + InsertRecordsRequest request = new InsertRecordsRequest(); + request.setIsAligned(false); + request.setDevices(Arrays.asList("root.sg.d1", "root.sg.d2")); + request.setTimestamps(Arrays.asList(1L, 2L)); + request.setMeasurementsList( + Arrays.asList(Arrays.asList("s1", "s2"), Collections.singletonList("s1"))); + request.setDataTypesList( + Arrays.asList(Arrays.asList("INT64", "INT64"), Collections.singletonList("INT64"))); + request.setValuesList(Arrays.asList(Arrays.asList(1L, 2L), Collections.singletonList(3L))); + + validateInsertRecordsRequest(request); + } + + @Test(expected = RequestLimitExceededException.class) + public void testTableInsertTabletRejectsTooManyColumns() { + config.setRestMaxInsertColumns(1); + + org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest request = + new org.apache.iotdb.rest.protocol.table.v1.model.InsertTabletRequest(); + request.setDatabase("db"); + request.setTable("t1"); + request.setColumnNames(Arrays.asList("tag1", "s1")); + request.setColumnCategories(Arrays.asList("TAG", "FIELD")); + request.setDataTypes(Arrays.asList("STRING", "INT64")); + request.setTimestamps(Collections.singletonList(1L)); + request.setValues(Collections.singletonList(Arrays.asList("a", 1L))); + + org.apache.iotdb.rest.protocol.table.v1.handler.RequestValidationHandler + .validateInsertTabletRequest(request); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java index 9476e84e15e54..73c27a9b805e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java @@ -36,6 +36,9 @@ public class DataNodeMemoryConfig { public static final String SCHEMA_CACHE = "SchemaCache"; public static final String SCHEMA_REGION = "SchemaRegion"; public static final String PARTITION_CACHE = "PartitionCache"; + private static final String DATANODE_MEMORY_PROPORTION = "datanode_memory_proportion"; + private static final String STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_PROPORTION = + "storage_query_schema_consensus_free_memory_proportion"; /** Reject proportion for system */ private double rejectProportion = 0.8; @@ -152,26 +155,27 @@ public class DataNodeMemoryConfig { /** The memory manager of direct Buffer */ private MemoryManager directBufferMemoryManager; + public static long getDefaultAutoResizingBufferMemorySizeInBytes() { + return Runtime.getRuntime().maxMemory() / 20; + } + + public static long calculateAutoResizingBufferMemorySizeInBytes(TrimProperties properties) { + return calculateAutoResizingBufferMemorySizeInBytes( + getMemoryAllocateProportion(properties, false)); + } + public void init(TrimProperties properties) { // on heap memory - String memoryAllocateProportion = properties.getProperty("datanode_memory_proportion", null); + String memoryAllocateProportion = getMemoryAllocateProportion(properties, true); // Get global memory manager here - if (memoryAllocateProportion == null) { - memoryAllocateProportion = - properties.getProperty("storage_query_schema_consensus_free_memory_proportion"); - if (memoryAllocateProportion != null) { - LOGGER.warn( - DataNodeMiscMessages - .MISC_LOG_THE_PARAMETER_STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_51C9A377); - } - } long storageEngineMemorySize = Runtime.getRuntime().maxMemory() * 3 / 10; long queryEngineMemorySize = Runtime.getRuntime().maxMemory() * 3 / 10; long schemaEngineMemorySize = Runtime.getRuntime().maxMemory() / 10; long consensusMemorySize = Runtime.getRuntime().maxMemory() / 10; long pipeMemorySize = Runtime.getRuntime().maxMemory() / 10; - autoResizingBufferMemorySize = Runtime.getRuntime().maxMemory() / 20; + autoResizingBufferMemorySize = + calculateAutoResizingBufferMemorySizeInBytes(memoryAllocateProportion); if (memoryAllocateProportion != null) { String[] proportions = memoryAllocateProportion.split(":"); int proportionSum = 0; @@ -193,11 +197,6 @@ public void init(TrimProperties properties) { if (proportions.length >= 6) { pipeMemorySize = maxMemoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum; - autoResizingBufferMemorySize = - maxMemoryAvailable - * Integer.parseInt(proportions[proportions.length - 1].trim()) - / proportionSum - / 2; } else { pipeMemorySize = (maxMemoryAvailable @@ -266,6 +265,42 @@ public void activateAutoResizingBufferMemoryControl() { .setAutoResizingBufferMemoryControl(onHeapMemoryManager, autoResizingBufferMemorySize); } + private static String getMemoryAllocateProportion( + TrimProperties properties, boolean warnDeprecatedProperty) { + String memoryAllocateProportion = properties.getProperty(DATANODE_MEMORY_PROPORTION, null); + if (memoryAllocateProportion == null) { + memoryAllocateProportion = + properties.getProperty(STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_PROPORTION); + if (memoryAllocateProportion != null && warnDeprecatedProperty) { + LOGGER.warn( + DataNodeMiscMessages + .MISC_LOG_THE_PARAMETER_STORAGE_QUERY_SCHEMA_CONSENSUS_FREE_MEMORY_51C9A377); + } + } + return memoryAllocateProportion; + } + + private static long calculateAutoResizingBufferMemorySizeInBytes( + String memoryAllocateProportion) { + long autoResizingBufferMemorySize = getDefaultAutoResizingBufferMemorySizeInBytes(); + if (memoryAllocateProportion != null) { + String[] proportions = memoryAllocateProportion.split(":"); + int proportionSum = 0; + for (String proportion : proportions) { + proportionSum += Integer.parseInt(proportion.trim()); + } + long maxMemoryAvailable = Runtime.getRuntime().maxMemory(); + if (proportionSum != 0 && proportions.length >= 6) { + autoResizingBufferMemorySize = + maxMemoryAvailable + * Integer.parseInt(proportions[proportions.length - 1].trim()) + / proportionSum + / 2; + } + } + return autoResizingBufferMemorySize; + } + @SuppressWarnings("squid:S3518") private void initSchemaMemoryAllocate( MemoryManager schemaEngineMemoryManager, TrimProperties properties) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 1ab0c22bbf622..ecd3d73a3e30d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -39,6 +39,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig; import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig; import org.apache.iotdb.consensus.config.IoTConsensusV2Config; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheLoadStrategy; @@ -2319,6 +2320,9 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) // update trusted_uri_pattern loadTrustedUriPattern(properties); + // update REST runtime limit config + IoTDBRestServiceDescriptor.getInstance().loadHotModifiedProps(properties); + // update cache_eviction_memory_computation_threshold conf.setCacheEvictionMemoryComputationThreshold( Integer.parseInt( @@ -2380,6 +2384,7 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) ConfigurationFileUtils.updateAppliedProperties(properties, true); // Overwrite the keys whose setters above may have rewritten, so `show configuration` // displays the effective values rather than the raw file values. + IoTDBRestServiceDescriptor.getInstance().overwriteAppliedRuntimeLimitProperties(); overlayEffectiveConfigurationValues(); } catch (Exception e) { if (e instanceof InterruptedException) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java index b7af9dcb201a4..2f202ceaa4ea2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceConfig.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.conf.rest; +import org.apache.iotdb.db.conf.DataNodeMemoryConfig; + public class IoTDBRestServiceConfig { /** If the enableRestService is true, we will start REST Service. */ @@ -62,6 +64,22 @@ public class IoTDBRestServiceConfig { private int restQueryDefaultRowSizeLimit = 10000; + /** Maximum accepted REST request body size in bytes. */ + private long restMaxRequestBodySizeInBytes = 16 * 1024 * 1024L; + + /** Maximum total in-flight REST request body size in bytes across concurrent requests. */ + private long restMaxTotalConcurrentRequestBodySizeInBytes = + DataNodeMemoryConfig.getDefaultAutoResizingBufferMemorySizeInBytes(); + + /** Maximum row count accepted by a single REST write request. */ + private int restMaxInsertRows = 100000; + + /** Maximum column count accepted by a single REST write request. */ + private int restMaxInsertColumns = 1024; + + /** Maximum value cell count accepted by a single REST write request. */ + private long restMaxInsertValues = 1000000L; + /** Is client authentication required. */ private boolean clientAuth = false; @@ -184,4 +202,46 @@ public int getRestQueryDefaultRowSizeLimit() { public void setRestQueryDefaultRowSizeLimit(int restQueryDefaultRowSizeLimit) { this.restQueryDefaultRowSizeLimit = restQueryDefaultRowSizeLimit; } + + public long getRestMaxRequestBodySizeInBytes() { + return restMaxRequestBodySizeInBytes; + } + + public void setRestMaxRequestBodySizeInBytes(long restMaxRequestBodySizeInBytes) { + this.restMaxRequestBodySizeInBytes = restMaxRequestBodySizeInBytes; + } + + public long getRestMaxTotalConcurrentRequestBodySizeInBytes() { + return restMaxTotalConcurrentRequestBodySizeInBytes; + } + + public void setRestMaxTotalConcurrentRequestBodySizeInBytes( + long restMaxTotalConcurrentRequestBodySizeInBytes) { + this.restMaxTotalConcurrentRequestBodySizeInBytes = + restMaxTotalConcurrentRequestBodySizeInBytes; + } + + public int getRestMaxInsertRows() { + return restMaxInsertRows; + } + + public void setRestMaxInsertRows(int restMaxInsertRows) { + this.restMaxInsertRows = restMaxInsertRows; + } + + public int getRestMaxInsertColumns() { + return restMaxInsertColumns; + } + + public void setRestMaxInsertColumns(int restMaxInsertColumns) { + this.restMaxInsertColumns = restMaxInsertColumns; + } + + public long getRestMaxInsertValues() { + return restMaxInsertValues; + } + + public void setRestMaxInsertValues(long restMaxInsertValues) { + this.restMaxInsertValues = restMaxInsertValues; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java index 003489e22b7fb..7cfdc0dd471f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/rest/IoTDBRestServiceDescriptor.java @@ -19,8 +19,10 @@ package org.apache.iotdb.db.conf.rest; import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.conf.TrimProperties; +import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; import org.apache.iotdb.rpc.RpcSslUtils; @@ -40,6 +42,16 @@ public class IoTDBRestServiceDescriptor { private static final Logger logger = LoggerFactory.getLogger(IoTDBRestServiceDescriptor.class); + private static final String REST_QUERY_DEFAULT_ROW_SIZE_LIMIT = + "rest_query_default_row_size_limit"; + private static final String REST_MAX_REQUEST_BODY_SIZE_IN_BYTES = + "rest_max_request_body_size_in_bytes"; + private static final String REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES = + "rest_max_total_concurrent_request_body_size_in_bytes"; + private static final String REST_MAX_INSERT_ROWS = "rest_max_insert_rows"; + private static final String REST_MAX_INSERT_COLUMNS = "rest_max_insert_columns"; + private static final String REST_MAX_INSERT_VALUES = "rest_max_insert_values"; + private final IoTDBRestServiceConfig conf = new IoTDBRestServiceConfig(); protected IoTDBRestServiceDescriptor() { @@ -88,11 +100,7 @@ private void loadProps(TrimProperties trimProperties) { Integer.parseInt( trimProperties.getProperty( "rest_service_port", Integer.toString(conf.getRestServicePort())))); - conf.setRestQueryDefaultRowSizeLimit( - Integer.parseInt( - trimProperties.getProperty( - "rest_query_default_row_size_limit", - Integer.toString(conf.getRestQueryDefaultRowSizeLimit())))); + loadRuntimeLimitProps(trimProperties); conf.setEnableSwagger( Boolean.parseBoolean( trimProperties.getProperty( @@ -118,6 +126,71 @@ private void loadProps(TrimProperties trimProperties) { "idle_timeout_in_seconds", Integer.toString(conf.getIdleTimeoutInSeconds())))); } + public synchronized void loadHotModifiedProps(TrimProperties trimProperties) { + loadRuntimeLimitProps(trimProperties); + } + + public synchronized void overwriteAppliedRuntimeLimitProperties() { + overlayRuntimeLimitProperties(); + } + + private void loadRuntimeLimitProps(TrimProperties trimProperties) { + conf.setRestQueryDefaultRowSizeLimit( + Integer.parseInt( + trimProperties.getProperty( + REST_QUERY_DEFAULT_ROW_SIZE_LIMIT, + Integer.toString(conf.getRestQueryDefaultRowSizeLimit())))); + conf.setRestMaxRequestBodySizeInBytes( + Long.parseLong( + trimProperties.getProperty( + REST_MAX_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxRequestBodySizeInBytes())))); + conf.setRestMaxTotalConcurrentRequestBodySizeInBytes( + parseMaxTotalConcurrentRequestBodySizeInBytes(trimProperties)); + conf.setRestMaxInsertRows( + Integer.parseInt( + trimProperties.getProperty( + REST_MAX_INSERT_ROWS, Integer.toString(conf.getRestMaxInsertRows())))); + conf.setRestMaxInsertColumns( + Integer.parseInt( + trimProperties.getProperty( + REST_MAX_INSERT_COLUMNS, Integer.toString(conf.getRestMaxInsertColumns())))); + conf.setRestMaxInsertValues( + Long.parseLong( + trimProperties.getProperty( + REST_MAX_INSERT_VALUES, Long.toString(conf.getRestMaxInsertValues())))); + overlayRuntimeLimitProperties(); + } + + private long parseMaxTotalConcurrentRequestBodySizeInBytes(TrimProperties trimProperties) { + long maxTotalConcurrentRequestBodySizeInBytes = + Long.parseLong( + trimProperties.getProperty( + REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxTotalConcurrentRequestBodySizeInBytes()))); + return maxTotalConcurrentRequestBodySizeInBytes == 0 + ? DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(trimProperties) + : maxTotalConcurrentRequestBodySizeInBytes; + } + + private void overlayRuntimeLimitProperties() { + ConfigurationFileUtils.updateAppliedProperties( + REST_QUERY_DEFAULT_ROW_SIZE_LIMIT, + Integer.toString(conf.getRestQueryDefaultRowSizeLimit())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxRequestBodySizeInBytes())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_TOTAL_CONCURRENT_REQUEST_BODY_SIZE_IN_BYTES, + Long.toString(conf.getRestMaxTotalConcurrentRequestBodySizeInBytes())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_INSERT_ROWS, Integer.toString(conf.getRestMaxInsertRows())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_INSERT_COLUMNS, Integer.toString(conf.getRestMaxInsertColumns())); + ConfigurationFileUtils.updateAppliedProperties( + REST_MAX_INSERT_VALUES, Long.toString(conf.getRestMaxInsertValues())); + } + /** * get props url location * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java index 0d3447bbefbe5..7d227abcd0f2b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DataNodeMemoryConfigTest.java @@ -19,22 +19,51 @@ package org.apache.iotdb.db.conf; +import org.apache.iotdb.commons.conf.TrimProperties; import org.apache.iotdb.commons.memory.MemoryConfig; -import org.junit.Assert; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class DataNodeMemoryConfigTest { @Test public void testRpcMemoryControlIsActivatedOnlyExplicitly() { DataNodeMemoryConfig memoryConfig = IoTDBDescriptor.getInstance().getMemoryConfig(); - Assert.assertEquals( - 0, MemoryConfig.getInstance().getAutoResizingBufferMemoryTotalSizeInBytes()); + assertEquals(0, MemoryConfig.getInstance().getAutoResizingBufferMemoryTotalSizeInBytes()); memoryConfig.activateAutoResizingBufferMemoryControl(); - Assert.assertTrue(MemoryConfig.getInstance().getAutoResizingBufferMemoryTotalSizeInBytes() > 0); + assertTrue(MemoryConfig.getInstance().getAutoResizingBufferMemoryTotalSizeInBytes() > 0); + } + + @Test + public void testDefaultAutoResizingBufferMemorySize() { + assertEquals( + Runtime.getRuntime().maxMemory() / 20, + DataNodeMemoryConfig.getDefaultAutoResizingBufferMemorySizeInBytes()); + } + + @Test + public void testCalculateAutoResizingBufferMemorySizeWithDataNodeMemoryProportion() { + TrimProperties properties = new TrimProperties(); + properties.setProperty("datanode_memory_proportion", "1:1:1:1:1:5"); + + assertEquals( + Runtime.getRuntime().maxMemory() / 4, + DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(properties)); + } + + @Test + public void testCalculateAutoResizingBufferMemorySizeWithDeprecatedMemoryProportion() { + TrimProperties properties = new TrimProperties(); + properties.setProperty("storage_query_schema_consensus_free_memory_proportion", "1:1:1:1:1:2"); + + assertEquals( + Runtime.getRuntime().maxMemory() / 7, + DataNodeMemoryConfig.calculateAutoResizingBufferMemorySizeInBytes(properties)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java index f4daffbf303b9..24cb6a1b8c522 100755 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/PropertiesTest.java @@ -19,8 +19,11 @@ package org.apache.iotdb.db.conf; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.TrimProperties; import org.apache.iotdb.commons.utils.RegionMigrationFileRemoveRateLimiter; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceConfig; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; import com.tngtech.archunit.core.domain.JavaClasses; import com.tngtech.archunit.core.importer.ClassFileImporter; @@ -65,6 +68,66 @@ public void testHotReloadRegionMigrationFileRemoveSpeedLimit() throws Exception } } + @Test + public void testHotReloadRestRuntimeLimitProperties() throws Exception { + IoTDBRestServiceConfig restConfig = IoTDBRestServiceDescriptor.getInstance().getConfig(); + int originalQueryDefaultRowSizeLimit = restConfig.getRestQueryDefaultRowSizeLimit(); + long originalMaxRequestBodySizeInBytes = restConfig.getRestMaxRequestBodySizeInBytes(); + long originalMaxTotalConcurrentRequestBodySizeInBytes = + restConfig.getRestMaxTotalConcurrentRequestBodySizeInBytes(); + int originalMaxInsertRows = restConfig.getRestMaxInsertRows(); + int originalMaxInsertColumns = restConfig.getRestMaxInsertColumns(); + long originalMaxInsertValues = restConfig.getRestMaxInsertValues(); + int originalRestServicePort = restConfig.getRestServicePort(); + boolean originalEnableHttps = restConfig.isEnableHttps(); + try { + TrimProperties properties = new TrimProperties(); + properties.setProperty("rest_query_default_row_size_limit", "123"); + properties.setProperty("rest_max_request_body_size_in_bytes", "456"); + properties.setProperty("rest_max_total_concurrent_request_body_size_in_bytes", "789"); + properties.setProperty("rest_max_insert_rows", "11"); + properties.setProperty("rest_max_insert_columns", "12"); + properties.setProperty("rest_max_insert_values", "13"); + properties.setProperty("rest_service_port", Integer.toString(originalRestServicePort + 1)); + properties.setProperty("enable_https", Boolean.toString(!originalEnableHttps)); + + IoTDBDescriptor.getInstance().loadHotModifiedProps(properties); + + Assert.assertEquals(123, restConfig.getRestQueryDefaultRowSizeLimit()); + Assert.assertEquals(456, restConfig.getRestMaxRequestBodySizeInBytes()); + Assert.assertEquals(789, restConfig.getRestMaxTotalConcurrentRequestBodySizeInBytes()); + Assert.assertEquals(11, restConfig.getRestMaxInsertRows()); + Assert.assertEquals(12, restConfig.getRestMaxInsertColumns()); + Assert.assertEquals(13, restConfig.getRestMaxInsertValues()); + Assert.assertEquals(originalRestServicePort, restConfig.getRestServicePort()); + Assert.assertEquals(originalEnableHttps, restConfig.isEnableHttps()); + + properties.setProperty("rest_max_total_concurrent_request_body_size_in_bytes", "0"); + properties.setProperty("datanode_memory_proportion", "1:1:1:1:1:5"); + + IoTDBDescriptor.getInstance().loadHotModifiedProps(properties); + + Assert.assertEquals( + Runtime.getRuntime().maxMemory() / 4, + restConfig.getRestMaxTotalConcurrentRequestBodySizeInBytes()); + Assert.assertEquals( + Long.toString(Runtime.getRuntime().maxMemory() / 4), + ConfigurationFileUtils.getAppliedProperties() + .get("rest_max_total_concurrent_request_body_size_in_bytes")); + } finally { + restConfig.setRestQueryDefaultRowSizeLimit(originalQueryDefaultRowSizeLimit); + restConfig.setRestMaxRequestBodySizeInBytes(originalMaxRequestBodySizeInBytes); + restConfig.setRestMaxTotalConcurrentRequestBodySizeInBytes( + originalMaxTotalConcurrentRequestBodySizeInBytes); + restConfig.setRestMaxInsertRows(originalMaxInsertRows); + restConfig.setRestMaxInsertColumns(originalMaxInsertColumns); + restConfig.setRestMaxInsertValues(originalMaxInsertValues); + restConfig.setRestServicePort(originalRestServicePort); + restConfig.setEnableHttps(originalEnableHttps); + IoTDBRestServiceDescriptor.getInstance().overwriteAppliedRuntimeLimitProperties(); + } + } + @Test public void PropertiesWithSpace() { IoTDBDescriptor descriptor = IoTDBDescriptor.getInstance(); diff --git a/iotdb-core/datanode/src/test/resources/iotdb-common.properties b/iotdb-core/datanode/src/test/resources/iotdb-common.properties index d22cfaa637240..a0399353b20eb 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-common.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-common.properties @@ -34,6 +34,21 @@ enable_rest_service=true # The request rowLimit/row_limit value cannot exceed this limit. # rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes +# rest_max_request_body_size_in_bytes=16777216 + +# Maximum total in-flight REST request body size in bytes across concurrent requests. When set to 0, use the same budget as AutoResizingBuffer memory control. Set to a negative value to disable the limit. +# rest_max_total_concurrent_request_body_size_in_bytes=0 + +# Maximum rows accepted by a single REST write request +# rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request +# rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request +# rest_max_insert_values=1000000 + # is SSL enabled # enable_https=false diff --git a/iotdb-core/datanode/src/test/resources/iotdb-system.properties b/iotdb-core/datanode/src/test/resources/iotdb-system.properties index 4cde9826f0793..9e0e16caaa99f 100644 --- a/iotdb-core/datanode/src/test/resources/iotdb-system.properties +++ b/iotdb-core/datanode/src/test/resources/iotdb-system.properties @@ -52,6 +52,21 @@ enable_rest_service=true # The request rowLimit/row_limit value cannot exceed this limit. # rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes +# rest_max_request_body_size_in_bytes=16777216 + +# Maximum total in-flight REST request body size in bytes across concurrent requests. When set to 0, use the same budget as AutoResizingBuffer memory control. Set to a negative value to disable the limit. +# rest_max_total_concurrent_request_body_size_in_bytes=0 + +# Maximum rows accepted by a single REST write request +# rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request +# rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request +# rest_max_insert_values=1000000 + # is SSL enabled # enable_https=false diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 0150f9db6fa0a..dd3ba2d295ecb 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -610,10 +610,35 @@ enable_swagger=false # The maximum row limit for REST and Grafana query responses. # The request rowLimit/row_limit value cannot exceed this limit. # A non-positive value is invalid and falls back to the default (10000); it no longer means unlimited. -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: int rest_query_default_row_size_limit=10000 +# Maximum REST request body size in bytes. Set to 0 or a negative value to disable the limit. +# effectiveMode: hot_reload +# Datatype: long +rest_max_request_body_size_in_bytes=16777216 + +# Maximum total in-flight REST request body size in bytes across concurrent requests. When set to 0, use the same budget as AutoResizingBuffer memory control. Set to a negative value to disable the limit. +# effectiveMode: hot_reload +# Datatype: long +rest_max_total_concurrent_request_body_size_in_bytes=0 + +# Maximum rows accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: hot_reload +# Datatype: int +rest_max_insert_rows=100000 + +# Maximum columns accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: hot_reload +# Datatype: int +rest_max_insert_columns=1024 + +# Maximum values accepted by a single REST write request. Set to 0 or a negative value to disable the limit. +# effectiveMode: hot_reload +# Datatype: long +rest_max_insert_values=1000000 + # Is client authentication required # effectiveMode: restart # Datatype: boolean