diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index d4d6ebdc29383..46e4c6893fe09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -57,6 +57,7 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.reader.BufferedTsFileInput; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.chunk.AlignedChunkReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; @@ -156,11 +157,7 @@ public TsFileInsertionEventScanParser( PipeDataNodeResourceManager.memory() .forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed()); - tsFileSequenceReader = - new TsFileSequenceReader( - tsFile.getAbsolutePath(), - !currentModifications.isEmpty(), - !currentModifications.isEmpty()); + tsFileSequenceReader = createTsFileSequenceReader(tsFile, !currentModifications.isEmpty()); tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); prepareData(); @@ -193,6 +190,15 @@ public TsFileInsertionEventScanParser( isWithMod); } + private static TsFileSequenceReader createTsFileSequenceReader( + final File tsFile, final boolean hasModifications) throws IOException { + if (hasModifications) { + return new TsFileSequenceReader(tsFile.getAbsolutePath(), true, true); + } + + return new TsFileSequenceReader(new BufferedTsFileInput(tsFile.toPath()), false, false, null); + } + @Override public Iterable toTabletInsertionEvents() { if (tabletInsertionIterable == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 0e44a3dbde5d3..97fab7a290bc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -46,7 +46,6 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -74,9 +73,7 @@ public abstract class AlignedTVList extends TVList { private static final long BITMAP_RAM_COST_PER_BLOCK = - RamUsageEstimator.shallowSizeOfInstance(BitMap.class) - + RamUsageEstimator.sizeOfByteArray(BitMap.getSizeOfBytes(ARRAY_SIZE)) - + NUM_BYTES_OBJECT_REF; + new BitMap(ARRAY_SIZE).ramBytesUsed() + NUM_BYTES_OBJECT_REF; // Data types of this aligned tvList protected List dataTypes; @@ -1188,7 +1185,7 @@ public static long valueListArrayMemCost(TSDataType type) { long size = 0; // value array mem size size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); - // bitmap object, byte array, and reference in the bitmap list + // bitmap and its implementation, plus the reference in the bitmap list size += BITMAP_RAM_COST_PER_BLOCK; // array headers mem size size += NUM_BYTES_ARRAY_HEADER; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java index 040988997b2a3..f1d4a1572e6f3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java @@ -536,21 +536,21 @@ public void alignedTvListRamCostTest() true, new long[5]); IMemTable memTable = processor.getWorkMemTable(); - Assert.assertEquals(1776552, memTable.getTVListsRamCost()); + Assert.assertEquals(1824552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNode(100, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(1776552, memTable.getTVListsRamCost()); + Assert.assertEquals(1824552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNode(200, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(1776552, memTable.getTVListsRamCost()); + Assert.assertEquals(1824552, memTable.getTVListsRamCost()); Assert.assertEquals(90000, memTable.getTotalPointsNum()); Assert.assertEquals(720360, memTable.memSize()); // Test records @@ -559,7 +559,7 @@ public void alignedTvListRamCostTest() record.addTuple(DataPoint.getDataPoint(dataType, measurementId, String.valueOf(i))); processor.insert(buildInsertRowNodeByTSRecord(record), new long[5]); } - Assert.assertEquals(1778168, memTable.getTVListsRamCost()); + Assert.assertEquals(1826168, memTable.getTVListsRamCost()); Assert.assertEquals(90100, memTable.getTotalPointsNum()); Assert.assertEquals(721560, memTable.memSize()); } @@ -614,56 +614,56 @@ public void alignedTvListRamCostTest2() true, new long[5]); IMemTable memTable = processor.getWorkMemTable(); - Assert.assertEquals(1776552, memTable.getTVListsRamCost()); + Assert.assertEquals(1824552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNodeFors3000ToS6000(0, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(3552552, memTable.getTVListsRamCost()); + Assert.assertEquals(3648552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNode(100, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(3552552, memTable.getTVListsRamCost()); + Assert.assertEquals(3648552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNodeFors3000ToS6000(100, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(3552552, memTable.getTVListsRamCost()); + Assert.assertEquals(3648552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNode(200, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(3552552, memTable.getTVListsRamCost()); + Assert.assertEquals(3648552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNodeFors3000ToS6000(200, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(3552552, memTable.getTVListsRamCost()); + Assert.assertEquals(3648552, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNode(300, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(7105104, memTable.getTVListsRamCost()); + Assert.assertEquals(7297104, memTable.getTVListsRamCost()); processor.insertTablet( genInsertTableNodeFors3000ToS6000(300, true), Collections.singletonList(new int[] {0, 10}), new TSStatus[10], true, new long[5]); - Assert.assertEquals(7105104, memTable.getTVListsRamCost()); + Assert.assertEquals(7297104, memTable.getTVListsRamCost()); Assert.assertEquals(240000, memTable.getTotalPointsNum()); Assert.assertEquals(1920960, memTable.memSize()); @@ -673,14 +673,14 @@ public void alignedTvListRamCostTest2() record.addTuple(DataPoint.getDataPoint(dataType, measurementId, String.valueOf(i))); processor.insert(buildInsertRowNodeByTSRecord(record), new long[5]); } - Assert.assertEquals(7106720, memTable.getTVListsRamCost()); + Assert.assertEquals(7298720, memTable.getTVListsRamCost()); // Test records for (int i = 1; i <= 100; i++) { TSRecord record = new TSRecord(deviceId, i); record.addTuple(DataPoint.getDataPoint(dataType, "s1", String.valueOf(i))); processor.insert(buildInsertRowNodeByTSRecord(record), new long[5]); } - Assert.assertEquals(7108336, memTable.getTVListsRamCost()); + Assert.assertEquals(7300336, memTable.getTVListsRamCost()); Assert.assertEquals(240200, memTable.getTotalPointsNum()); Assert.assertEquals(1923360, memTable.memSize()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java index 5ab976c13104b..b9f3428cd08ed 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java @@ -34,9 +34,22 @@ import java.util.List; import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE; +import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; +import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; public class AlignedTVListTest { + @Test + public void testValueListArrayMemCostIncludesBitmapImplementation() { + long expected = + (long) ARRAY_SIZE * Long.BYTES + + new BitMap(ARRAY_SIZE).ramBytesUsed() + + NUM_BYTES_ARRAY_HEADER + + 2L * NUM_BYTES_OBJECT_REF; + + Assert.assertEquals(expected, AlignedTVList.valueListArrayMemCost(TSDataType.INT64)); + } + @Test public void testAlignedTVList1() { List dataTypes = new ArrayList<>(); diff --git a/pom.xml b/pom.xml index 4b0f44bbad353..79df3c2ebb172 100644 --- a/pom.xml +++ b/pom.xml @@ -145,7 +145,7 @@ 0.23.0 1.9 1.5.6-3 - 2.3.2-260616-SNAPSHOT + 2.3.2-260721-SNAPSHOT en