From c0aa3c925eab90d2c57e9f74e23fdeeca0951735 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 26 May 2026 15:27:19 +0800 Subject: [PATCH 1/2] Optimize aligned TVList with lazy column allocation and runtime null bitmap. Allocate value arrays on write only, use per-chunk BitMaps for explicit nulls, and treat unmaterialized columns as null. Adapt query iterators, flush encoding, and memtable memory estimation for sparse aligned writes. --- .../memtable/AlignedWritableMemChunk.java | 91 +++- .../dataregion/memtable/TsFileProcessor.java | 105 +++- .../db/utils/datastructure/AlignedTVList.java | 502 ++++++++++-------- 3 files changed, 443 insertions(+), 255 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index b82786a32ed3d..d981c2fdbc124 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -350,7 +350,6 @@ private void getAnySatisfiedTimestamp( alignedTVList.getMinTime(), alignedTVList.getMaxTime())) { return; } - BitMap allValueColDeletedMap = alignedTVList.getAllValueColDeletedMap(); int rowCount = alignedTVList.rowCount(); List valueColumnDeleteCursor = new ArrayList<>(); if (valueColumnsDeletionList != null) { @@ -379,8 +378,9 @@ private void getAnySatisfiedTimestamp( int limit = (i == timestampsList.size() - 1) ? rowCount - i * ARRAY_SIZE : ARRAY_SIZE; for (int j = 0; j < limit; j++) { row++; - // the row is deleted - if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) { + // the row is deleted or has no value columns (unwritten columns count as null) + int valueIndex = indices == null ? row : indices[j]; + if (ignoreAllNullRows && alignedTVList.isEmptyValueRowAtValueIndex(valueIndex)) { continue; } long timestamp = timestamps[j]; @@ -551,12 +551,8 @@ public void encodeWorkingAlignedTVList( BlockingQueue ioTaskQueue, long maxNumberOfPointsInChunk, int maxNumberOfPointsInPage) { - BitMap allValueColDeletedMap; AlignedTVList alignedWorkingListForFlush = (AlignedTVList) workingListForFlush; - allValueColDeletedMap = - ignoreAllNullRows ? alignedWorkingListForFlush.getAllValueColDeletedMap() : null; - boolean[] timeDuplicateInfo = null; List> chunkRange = new ArrayList<>(); @@ -592,8 +588,8 @@ public void encodeWorkingAlignedTVList( int nextRowIndex = sortedRowIndex + 1; while (nextRowIndex < alignedWorkingListForFlush.rowCount() - && ((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked( + && ((ignoreAllNullRows + && alignedWorkingListForFlush.isEmptyValueRowAtValueIndex( alignedWorkingListForFlush.getValueIndex(nextRowIndex))) || alignedWorkingListForFlush.isTimeDeleted(nextRowIndex))) { nextRowIndex++; @@ -615,15 +611,13 @@ public void encodeWorkingAlignedTVList( chunkRange.add(pageRange); } - handleEncoding( - ioTaskQueue, chunkRange, timeDuplicateInfo, allValueColDeletedMap, maxNumberOfPointsInPage); + handleEncoding(ioTaskQueue, chunkRange, timeDuplicateInfo, maxNumberOfPointsInPage); } private void handleEncoding( BlockingQueue ioTaskQueue, List> chunkRange, boolean[] timeDuplicateInfo, - BitMap allValueColDeletedMap, int maxNumberOfPointsInPage) { AlignedTVList alignedWorkingListForFlush = (AlignedTVList) workingListForFlush; List dataTypes = alignedWorkingListForFlush.getTsDataTypes(); @@ -643,8 +637,8 @@ private void handleEncoding( sortedRowIndex <= pageRange.get(pageNum * 2 + 1); sortedRowIndex++) { // skip empty row - if (allValueColDeletedMap != null - && allValueColDeletedMap.isMarked( + if (ignoreAllNullRows + && alignedWorkingListForFlush.isEmptyValueRowAtValueIndex( alignedWorkingListForFlush.getValueIndex(sortedRowIndex))) { continue; } @@ -753,8 +747,8 @@ private void handleEncoding( sortedRowIndex <= pageRange.get(pageNum * 2 + 1); sortedRowIndex++) { // skip empty row - if (((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked( + if (((ignoreAllNullRows + && alignedWorkingListForFlush.isEmptyValueRowAtValueIndex( alignedWorkingListForFlush.getValueIndex(sortedRowIndex))) || (alignedWorkingListForFlush.isTimeDeleted(sortedRowIndex)))) { continue; @@ -893,6 +887,71 @@ public boolean isEmpty() { return false; } + /** + * Extra memory for allocating value arrays in the current (last) chunk when columns are written + * for the first time in that chunk. + */ + public long getTvListArrayMemCostIncrement( + String[] insertingMeasurements, TSDataType[] insertingTypes, Object[] insertingValues) { + long memCostIncrement = 0; + for (int i = 0; i < insertingMeasurements.length; i++) { + if (insertingTypes[i] == null || insertingMeasurements[i] == null) { + continue; + } + Integer columnIndex = measurementIndexMap.get(insertingMeasurements[i]); + if (columnIndex == null) { + continue; + } + if (!list.isLastValueArrayUnallocated(columnIndex)) { + continue; + } + if (insertingValues != null && insertingValues[i] != null) { + memCostIncrement += AlignedTVList.valueListArrayMemCost(insertingTypes[i]); + } + } + return memCostIncrement; + } + + /** + * Extra memory for tablet insertion: allocate value arrays only when the column has non-null + * values in the inserting range of the last chunk. + */ + public long getTvListArrayMemCostIncrementForTablet( + String[] insertingMeasurements, + TSDataType[] insertingTypes, + Object[] columns, + BitMap[] bitMaps, + int start, + int end) { + long memCostIncrement = 0; + for (int i = 0; i < insertingMeasurements.length; i++) { + if (insertingTypes[i] == null || insertingMeasurements[i] == null || columns[i] == null) { + continue; + } + Integer columnIndex = measurementIndexMap.get(insertingMeasurements[i]); + if (columnIndex == null || !list.isLastValueArrayUnallocated(columnIndex)) { + continue; + } + if (columnHasNonNullInRange(columns[i], bitMaps == null ? null : bitMaps[i], start, end)) { + memCostIncrement += AlignedTVList.valueListArrayMemCost(insertingTypes[i]); + } + } + return memCostIncrement; + } + + private static boolean columnHasNonNullInRange( + Object column, BitMap bitMap, int start, int end) { + if (bitMap == null) { + return true; + } + for (int i = start; i < end; i++) { + if (!bitMap.isMarked(i)) { + return true; + } + } + return false; + } + @Override public int serializedSize() { int size = 0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index c4e488f84863f..1c1628ed441b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -92,6 +92,7 @@ import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.writer.RestorableTsFileIOWriter; import org.slf4j.Logger; @@ -544,6 +545,7 @@ private long[] checkAlignedMemCost( insertTabletNode.getMeasurements(), insertTabletNode.getDataTypes(), insertTabletNode.getColumns(), + insertTabletNode.getBitMaps(), insertTabletNode.getColumnCategories(), splitStart, splitEnd, @@ -784,11 +786,11 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( chunkMetadataIncrement += ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) * dataTypes.length; - memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories); + memTableIncrement += + AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories, values); } else { // For existed device of this mem table AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; - List dataTypesInTVList = new ArrayList<>(); for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements if (dataTypes[i] == null @@ -804,15 +806,19 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( + (alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0); - memTableIncrement += currentArrayNum * AlignedTVList.valueListArrayMemCost(dataTypes[i]); - dataTypesInTVList.add(dataTypes[i]); + long columnArrayCost = + values[i] != null + ? AlignedTVList.valueListArrayMemCost(dataTypes[i]) + : AlignedTVList.emptyValueListArrayMemCost(); + memTableIncrement += currentArrayNum * columnArrayCost; } } // this insertion will result in a new array if ((alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE) == 0) { - dataTypesInTVList.addAll(alignedMemChunk.getWorkingTVList().getTsDataTypes()); memTableIncrement += alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost(); } + memTableIncrement += + alignedMemChunk.getTvListArrayMemCostIncrement(measurements, dataTypes, values); } for (int i = 0; i < dataTypes.length; i++) { @@ -848,7 +854,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins chunkMetadataIncrement += ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) * dataTypes.length; - memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, null); + memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, null, values); for (int i = 0; i < dataTypes.length; i++) { // Skip failed Measurements if (dataTypes[i] == null @@ -867,7 +873,6 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins // For existed device of this mem table AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; int currentChunkPointNum = alignedMemChunk == null ? 0 : alignedMemChunk.alignedListSize(); - List dataTypesInTVList = new ArrayList<>(); Pair, Integer> addingPointNumInfo = increasingMemTableInfo.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 0)); for (int i = 0; i < dataTypes.length; i++) { @@ -892,22 +897,26 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins > 0 ? 1 : 0); - memTableIncrement += - currentArrayNum * AlignedTVList.valueListArrayMemCost(dataTypes[i]); + long columnArrayCost = + values[i] != null + ? AlignedTVList.valueListArrayMemCost(dataTypes[i]) + : AlignedTVList.emptyValueListArrayMemCost(); + memTableIncrement += currentArrayNum * columnArrayCost; } } int addingPointNum = addingPointNumInfo.right; // Here currentChunkPointNum + addingPointNum >= 1 if (((currentChunkPointNum + addingPointNum) % PrimitiveArrayManager.ARRAY_SIZE) == 0) { if (alignedMemChunk != null) { - dataTypesInTVList.addAll(alignedMemChunk.getWorkingTVList().getTsDataTypes()); + memTableIncrement += alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost(); + } else { + memTableIncrement += + AlignedTVList.alignedTvListArrayMemCost(dataTypes, null, values); } - dataTypesInTVList.addAll(addingPointNumInfo.left.values()); + } + if (alignedMemChunk != null) { memTableIncrement += - alignedMemChunk != null - ? alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost() - : AlignedTVList.alignedTvListArrayMemCost( - dataTypesInTVList.toArray(new TSDataType[0]), null); + alignedMemChunk.getTvListArrayMemCostIncrement(measurements, dataTypes, values); } addingPointNumInfo.setRight(addingPointNum + 1); } @@ -962,6 +971,7 @@ private long[] checkAlignedMemCostAndAddToTspForTablet( String[] measurements, TSDataType[] dataTypes, Object[] columns, + BitMap[] bitMaps, TsTableColumnCategory[] columnCategories, int start, int end, @@ -981,6 +991,7 @@ private long[] checkAlignedMemCostAndAddToTspForTablet( end, memIncrements, columns, + bitMaps, columnCategories, noFailure, results); @@ -1038,6 +1049,7 @@ private void updateAlignedMemCost( int end, long[] memIncrements, Object[] columns, + BitMap[] bitMaps, TsTableColumnCategory[] columnCategories, boolean noFailure, TSStatus[] results) { @@ -1078,11 +1090,15 @@ private void updateAlignedMemCost( int numArraysToAdd = incomingPointNum / PrimitiveArrayManager.ARRAY_SIZE + (incomingPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0); + boolean[] columnHasNonNull = + buildAlignedColumnHasNonNull( + dataTypes, columns, bitMaps, columnCategories, start, end); memIncrements[0] += - numArraysToAdd * AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories); + numArraysToAdd + * AlignedTVList.alignedTvListArrayMemCost( + dataTypes, columnCategories, null, columns, columnHasNonNull); } else { AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; - List dataTypesInTVList = new ArrayList<>(); int currentPointNum = alignedMemChunk.alignedListSize(); int newPointNum = currentPointNum + incomingPointNum; for (int i = 0; i < dataTypes.length; i++) { @@ -1097,11 +1113,17 @@ private void updateAlignedMemCost( } if (!alignedMemChunk.containsMeasurement(measurementIds[i])) { - // add a new column in the TVList, the new column should be as long as existing ones - memIncrements[0] += - (currentPointNum / PrimitiveArrayManager.ARRAY_SIZE + 1) - * AlignedTVList.valueListArrayMemCost(dataType); - dataTypesInTVList.add(dataType); + int newColumnArrayNum = + currentPointNum / PrimitiveArrayManager.ARRAY_SIZE + + (currentPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0) + + incomingPointNum / PrimitiveArrayManager.ARRAY_SIZE + + (incomingPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0); + long columnArrayCost = + alignedColumnHasNonNullInRange( + column, bitMaps == null ? null : bitMaps[i], start, end) + ? AlignedTVList.valueListArrayMemCost(dataType) + : AlignedTVList.emptyValueListArrayMemCost(); + memIncrements[0] += newColumnArrayNum * columnArrayCost; } } @@ -1115,11 +1137,12 @@ private void updateAlignedMemCost( long acquireArray = newArrayCnt - currentArrayCnt; if (acquireArray != 0) { - // memory of extending the TVList - dataTypesInTVList.addAll(alignedMemChunk.getWorkingTVList().getTsDataTypes()); memIncrements[0] += acquireArray * alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost(); } + memIncrements[0] += + alignedMemChunk.getTvListArrayMemCostIncrementForTablet( + measurementIds, dataTypes, columns, bitMaps, start, end); } // flexible-length data size @@ -1141,6 +1164,40 @@ private void updateAlignedMemCost( } } + private static boolean[] buildAlignedColumnHasNonNull( + TSDataType[] dataTypes, + Object[] columns, + BitMap[] bitMaps, + TsTableColumnCategory[] columnCategories, + int start, + int end) { + boolean[] columnHasNonNull = new boolean[dataTypes.length]; + for (int i = 0; i < dataTypes.length; i++) { + if (dataTypes[i] == null || columns[i] == null) { + continue; + } + if (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD) { + continue; + } + columnHasNonNull[i] = + alignedColumnHasNonNullInRange(columns[i], bitMaps == null ? null : bitMaps[i], start, end); + } + return columnHasNonNull; + } + + private static boolean alignedColumnHasNonNullInRange( + Object column, BitMap bitMap, int start, int end) { + if (bitMap == null) { + return true; + } + for (int i = start; i < end; i++) { + if (!bitMap.isMarked(i)) { + return true; + } + } + return false; + } + private void updateMemoryInfo( long memTableIncrement, long chunkMetadataIncrement, long textDataIncrement) throws WriteProcessRejectException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index dc3c31b855a00..9e2ae4b3f0917 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -107,6 +107,23 @@ public abstract class AlignedTVList extends TVList { } } + /** Allocate value primitive array for column {@code columnIndex} at {@code arrayIndex}. */ + private Object allocateValueArray(int columnIndex, int arrayIndex) { + Object array = getPrimitiveArraysByType(dataTypes.get(columnIndex)); + values.get(columnIndex).set(arrayIndex, array); + return array; + } + + /** Get existing value array or allocate on demand. */ + private Object getOrAllocateValueArray(int columnIndex, int arrayIndex) { + List columnValues = values.get(columnIndex); + Object array = columnValues.get(arrayIndex); + if (array == null) { + array = allocateValueArray(columnIndex, arrayIndex); + } + return array; + } + public static AlignedTVList newAlignedList(List dataTypes) { switch (TVLIST_SORT_ALGORITHM) { case QUICK: @@ -226,43 +243,40 @@ public synchronized void putAlignedValue(long timestamp, Object[] value) { timestamps.get(arrayIndex)[elementIndex] = timestamp; for (int i = 0; i < values.size(); i++) { Object columnValue = value[i]; - List columnValues = values.get(i); if (columnValue == null) { markNullValue(i, arrayIndex, elementIndex); + continue; } switch (dataTypes.get(i)) { case TEXT: case BLOB: case STRING: case OBJECT: - ((Binary[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null ? (Binary) columnValue : Binary.EMPTY_VALUE; - memoryBinaryChunkSize[i] += - columnValue != null - ? getBinarySize((Binary) columnValue) - : getBinarySize(Binary.EMPTY_VALUE); + Binary[] arrayT = (Binary[]) getOrAllocateValueArray(i, arrayIndex); + arrayT[elementIndex] = (Binary) columnValue; + memoryBinaryChunkSize[i] += getBinarySize((Binary) columnValue); break; case FLOAT: - ((float[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null ? (float) columnValue : Float.MIN_VALUE; + float[] arrayF = (float[]) getOrAllocateValueArray(i, arrayIndex); + arrayF[elementIndex] = (float) columnValue; break; case INT32: case DATE: - ((int[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null ? (int) columnValue : Integer.MIN_VALUE; + int[] arrayI = (int[]) getOrAllocateValueArray(i, arrayIndex); + arrayI[elementIndex] = (int) columnValue; break; case INT64: case TIMESTAMP: - ((long[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null ? (long) columnValue : Long.MIN_VALUE; + long[] arrayL = (long[]) getOrAllocateValueArray(i, arrayIndex); + arrayL[elementIndex] = (long) columnValue; break; case DOUBLE: - ((double[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null ? (double) columnValue : Double.MIN_VALUE; + double[] arrayD = (double[]) getOrAllocateValueArray(i, arrayIndex); + arrayD[elementIndex] = (double) columnValue; break; case BOOLEAN: - ((boolean[]) columnValues.get(arrayIndex))[elementIndex] = - columnValue != null && (boolean) columnValue; + boolean[] arrayB = (boolean[]) getOrAllocateValueArray(i, arrayIndex); + arrayB[elementIndex] = (boolean) columnValue; break; default: break; @@ -393,33 +407,7 @@ public void extendColumn(TSDataType dataType) { List columnValue = new ArrayList<>(timestamps.size()); List columnBitMaps = new ArrayList<>(timestamps.size()); for (int i = 0; i < timestamps.size(); i++) { - switch (dataType) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - columnValue.add(getPrimitiveArraysByType(TSDataType.TEXT)); - break; - case FLOAT: - columnValue.add(getPrimitiveArraysByType(TSDataType.FLOAT)); - break; - case INT32: - case DATE: - columnValue.add(getPrimitiveArraysByType(TSDataType.INT32)); - break; - case INT64: - case TIMESTAMP: - columnValue.add(getPrimitiveArraysByType(TSDataType.INT64)); - break; - case DOUBLE: - columnValue.add(getPrimitiveArraysByType(TSDataType.DOUBLE)); - break; - case BOOLEAN: - columnValue.add(getPrimitiveArraysByType(TSDataType.BOOLEAN)); - break; - default: - break; - } + columnValue.add(null); BitMap bitMap = new BitMap(ARRAY_SIZE); // The following code is for these 2 kinds of scenarios. @@ -451,24 +439,32 @@ private Object getObjectByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); + Object valueArray = columnValues.get(arrayIndex); + if (valueArray == null) { + throw new IllegalStateException( + "Value array is not allocated for column " + + columnIndex + + " at array index " + + arrayIndex); + } switch (dataTypes.get(columnIndex)) { case INT32: case DATE: - return ((int[]) columnValues.get(arrayIndex))[elementIndex]; + return ((int[]) valueArray)[elementIndex]; case INT64: case TIMESTAMP: - return ((long[]) columnValues.get(arrayIndex))[elementIndex]; + return ((long[]) valueArray)[elementIndex]; case FLOAT: - return ((float[]) columnValues.get(arrayIndex))[elementIndex]; + return ((float[]) valueArray)[elementIndex]; case DOUBLE: - return ((double[]) columnValues.get(arrayIndex))[elementIndex]; + return ((double[]) valueArray)[elementIndex]; case BOOLEAN: - return ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; + return ((boolean[]) valueArray)[elementIndex]; case STRING: case BLOB: case TEXT: case OBJECT: - return ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; + return ((Binary[]) valueArray)[elementIndex]; default: throw new IllegalArgumentException( dataTypes.get(columnIndex) + StorageEngineMessages.IS_NOT_SUPPORTED); @@ -614,17 +610,20 @@ public boolean isNullValue(int unsortedRowIndex, int columnIndex) { if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(unsortedRowIndex)) { return true; } + int arrayIndex = unsortedRowIndex / ARRAY_SIZE; + int elementIndex = unsortedRowIndex % ARRAY_SIZE; - if (columnIndex < 0 || columnIndex >= values.size() || values.get(columnIndex) == null) { + if (columnIndex < 0 + || columnIndex >= values.size() + || values.get(columnIndex) == null + || values.get(columnIndex).get(arrayIndex) == null) { return true; } if (bitMaps == null || bitMaps.get(columnIndex) == null - || bitMaps.get(columnIndex).get(unsortedRowIndex / ARRAY_SIZE) == null) { + || bitMaps.get(columnIndex).get(arrayIndex) == null) { return false; } - int arrayIndex = unsortedRowIndex / ARRAY_SIZE; - int elementIndex = unsortedRowIndex % ARRAY_SIZE; List columnBitMaps = bitMaps.get(columnIndex); return columnBitMaps.get(arrayIndex).isMarked(elementIndex); } @@ -637,6 +636,10 @@ public List getTsDataTypes() { return dataTypes; } + public List getIndices() { + return indices; + } + @Override public int delete(long lowerBound, long upperBound) { int deletedNumber = 0; @@ -728,19 +731,26 @@ public void deleteColumn(int columnIndex) { if (bitMaps.get(columnIndex) == null) { List columnBitMaps = new ArrayList<>(values.get(columnIndex).size()); for (int i = 0; i < values.get(columnIndex).size(); i++) { - columnBitMaps.add(new BitMap(ARRAY_SIZE)); + columnBitMaps.add(null); } bitMaps.set(columnIndex, columnBitMaps); } - for (int i = 0; i < bitMaps.get(columnIndex).size(); i++) { - if (bitMaps.get(columnIndex).get(i) == null) { - bitMaps.get(columnIndex).set(i, new BitMap(ARRAY_SIZE)); + List columnBitMaps = bitMaps.get(columnIndex); + for (int i = 0; i < values.get(columnIndex).size(); i++) { + if (i >= columnBitMaps.size()) { + columnBitMaps.add(null); + } + if (columnBitMaps.get(i) == null) { + columnBitMaps.set(i, new BitMap(ARRAY_SIZE)); } - bitMaps.get(columnIndex).get(i).markAll(); + columnBitMaps.get(i).markAll(); } } protected Object cloneValue(TSDataType type, Object value) { + if (value == null) { + return null; + } switch (type) { case TEXT: case BLOB: @@ -788,7 +798,9 @@ protected void clearValue() { List columnValues = values.get(i); if (columnValues != null) { for (Object dataArray : columnValues) { - PrimitiveArrayManager.release(dataArray); + if (dataArray != null) { + PrimitiveArrayManager.release(dataArray); + } } columnValues.clear(); } @@ -814,7 +826,7 @@ protected void expandValues() { indices.add((int[]) getPrimitiveArraysByType(TSDataType.INT32)); } for (int i = 0; i < dataTypes.size(); i++) { - values.get(i).add(getPrimitiveArraysByType(dataTypes.get(i))); + values.get(i).add(null); if (bitMaps != null && bitMaps.get(i) != null) { bitMaps.get(i).add(null); } @@ -977,7 +989,7 @@ private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex case BLOB: case STRING: case OBJECT: - Binary[] arrayT = ((Binary[]) columnValues.get(arrayIndex)); + Binary[] arrayT = (Binary[]) getOrAllocateValueArray(i, arrayIndex); System.arraycopy(value[i], idx, arrayT, elementIndex, remaining); // update raw size of Text chunk @@ -987,25 +999,25 @@ private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex } break; case FLOAT: - float[] arrayF = ((float[]) columnValues.get(arrayIndex)); + float[] arrayF = (float[]) getOrAllocateValueArray(i, arrayIndex); System.arraycopy(value[i], idx, arrayF, elementIndex, remaining); break; case INT32: case DATE: - int[] arrayI = ((int[]) columnValues.get(arrayIndex)); + int[] arrayI = (int[]) getOrAllocateValueArray(i, arrayIndex); System.arraycopy(value[i], idx, arrayI, elementIndex, remaining); break; case INT64: case TIMESTAMP: - long[] arrayL = ((long[]) columnValues.get(arrayIndex)); + long[] arrayL = (long[]) getOrAllocateValueArray(i, arrayIndex); System.arraycopy(value[i], idx, arrayL, elementIndex, remaining); break; case DOUBLE: - double[] arrayD = ((double[]) columnValues.get(arrayIndex)); + double[] arrayD = (double[]) getOrAllocateValueArray(i, arrayIndex); System.arraycopy(value[i], idx, arrayD, elementIndex, remaining); break; case BOOLEAN: - boolean[] arrayB = ((boolean[]) columnValues.get(arrayIndex)); + boolean[] arrayB = (boolean[]) getOrAllocateValueArray(i, arrayIndex); System.arraycopy(value[i], idx, arrayB, elementIndex, remaining); break; default: @@ -1024,21 +1036,26 @@ private BitMap getBitMap(int columnIndex, int arrayIndex) { bitMaps = localBitMaps; } - // if the bitmap in columnIndex is null, init the bitmap of this column from the beginning + // Lazy init column bitmap list (null slots until a chunk needs explicit null marks) if (bitMaps.get(columnIndex) == null) { List columnBitMaps = new ArrayList<>(values.get(columnIndex).size()); for (int i = 0; i < values.get(columnIndex).size(); i++) { - columnBitMaps.add(new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE])); + columnBitMaps.add(null); } bitMaps.set(columnIndex, columnBitMaps); } - // if the bitmap in arrayIndex is null, init the bitmap - if (bitMaps.get(columnIndex).get(arrayIndex) == null) { - bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE])); + List columnBitMaps = bitMaps.get(columnIndex); + while (columnBitMaps.size() <= arrayIndex) { + columnBitMaps.add(null); + } + + // if the bitmap in arrayIndex is null, init the bitmap for this chunk only + if (columnBitMaps.get(arrayIndex) == null) { + columnBitMaps.set(arrayIndex, new BitMap(ARRAY_SIZE)); } - return bitMaps.get(columnIndex).get(arrayIndex); + return columnBitMaps.get(arrayIndex); } private void markNullValue( @@ -1078,84 +1095,127 @@ public synchronized RamInfo calculateRamSize() { * @param types the types in the vector * @return AlignedTvListArrayMemSize */ - public static long alignedTvListArrayMemCost( - TSDataType[] types, TsTableColumnCategory[] columnCategories) { + /** Memory cost of one timestamp primitive array chunk and list overhead. */ + public static long alignedTimestampArrayMemCost(int fieldColumnCount) { + long size = PrimitiveArrayManager.ARRAY_SIZE * 8L; + size += (long) NUM_BYTES_ARRAY_HEADER * (2L + fieldColumnCount); + size += (long) NUM_BYTES_OBJECT_REF * (2L + fieldColumnCount); + return size; + } + + /** Memory cost of one value-column chunk that only stores nulls (bitmap only). */ + public static long emptyValueListArrayMemCost() { + long size = PrimitiveArrayManager.ARRAY_SIZE / 8L + 1; + size += NUM_BYTES_OBJECT_REF; + return size; + } + + /** Memory cost of one allocated value primitive array chunk. */ + public static long valueListArrayMemCost(TSDataType type) { + return (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize() + + NUM_BYTES_ARRAY_HEADER + + NUM_BYTES_OBJECT_REF; + } - int measurementColumnNum = 0; + /** + * Estimate memory for one new array chunk when inserting aligned data. + * + * @param types measurement types + * @param columnCategories table column categories, nullable for tree model + * @param values row values for insertRow, nullable for tablet-only estimation + * @param columns tablet columns for insertTablet, nullable for row-only estimation + * @param columnHasNonNull whether each tablet column has non-null in the inserting range + */ + public static long alignedTvListArrayMemCost( + TSDataType[] types, + TsTableColumnCategory[] columnCategories, + Object[] values, + Object[] columns, + boolean[] columnHasNonNull) { + int fieldColumnCount = 0; long size = 0; - // value array mem size for (int i = 0; i < types.length; i++) { TSDataType type = types[i]; - if (type != null - && (columnCategories == null || columnCategories[i] == TsTableColumnCategory.FIELD)) { - size += (long) ARRAY_SIZE * (long) type.getDataTypeSize(); - measurementColumnNum++; - } - } - // size is 0 when all types are null - if (size == 0) { - return size; - } - // time array mem size - size += PrimitiveArrayManager.ARRAY_SIZE * 8L; - // array headers mem size - size += (long) NUM_BYTES_ARRAY_HEADER * (2 + measurementColumnNum); - // Object references size in ArrayList - size += (long) NUM_BYTES_OBJECT_REF * (2 + measurementColumnNum); - return size; + if (type == null + || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { + continue; + } + fieldColumnCount++; + boolean allocateValueArray; + if (values != null) { + allocateValueArray = values[i] != null; + } else if (columnHasNonNull != null) { + allocateValueArray = columnHasNonNull[i]; + } else { + allocateValueArray = columns != null && columns[i] != null; + } + size += + allocateValueArray ? valueListArrayMemCost(type) : emptyValueListArrayMemCost(); + } + if (fieldColumnCount == 0) { + return 0; + } + return size + alignedTimestampArrayMemCost(fieldColumnCount); + } + + public static long alignedTvListArrayMemCost( + TSDataType[] types, TsTableColumnCategory[] columnCategories) { + return alignedTvListArrayMemCost(types, columnCategories, null, null, null); + } + + public static long alignedTvListArrayMemCost( + TSDataType[] types, + TsTableColumnCategory[] columnCategories, + Object[] values) { + return alignedTvListArrayMemCost(types, columnCategories, values, null, null); } /** - * Get the single alignedTVList array mem cost by give types. - * - * @return AlignedTvListArrayMemSize + * Memory cost of one new array chunk when the TVList expands (timestamp chunk + bitmap slots for + * columns that track nulls). Value arrays are allocated lazily on write. */ public long alignedTvListArrayMemCost() { - long size = 0; - // value & bitmap array mem size - for (int column = 0; column < dataTypes.size(); column++) { - TSDataType type = dataTypes.get(column); - if (type != null) { - size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); - if (bitMaps != null && bitMaps.get(column) != null) { - size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1; + long size = PrimitiveArrayManager.ARRAY_SIZE * 8L; + if (indices != null) { + size += PrimitiveArrayManager.ARRAY_SIZE * 4L; + } + if (bitMaps != null) { + for (int column = 0; column < dataTypes.size(); column++) { + if (bitMaps.get(column) != null) { + size += emptyValueListArrayMemCost(); } } } - // size is 0 when all types are null - if (size == 0) { - return size; - } - // time array mem size - size += PrimitiveArrayManager.ARRAY_SIZE * 8L; - // index array mem size - size += (indices != null) ? PrimitiveArrayManager.ARRAY_SIZE * 4L : 0; - // array headers mem size size += (long) NUM_BYTES_ARRAY_HEADER * (2 + dataTypes.size()); - // Object references size in ArrayList size += (long) NUM_BYTES_OBJECT_REF * (2 + dataTypes.size()); return size; } /** - * Get the single column array mem cost by give type. - * - * @param type the type of the value column - * @return valueListArrayMemCost + * Memory cost of one new array chunk when expanding, including value arrays for columns written + * in the current insertion batch. */ - public static long valueListArrayMemCost(TSDataType type) { - long size = 0; - // value array mem size - size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); - // bitmap array mem size - size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1; - // array headers mem size - size += NUM_BYTES_ARRAY_HEADER; - // Object references size in ArrayList - size += NUM_BYTES_OBJECT_REF; + public long alignedTvListArrayMemCostForExpansion(List insertingTypes) { + long size = alignedTvListArrayMemCost(); + if (insertingTypes != null) { + for (TSDataType type : insertingTypes) { + if (type != null) { + size += valueListArrayMemCost(type); + } + } + } return size; } + /** Whether the last array chunk of the column has not allocated a value array yet. */ + public boolean isLastValueArrayUnallocated(int columnIndex) { + List columnValues = values.get(columnIndex); + if (columnValues.isEmpty()) { + return true; + } + return columnValues.get(columnValues.size() - 1) == null; + } + /** Build TsBlock by column. */ @SuppressWarnings("java:S6541") public TsBlock buildTsBlock( @@ -1175,8 +1235,8 @@ public TsBlock buildTsBlock( // time column for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) { // skip empty row - if (allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(getValueIndex(sortedRowIndex))) { + if (ignoreAllNullRows + && isEmptyValueRowAtValueIndex(getValueIndex(sortedRowIndex))) { continue; } if (isTimeDeleted(sortedRowIndex)) { @@ -1184,8 +1244,8 @@ public TsBlock buildTsBlock( } int nextRowIndex = sortedRowIndex + 1; while (nextRowIndex < rowCount - && ((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(getValueIndex(nextRowIndex))) + && ((ignoreAllNullRows + && isEmptyValueRowAtValueIndex(getValueIndex(nextRowIndex))) || (isTimeDeleted(nextRowIndex)))) { nextRowIndex++; } @@ -1218,8 +1278,8 @@ public TsBlock buildTsBlock( currentWriteRowIndex = 0; for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) { // skip empty row - if ((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(getValueIndex(sortedRowIndex))) + if ((ignoreAllNullRows + && isEmptyValueRowAtValueIndex(getValueIndex(sortedRowIndex))) || (isTimeDeleted(sortedRowIndex))) { continue; } @@ -1427,52 +1487,55 @@ public void serializeToWAL(IWALByteBufferView buffer) { for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; + boolean isNull = isNullValue(rowIndex, columnIndex); + Object valueArray = columnValues.get(arrayIndex); // value switch (dataTypes.get(columnIndex)) { case TEXT: case BLOB: case STRING: case OBJECT: - Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; - // In some scenario, the Binary in AlignedTVList will be null if this field is empty in - // current row. We need to handle this scenario to get rid of NPE. See the similar issue - // here: https://github.com/apache/iotdb/pull/9884 - // Furthermore, we use an empty Binary as a placeholder here. It won't lead to data - // error because whether this field is null or not is decided by the bitMap rather than - // the object's value here. - if (valueT != null) { - WALWriteUtils.write(valueT, buffer); - } else { - WALWriteUtils.write(new Binary(new byte[0]), buffer); - } + WALWriteUtils.write( + isNull || valueArray == null + ? Binary.EMPTY_VALUE + : ((Binary[]) valueArray)[elementIndex], + buffer); break; case FLOAT: - float valueF = ((float[]) columnValues.get(arrayIndex))[elementIndex]; - buffer.putFloat(valueF); + buffer.putFloat( + isNull || valueArray == null + ? Float.MIN_VALUE + : ((float[]) valueArray)[elementIndex]); break; case INT32: case DATE: - int valueI = ((int[]) columnValues.get(arrayIndex))[elementIndex]; - buffer.putInt(valueI); + buffer.putInt( + isNull || valueArray == null + ? Integer.MIN_VALUE + : ((int[]) valueArray)[elementIndex]); break; case INT64: case TIMESTAMP: - long valueL = ((long[]) columnValues.get(arrayIndex))[elementIndex]; - buffer.putLong(valueL); + buffer.putLong( + isNull || valueArray == null + ? Long.MIN_VALUE + : ((long[]) valueArray)[elementIndex]); break; case DOUBLE: - double valueD = ((double[]) columnValues.get(arrayIndex))[elementIndex]; - buffer.putDouble(valueD); + buffer.putDouble( + isNull || valueArray == null + ? Double.MIN_VALUE + : ((double[]) valueArray)[elementIndex]); break; case BOOLEAN: - boolean valueB = ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; - WALWriteUtils.write(valueB, buffer); + WALWriteUtils.write( + !isNull && valueArray != null && ((boolean[]) valueArray)[elementIndex], buffer); break; default: throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); } // bitmap - WALWriteUtils.write(isNullValue(rowIndex, columnIndex), buffer); + WALWriteUtils.write(isNull, buffer); } } @@ -1619,60 +1682,66 @@ public BitMap getAllValueColDeletedMap() { return getAllValueColDeletedMap(rowCount); } - public BitMap getAllValueColDeletedMap(int rowCount) { - // row exists when any column value exists - if (bitMaps == null) { - return null; + /** + * Whether all value columns are null at the given value index. Unmaterialized value arrays count + * as null. + */ + public boolean isAllValueColumnsNullAtValueIndex(int valueIndex) { + if (valueIndex >= rowCount) { + return false; } for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) { - if (values.get(columnIndex) != null && bitMaps.get(columnIndex) == null) { - return null; + if (!isNullValue(valueIndex, columnIndex)) { + return false; } } + return true; + } - byte[] rowBitsArr = new byte[rowCount / Byte.SIZE + 1]; - int bitsMapSize = - rowCount % ARRAY_SIZE == 0 ? rowCount / ARRAY_SIZE : rowCount / ARRAY_SIZE + 1; - boolean[] allNotNullArray = new boolean[bitsMapSize]; - Arrays.fill(rowBitsArr, (byte) 0xFF); - for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) { - List columnBitMaps = bitMaps.get(columnIndex); - if (columnBitMaps == null) { - Arrays.fill(rowBitsArr, (byte) 0x00); - break; - } else if (values.get(columnIndex) != null) { - int row = 0; - boolean isEnd = true; - for (int i = 0; i < bitsMapSize; i++) { - if (allNotNullArray[i]) { - row += ARRAY_SIZE; - continue; - } - - BitMap bitMap = columnBitMaps.get(i); - int index = row / Byte.SIZE; - int size = ((Math.min((rowCount - row), ARRAY_SIZE)) + 7) >>> 3; - row += ARRAY_SIZE; + /** + * Whether the row has no value columns for ignore-all-null-rows optimization (query / flush). + */ + public boolean isEmptyValueRowAtValueIndex(int valueIndex) { + if (allValueColDeletedMap != null && valueIndex < allValueColDeletedMap.getSize()) { + return allValueColDeletedMap.isMarked(valueIndex); + } + return isAllValueColumnsNullAtValueIndex(valueIndex); + } - if (bitMap == null) { - Arrays.fill(rowBitsArr, index, index + size, (byte) 0x00); - allNotNullArray[i] = true; - continue; - } + public BitMap getAllValueColDeletedMap(int rowCount) { + if (rowCount <= 0) { + return null; + } + // Marked bit means "all value columns are null" at this row index. + byte[] rowBitsArr = new byte[(rowCount + Byte.SIZE - 1) / Byte.SIZE]; + Arrays.fill(rowBitsArr, (byte) 0xFF); - byte bits = (byte) 0X00; + int bitsMapSize = (rowCount + ARRAY_SIZE - 1) / ARRAY_SIZE; + for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) { + List columnValues = values.get(columnIndex); + List columnBitMaps = bitMaps == null ? null : bitMaps.get(columnIndex); + + int row = 0; + for (int i = 0; i < bitsMapSize; i++) { + int limit = Math.min(ARRAY_SIZE, rowCount - row); + int index = row / Byte.SIZE; + int size = (limit + 7) >>> 3; + + Object valueArray = i < columnValues.size() ? columnValues.get(i) : null; + BitMap chunkBitMap = + columnBitMaps != null && i < columnBitMaps.size() ? columnBitMaps.get(i) : null; + + if (valueArray != null && chunkBitMap == null) { + // Materialized values without null marks in this chunk: row is not all-null. + Arrays.fill(rowBitsArr, index, index + size, (byte) 0x00); + } else if (chunkBitMap != null) { for (int j = 0; j < size; j++) { - rowBitsArr[index] &= bitMap.getByteArray()[j]; - bits |= rowBitsArr[index++]; - isEnd = false; + rowBitsArr[index + j] &= chunkBitMap.getByteArray()[j]; } - - allNotNullArray[i] = bits == (byte) 0; } + // valueArray == null and chunkBitMap == null: column unwritten in this chunk, all null. - if (isEnd) { - break; - } + row += ARRAY_SIZE; } } return new BitMap(rowCount, rowBitsArr); @@ -1822,6 +1891,16 @@ public AlignedTVListIterator( } } + private boolean isEmptyValueRow(int valueIndex) { + if (!ignoreAllNullRows) { + return false; + } + if (allValueColDeletedMap != null && valueIndex < allValueColDeletedMap.getSize()) { + return allValueColDeletedMap.isMarked(valueIndex); + } + return outer.isAllValueColumnsNullAtValueIndex(valueIndex); + } + @Override @SuppressWarnings("java:S6541") protected void prepareNext() { @@ -1833,8 +1912,7 @@ protected void prepareNext() { while (index < rows && !findValidRow) { // all columns values are deleted int convertedScanOrderValueIndex = getValueIndex(getScanOrderIndex(index)); - if ((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(convertedScanOrderValueIndex)) + if (isEmptyValueRow(convertedScanOrderValueIndex) || isTimeDeleted(convertedScanOrderValueIndex, false)) { index++; continue; @@ -1875,9 +1953,8 @@ protected void prepareNext() { while (index + 1 < rows && getTime(getScanOrderIndex(index + 1)) == getTime(getScanOrderIndex(index))) { index++; - // skip all-Null rows if allValueColDeletedMap exists - if (allValueColDeletedMap == null - || !allValueColDeletedMap.isMarked(getValueIndex(getScanOrderIndex(index)))) { + // skip all-null rows when merging duplicate timestamps + if (!isEmptyValueRow(getValueIndex(getScanOrderIndex(index)))) { for (int columnIndex = 0; columnIndex < dataTypeList.size(); columnIndex++) { if (!scanOrder.isAscending() && selectedIndices[columnIndex] != -1) { // non -1 value means it already set the latest point index @@ -2032,8 +2109,7 @@ public boolean hasNextBatch() { private boolean isRowInvalid( int rowIndex, long time, int[] deleteCursor, long[] filteredRowsByTimeFilter) { - if ((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(getValueIndex(getScanOrderIndex(rowIndex)))) + if (isEmptyValueRow(getValueIndex(getScanOrderIndex(rowIndex))) || isTimeDeleted(getScanOrderIndex(rowIndex)) || isPointDeleted(time, timeColumnDeletion, deleteCursor, scanOrder)) { return true; @@ -2367,7 +2443,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo break; } // skip empty row - if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(getValueIndex(index))) { + if (isEmptyValueRow(getValueIndex(index))) { continue; } if (isTimeDeleted(index)) { @@ -2375,9 +2451,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo } int nextRowIndex = index + 1; while (nextRowIndex < rows - && ((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(getValueIndex(nextRowIndex))) - || (isTimeDeleted(nextRowIndex)))) { + && (isEmptyValueRow(getValueIndex(nextRowIndex)) || (isTimeDeleted(nextRowIndex)))) { nextRowIndex++; } long time = getTime(index); @@ -2407,9 +2481,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo } for (int sortedRowIndex = startIndex; sortedRowIndex < index; sortedRowIndex++) { // skip empty row - if ((allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(getValueIndex(sortedRowIndex))) - || (isTimeDeleted(sortedRowIndex))) { + if (isEmptyValueRow(getValueIndex(sortedRowIndex)) || (isTimeDeleted(sortedRowIndex))) { continue; } long time = getTime(sortedRowIndex); From ff1190cd414115ebf695a9f1ec11913db22c5dcc Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 26 May 2026 15:36:31 +0800 Subject: [PATCH 2/2] spotless --- .../memtable/AlignedWritableMemChunk.java | 3 +-- .../dataregion/memtable/TsFileProcessor.java | 9 ++++----- .../db/utils/datastructure/AlignedTVList.java | 20 ++++++------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index d981c2fdbc124..e88ba8347ab9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -939,8 +939,7 @@ public long getTvListArrayMemCostIncrementForTablet( return memCostIncrement; } - private static boolean columnHasNonNullInRange( - Object column, BitMap bitMap, int start, int end) { + private static boolean columnHasNonNullInRange(Object column, BitMap bitMap, int start, int end) { if (bitMap == null) { return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 1c1628ed441b1..7670c8e033e3a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -910,8 +910,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins if (alignedMemChunk != null) { memTableIncrement += alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost(); } else { - memTableIncrement += - AlignedTVList.alignedTvListArrayMemCost(dataTypes, null, values); + memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, null, values); } } if (alignedMemChunk != null) { @@ -1091,8 +1090,7 @@ private void updateAlignedMemCost( incomingPointNum / PrimitiveArrayManager.ARRAY_SIZE + (incomingPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0); boolean[] columnHasNonNull = - buildAlignedColumnHasNonNull( - dataTypes, columns, bitMaps, columnCategories, start, end); + buildAlignedColumnHasNonNull(dataTypes, columns, bitMaps, columnCategories, start, end); memIncrements[0] += numArraysToAdd * AlignedTVList.alignedTvListArrayMemCost( @@ -1180,7 +1178,8 @@ private static boolean[] buildAlignedColumnHasNonNull( continue; } columnHasNonNull[i] = - alignedColumnHasNonNullInRange(columns[i], bitMaps == null ? null : bitMaps[i], start, end); + alignedColumnHasNonNullInRange( + columns[i], bitMaps == null ? null : bitMaps[i], start, end); } return columnHasNonNull; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 9e2ae4b3f0917..f2b95d9da3320 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -1149,8 +1149,7 @@ public static long alignedTvListArrayMemCost( } else { allocateValueArray = columns != null && columns[i] != null; } - size += - allocateValueArray ? valueListArrayMemCost(type) : emptyValueListArrayMemCost(); + size += allocateValueArray ? valueListArrayMemCost(type) : emptyValueListArrayMemCost(); } if (fieldColumnCount == 0) { return 0; @@ -1164,9 +1163,7 @@ public static long alignedTvListArrayMemCost( } public static long alignedTvListArrayMemCost( - TSDataType[] types, - TsTableColumnCategory[] columnCategories, - Object[] values) { + TSDataType[] types, TsTableColumnCategory[] columnCategories, Object[] values) { return alignedTvListArrayMemCost(types, columnCategories, values, null, null); } @@ -1235,8 +1232,7 @@ public TsBlock buildTsBlock( // time column for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) { // skip empty row - if (ignoreAllNullRows - && isEmptyValueRowAtValueIndex(getValueIndex(sortedRowIndex))) { + if (ignoreAllNullRows && isEmptyValueRowAtValueIndex(getValueIndex(sortedRowIndex))) { continue; } if (isTimeDeleted(sortedRowIndex)) { @@ -1244,8 +1240,7 @@ && isEmptyValueRowAtValueIndex(getValueIndex(sortedRowIndex))) { } int nextRowIndex = sortedRowIndex + 1; while (nextRowIndex < rowCount - && ((ignoreAllNullRows - && isEmptyValueRowAtValueIndex(getValueIndex(nextRowIndex))) + && ((ignoreAllNullRows && isEmptyValueRowAtValueIndex(getValueIndex(nextRowIndex))) || (isTimeDeleted(nextRowIndex)))) { nextRowIndex++; } @@ -1278,8 +1273,7 @@ && isEmptyValueRowAtValueIndex(getValueIndex(nextRowIndex))) currentWriteRowIndex = 0; for (int sortedRowIndex = 0; sortedRowIndex < rowCount; sortedRowIndex++) { // skip empty row - if ((ignoreAllNullRows - && isEmptyValueRowAtValueIndex(getValueIndex(sortedRowIndex))) + if ((ignoreAllNullRows && isEmptyValueRowAtValueIndex(getValueIndex(sortedRowIndex))) || (isTimeDeleted(sortedRowIndex))) { continue; } @@ -1698,9 +1692,7 @@ public boolean isAllValueColumnsNullAtValueIndex(int valueIndex) { return true; } - /** - * Whether the row has no value columns for ignore-all-null-rows optimization (query / flush). - */ + /** Whether the row has no value columns for ignore-all-null-rows optimization (query / flush). */ public boolean isEmptyValueRowAtValueIndex(int valueIndex) { if (allValueColDeletedMap != null && valueIndex < allValueColDeletedMap.getSize()) { return allValueColDeletedMap.isMarked(valueIndex);