From d295b5c34afff42670cbcba3782e6a5743c3c551 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:12:28 +0800 Subject: [PATCH 001/131] add interface for addValueOfDataType --- java/pom.xml | 2 +- java/tsfile/pom.xml | 6 +- .../read/common/type/AbstractIntType.java | 6 + .../read/common/type/AbstractLongType.java | 6 + .../tsfile/read/common/type/AbstractType.java | 13 ++ .../read/common/type/AbstractVarcharType.java | 24 ++++ .../tsfile/read/common/type/BlobType.java | 22 ++++ .../tsfile/read/common/type/BooleanType.java | 6 + .../tsfile/read/common/type/DateType.java | 9 ++ .../tsfile/read/common/type/DoubleType.java | 6 + .../tsfile/read/common/type/FloatType.java | 6 + .../tsfile/read/common/type/ObjectType.java | 22 ++++ .../apache/tsfile/read/common/type/Type.java | 10 ++ .../apache/tsfile/write/record/Tablet.java | 117 +----------------- pom.xml | 4 +- 15 files changed, 138 insertions(+), 121 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 3106d99bb..00a18d68d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -188,7 +188,7 @@ org.apache.tsfile,,javax,java,\# - + UNIX diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml index ec327381c..768d23f95 100644 --- a/java/tsfile/pom.xml +++ b/java/tsfile/pom.xml @@ -145,10 +145,10 @@ shade - + false - + @@ -185,7 +185,7 @@ org.apache.tsfile.* common;inline=true false - + <_removeheaders>Bnd-LastModified,Built-By org.apache.tsfile diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 8425a8793..f35453ad7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -68,6 +68,12 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeInt((int) value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + checkValueType(value, Integer.class, "Integer"); + ((int[]) column)[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index e8e1ecbc3..7726792c6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -68,6 +68,12 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeLong((long) value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + checkValueType(value, Long.class, "Long"); + ((long[]) column)[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 937b7fdcc..8f7c116b3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -19,8 +19,21 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.i18n.Messages; + public abstract class AbstractType implements Type { + protected void checkValueType(Object value, Class expectedClass, String expectedType) { + if (value != null && !expectedClass.isInstance(value)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + expectedType, + getDisplayName(), + value.getClass().getName())); + } + } + @Override public String toString() { return getDisplayName(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index d09ed07c6..9e87df4a7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -38,6 +40,28 @@ public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + ((Binary[]) column)[rowIndex] = toBinaryValue(value); + } + + protected Binary toBinaryValue(Object value) { + if (value != null && !(value instanceof Binary) && !(value instanceof String)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + "Binary or String", + getDisplayName(), + value.getClass().getName())); + } + if (value instanceof Binary) { + return (Binary) value; + } + return value != null + ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) + : Binary.EMPTY_VALUE; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index d02ca7168..0bf138632 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -43,6 +45,26 @@ public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + if (value != null && !(value instanceof Binary) && !(value instanceof String)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + "Binary or String", + getDisplayName(), + value.getClass().getName())); + } + if (value instanceof Binary) { + ((Binary[]) column)[rowIndex] = (Binary) value; + } else { + ((Binary[]) column)[rowIndex] = + value != null + ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) + : Binary.EMPTY_VALUE; + } + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index e00d7ecb8..e7a8df126 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -42,6 +42,12 @@ public void writeBoolean(ColumnBuilder builder, boolean value) { builder.writeBoolean(value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + checkValueType(value, Boolean.class, "Boolean"); + ((boolean[]) column)[rowIndex] = value != null && (boolean) value; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 88d17e242..94deb5957 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -19,12 +19,21 @@ package org.apache.tsfile.read.common.type; +import java.time.LocalDate; + public class DateType extends AbstractIntType { public static final DateType DATE = new DateType(); + private static final LocalDate EMPTY_DATE = LocalDate.of(1000, 1, 1); private DateType() {} + @Override + public void addValue(int rowIndex, Object value, Object column) { + checkValueType(value, LocalDate.class, "LocalDate"); + ((LocalDate[]) column)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 6e325c066..a475f9b96 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -72,6 +72,12 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeDouble(value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + checkValueType(value, Double.class, "Double"); + ((double[]) column)[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 6417f0f52..d27e44635 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -72,6 +72,12 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeFloat((float) value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + checkValueType(value, Float.class, "Float"); + ((float[]) column)[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index b30b621f1..76cc91571 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -43,6 +45,26 @@ public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); } + @Override + public void addValue(int rowIndex, Object value, Object column) { + if (value != null && !(value instanceof Binary) && !(value instanceof String)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + "Binary or String", + getDisplayName(), + value.getClass().getName())); + } + if (value instanceof Binary) { + ((Binary[]) column)[rowIndex] = (Binary) value; + } else { + ((Binary[]) column)[rowIndex] = + value != null + ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) + : Binary.EMPTY_VALUE; + } + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index cb6b60944..e514cb08f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -21,12 +21,17 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import java.util.List; public interface Type { + static Type fromTsDataType(TSDataType tsDataType) { + return TypeFactory.getType(tsDataType); + } + /** Gets a boolean at {@code position}. */ default boolean getBoolean(Column c, int position) { throw new UnsupportedOperationException(getClass().getName()); @@ -97,6 +102,11 @@ default void writeObject(ColumnBuilder builder, Object value) { builder.writeObject(value); } + /** Adds a value to the array column at {@code rowIndex}. */ + default void addValue(int rowIndex, Object value, Object column) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 2bad6c953..ef4d0f046 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -28,6 +28,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -67,7 +68,6 @@ public class Tablet implements Accountable { private static final long TABLET_SIZE = RamUsageEstimator.shallowSizeOfInstance(Tablet.class); private static final int DEFAULT_SIZE = 1024; - private static final LocalDate EMPTY_DATE = LocalDate.of(1000, 1, 1); /** DeviceId if using tree-view interfaces or TableName when using table-view interfaces. */ private String insertTargetName; @@ -310,120 +310,7 @@ private void addValueOfDataType( // Mark the null value position updateBitMap(rowIndex, indexOfSchema, value == null); - switch (dataType) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - { - if (value != null && !(value instanceof Binary) && !(value instanceof String)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Binary or String", - dataType, - value.getClass().getName())); - } - final Binary[] sensor = (Binary[]) values[indexOfSchema]; - if (value instanceof Binary) { - sensor[rowIndex] = (Binary) value; - } else { - sensor[rowIndex] = - value != null - ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) - : Binary.EMPTY_VALUE; - } - break; - } - case FLOAT: - { - if (value != null && !(value instanceof Float)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Float", - dataType, - value.getClass().getName())); - } - final float[] sensor = (float[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; - break; - } - case INT32: - { - if (value != null && !(value instanceof Integer)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Integer", - dataType, - value.getClass().getName())); - } - final int[] sensor = (int[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; - break; - } - case DATE: - { - if (value != null && !(value instanceof LocalDate)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "LocalDate", - dataType, - value.getClass().getName())); - } - final LocalDate[] sensor = (LocalDate[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; - break; - } - case INT64: - case TIMESTAMP: - { - if (value != null && !(value instanceof Long)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Long", - dataType, - value.getClass().getName())); - } - final long[] sensor = (long[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; - break; - } - case DOUBLE: - { - if (value != null && !(value instanceof Double)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Double", - dataType, - value.getClass().getName())); - } - final double[] sensor = (double[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; - break; - } - case BOOLEAN: - { - if (value != null && !(value instanceof Boolean)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Boolean", - dataType, - value.getClass().getName())); - } - final boolean[] sensor = (boolean[]) values[indexOfSchema]; - sensor[rowIndex] = value != null && (boolean) value; - break; - } - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + Type.fromTsDataType(dataType).addValue(rowIndex, value, values[indexOfSchema]); } @TsFileApi diff --git a/pom.xml b/pom.xml index eb9380fb9..68eef8e04 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 17 17 17 - + false 3.30.2-b1 2.44.3 @@ -263,7 +263,7 @@ validate - + From 23c2f35225201a4d86de1e837d70be4db5aa4f49 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:17:13 +0800 Subject: [PATCH 002/131] add interface for createValueColumnOfDataType --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DateType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../apache/tsfile/write/record/Tablet.java | 34 +------------------ 11 files changed, 51 insertions(+), 33 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index f35453ad7..e4085ae70 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -74,6 +74,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((int[]) column)[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; } + @Override + public Object createArray(int capacity) { + return new int[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 7726792c6..9dfbad8ab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -74,6 +74,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((long[]) column)[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; } + @Override + public Object createArray(int capacity) { + return new long[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 9e87df4a7..2b3a924d0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -62,6 +62,11 @@ protected Binary toBinaryValue(Object value) { : Binary.EMPTY_VALUE; } + @Override + public Object createArray(int capacity) { + return new Binary[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 0bf138632..a6dc0cb55 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -65,6 +65,11 @@ public void addValue(int rowIndex, Object value, Object column) { } } + @Override + public Object createArray(int capacity) { + return new Binary[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index e7a8df126..6618b9a48 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -48,6 +48,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((boolean[]) column)[rowIndex] = value != null && (boolean) value; } + @Override + public Object createArray(int capacity) { + return new boolean[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 94deb5957..763080e3e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -34,6 +34,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((LocalDate[]) column)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; } + @Override + public Object createArray(int capacity) { + return new LocalDate[capacity]; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index a475f9b96..b74188363 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -78,6 +78,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((double[]) column)[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; } + @Override + public Object createArray(int capacity) { + return new double[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index d27e44635..d9f754b30 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -78,6 +78,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((float[]) column)[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; } + @Override + public Object createArray(int capacity) { + return new float[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 76cc91571..fefb55df8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -65,6 +65,11 @@ public void addValue(int rowIndex, Object value, Object column) { } } + @Override + public Object createArray(int capacity) { + return new Binary[capacity]; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index e514cb08f..6c28f47c5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -107,6 +107,11 @@ default void addValue(int rowIndex, Object value, Object column) { throw new UnsupportedOperationException(getClass().getName()); } + /** Creates an array column with {@code capacity} entries for this type. */ + default Object createArray(int capacity) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index ef4d0f046..c226694ba 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -598,39 +598,7 @@ private Object createValueColumnOfDataType(TSDataType dataType) { } private Object createValueColumnOfDataType(TSDataType dataType, int capacity) { - - Object valueColumn; - switch (dataType) { - case INT32: - valueColumn = new int[capacity]; - break; - case INT64: - case TIMESTAMP: - valueColumn = new long[capacity]; - break; - case FLOAT: - valueColumn = new float[capacity]; - break; - case DOUBLE: - valueColumn = new double[capacity]; - break; - case BOOLEAN: - valueColumn = new boolean[capacity]; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valueColumn = new Binary[capacity]; - break; - case DATE: - valueColumn = new LocalDate[capacity]; - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } - return valueColumn; + return Type.fromTsDataType(dataType).createArray(capacity); } /** Serialize {@link Tablet} */ From 76b1c08acd0c009fba54dbd9f77fa855e9fd3542 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:30:24 +0800 Subject: [PATCH 003/131] add interface for serializeColumn --- .../read/common/type/AbstractIntType.java | 17 +++ .../read/common/type/AbstractLongType.java | 17 +++ .../tsfile/read/common/type/AbstractType.java | 29 +++++ .../read/common/type/AbstractVarcharType.java | 13 +++ .../tsfile/read/common/type/BlobType.java | 13 +++ .../tsfile/read/common/type/BooleanType.java | 18 +++ .../tsfile/read/common/type/DateType.java | 18 +++ .../tsfile/read/common/type/DoubleType.java | 17 +++ .../tsfile/read/common/type/FloatType.java | 17 +++ .../tsfile/read/common/type/ObjectType.java | 13 +++ .../apache/tsfile/read/common/type/Type.java | 13 +++ .../apache/tsfile/write/record/Tablet.java | 110 ++---------------- 12 files changed, 194 insertions(+), 101 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index e4085ae70..d4b19d6cf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -22,7 +22,10 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -79,6 +82,20 @@ public Object createArray(int capacity) { return new int[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return Math.multiplyExact(Integer.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + int[] values = (int[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 9dfbad8ab..42bd2590b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -22,7 +22,10 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -79,6 +82,20 @@ public Object createArray(int capacity) { return new long[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return Math.multiplyExact(Long.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + long[] values = (long[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 8f7c116b3..46e8aeffe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -20,6 +20,12 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; public abstract class AbstractType implements Type { @@ -34,6 +40,29 @@ protected void checkValueType(Object value, Class expectedClass, String expec } } + protected int serializedSizeOfBinaryValues(Object column, int rowSize) { + Binary[] binaryValues = (Binary[]) column; + int size = 0; + for (int i = 0; i < rowSize; i++) { + size = Math.addExact(size, Byte.BYTES); + if (binaryValues[i] != null) { + size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(binaryValues[i])); + } + } + return size; + } + + protected void serializeBinaryValues(Object array, int rowSize, DataOutputStream stream) + throws IOException { + Binary[] binaryValues = (Binary[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(binaryValues[i] != null), stream); + if (binaryValues[i] != null) { + ReadWriteIOUtils.write(binaryValues[i], stream); + } + } + } + @Override public String toString() { return getDisplayName(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 2b3a924d0..a62294643 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -26,6 +26,8 @@ import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -67,6 +69,17 @@ public Object createArray(int capacity) { return new Binary[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return serializedSizeOfBinaryValues(column, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + serializeBinaryValues(array, rowSize, stream); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index a6dc0cb55..d058c6240 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -26,6 +26,8 @@ import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -70,6 +72,17 @@ public Object createArray(int capacity) { return new Binary[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return serializedSizeOfBinaryValues(column, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + serializeBinaryValues(array, rowSize, stream); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 6618b9a48..f504cae03 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -22,7 +22,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -53,6 +57,20 @@ public Object createArray(int capacity) { return new boolean[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return rowSize; + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + boolean[] values = (boolean[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(values[i]), stream); + } + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 763080e3e..9ca32e56e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -19,6 +19,11 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; import java.time.LocalDate; public class DateType extends AbstractIntType { @@ -39,6 +44,19 @@ public Object createArray(int capacity) { return new LocalDate[capacity]; } + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + LocalDate[] values = (LocalDate[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write( + values[i] == null + ? DateUtils.EMPTY_DATE_INT + : DateUtils.parseDateExpressionToInt(values[i]), + stream); + } + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index b74188363..4a6d8e6b9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -22,7 +22,10 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -83,6 +86,20 @@ public Object createArray(int capacity) { return new double[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return Math.multiplyExact(Double.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + double[] values = (double[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index d9f754b30..a58bff9f0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -22,7 +22,10 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -83,6 +86,20 @@ public Object createArray(int capacity) { return new float[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return Math.multiplyExact(Float.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + float[] values = (float[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index fefb55df8..4d1165a49 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -26,6 +26,8 @@ import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -70,6 +72,17 @@ public Object createArray(int capacity) { return new Binary[capacity]; } + @Override + public int serializedSize(Object column, int rowSize) { + return serializedSizeOfBinaryValues(column, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + serializeBinaryValues(array, rowSize, stream); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 6c28f47c5..f1086a936 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -24,6 +24,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.List; public interface Type { @@ -112,6 +114,17 @@ default Object createArray(int capacity) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the serialized byte size of the array column with {@code rowSize} entries. */ + default int serializedSize(Object column, int rowSize) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Serializes the array column with {@code rowSize} entries. */ + default void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index c226694ba..4e3b35234 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -679,55 +679,21 @@ private int serializedSizeOfValues() { if (values != null) { final int columnCount = schemas == null ? 0 : schemas.size(); for (int i = 0; i < columnCount; i++) { - size = Math.addExact(size, serializedSizeOfColumn(schemas.get(i).getType(), values[i])); + size = + Math.addExact( + size, + serializedSizeOfColumn(Type.fromTsDataType(schemas.get(i).getType()), values[i])); } } return size; } - private int serializedSizeOfColumn(final TSDataType dataType, final Object column) { + private int serializedSizeOfColumn(final Type type, final Object column) { int size = Byte.BYTES; if (column == null) { return size; } - switch (dataType) { - case INT32: - return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize)); - case DATE: - return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize)); - case INT64: - case TIMESTAMP: - return Math.addExact(size, Math.multiplyExact(Long.BYTES, rowSize)); - case FLOAT: - return Math.addExact(size, Math.multiplyExact(Float.BYTES, rowSize)); - case DOUBLE: - return Math.addExact(size, Math.multiplyExact(Double.BYTES, rowSize)); - case BOOLEAN: - return Math.addExact(size, rowSize); - case TEXT: - case STRING: - case BLOB: - case OBJECT: - return Math.addExact(size, serializedSizeOfBinaryValues((Binary[]) column)); - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } - } - - private static int serializedSizeOfBinaryValues(final Binary[] binaryValues, final int rowSize) { - int size = 0; - for (int j = 0; j < rowSize; j++) { - size = Math.addExact(size, Byte.BYTES); - if (binaryValues[j] != null) { - size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(binaryValues[j])); - } - } - return size; - } - - private int serializedSizeOfBinaryValues(final Binary[] binaryValues) { - return serializedSizeOfBinaryValues(binaryValues, rowSize); + return Math.addExact(size, type.serializedSize(column, rowSize)); } /** Serialize {@link MeasurementSchema}s */ @@ -781,75 +747,17 @@ private void writeValues(DataOutputStream stream) throws IOException { if (values != null) { int size = (schemas == null ? 0 : schemas.size()); for (int i = 0; i < size; i++) { - serializeColumn(schemas.get(i).getType(), values[i], stream, columnCategories.get(i)); + serializeColumn(Type.fromTsDataType(schemas.get(i).getType()), values[i], stream); } } } - private void serializeColumn( - TSDataType dataType, Object column, DataOutputStream stream, ColumnCategory columnCategory) + private void serializeColumn(Type type, Object column, DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(column != null), stream); if (column != null) { - switch (dataType) { - case INT32: - int[] intValues = (int[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(intValues[j], stream); - } - break; - case DATE: - LocalDate[] dateValues = (LocalDate[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write( - dateValues[j] == null - ? DateUtils.EMPTY_DATE_INT - : DateUtils.parseDateExpressionToInt(dateValues[j]), - stream); - } - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(longValues[j], stream); - } - break; - case FLOAT: - float[] floatValues = (float[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(floatValues[j], stream); - } - break; - case DOUBLE: - double[] doubleValues = (double[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(doubleValues[j], stream); - } - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(BytesUtils.boolToByte(boolValues[j]), stream); - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[] binaryValues = (Binary[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(BytesUtils.boolToByte(binaryValues[j] != null), stream); - if (binaryValues[j] != null) { - ReadWriteIOUtils.write(binaryValues[j], stream); - } - } - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + type.serializeArray(column, rowSize, stream); } } From 2f2b5cf2b103ffb55ab1e5dcc4f79c1386c042cb Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:37:59 +0800 Subject: [PATCH 004/131] add interface for readvaluesFromBuffer --- .../read/common/type/AbstractIntType.java | 10 +++ .../read/common/type/AbstractLongType.java | 10 +++ .../tsfile/read/common/type/AbstractType.java | 10 +++ .../read/common/type/AbstractVarcharType.java | 6 ++ .../tsfile/read/common/type/BlobType.java | 6 ++ .../tsfile/read/common/type/BooleanType.java | 10 +++ .../tsfile/read/common/type/DateType.java | 10 +++ .../tsfile/read/common/type/DoubleType.java | 10 +++ .../tsfile/read/common/type/FloatType.java | 10 +++ .../tsfile/read/common/type/ObjectType.java | 6 ++ .../apache/tsfile/read/common/type/Type.java | 6 ++ .../apache/tsfile/write/record/Tablet.java | 66 +------------------ 12 files changed, 95 insertions(+), 65 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index d4b19d6cf..ec6341939 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -96,6 +97,15 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + int[] values = new int[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readInt(buffer); + } + return values; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 42bd2590b..4dfeafefd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -96,6 +97,15 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + long[] values = new long[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readLong(buffer); + } + return values; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 46e8aeffe..c0bce11c3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; public abstract class AbstractType implements Type { @@ -63,6 +64,15 @@ protected void serializeBinaryValues(Object array, int rowSize, DataOutputStream } } + protected Object deserializeBinaryValues(ByteBuffer buffer, int rowSize) { + Binary[] values = new Binary[rowSize]; + for (int i = 0; i < rowSize; i++) { + boolean isNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); + values[i] = isNotNull ? ReadWriteIOUtils.readBinary(buffer) : Binary.EMPTY_VALUE; + } + return values; + } + @Override public String toString() { return getDisplayName(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index a62294643..bf48a1811 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -28,6 +28,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -80,6 +81,11 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) serializeBinaryValues(array, rowSize, stream); } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + return deserializeBinaryValues(buffer, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index d058c6240..4c694fb07 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -28,6 +28,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -83,6 +84,11 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) serializeBinaryValues(array, rowSize, stream); } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + return deserializeBinaryValues(buffer, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index f504cae03..864b83b1a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -71,6 +72,15 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + boolean[] values = new boolean[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); + } + return values; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 9ca32e56e..3cc2787e3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -24,6 +24,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.time.LocalDate; public class DateType extends AbstractIntType { @@ -57,6 +58,15 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + LocalDate[] values = new LocalDate[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = DateUtils.parseIntToLocalDate(ReadWriteIOUtils.readInt(buffer)); + } + return values; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 4a6d8e6b9..dceb76a95 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -100,6 +101,15 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + double[] values = new double[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readDouble(buffer); + } + return values; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index a58bff9f0..4f3783f86 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -100,6 +101,15 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + float[] values = new float[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readFloat(buffer); + } + return values; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 4d1165a49..7bd38308a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -28,6 +28,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -83,6 +84,11 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) serializeBinaryValues(array, rowSize, stream); } + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + return deserializeBinaryValues(buffer, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index f1086a936..b0ed44a82 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; public interface Type { @@ -125,6 +126,11 @@ default void serializeArray(Object array, int rowSize, DataOutputStream stream) throw new UnsupportedOperationException(getClass().getName()); } + /** Deserializes an array with {@code rowSize} entries from {@code buffer}. */ + default Object deserializeArray(ByteBuffer buffer, int rowSize) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 4e3b35234..260fdb8a6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -33,7 +33,6 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -844,70 +843,7 @@ public static Object[] readvaluesFromBuffer( boolean isValueColumnsNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); if (isValueColumnsNotNull) { - switch (types[i]) { - case BOOLEAN: - boolean[] boolValues = new boolean[rowSize]; - for (int index = 0; index < rowSize; index++) { - boolValues[index] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); - } - values[i] = boolValues; - break; - case INT32: - int[] intValues = new int[rowSize]; - for (int index = 0; index < rowSize; index++) { - intValues[index] = ReadWriteIOUtils.readInt(byteBuffer); - } - values[i] = intValues; - break; - case DATE: - LocalDate[] dateValues = new LocalDate[rowSize]; - for (int index = 0; index < rowSize; index++) { - dateValues[index] = - DateUtils.parseIntToLocalDate(ReadWriteIOUtils.readInt(byteBuffer)); - } - values[i] = dateValues; - break; - case INT64: - case TIMESTAMP: - long[] longValues = new long[rowSize]; - for (int index = 0; index < rowSize; index++) { - longValues[index] = ReadWriteIOUtils.readLong(byteBuffer); - } - values[i] = longValues; - break; - case FLOAT: - float[] floatValues = new float[rowSize]; - for (int index = 0; index < rowSize; index++) { - floatValues[index] = ReadWriteIOUtils.readFloat(byteBuffer); - } - values[i] = floatValues; - break; - case DOUBLE: - double[] doubleValues = new double[rowSize]; - for (int index = 0; index < rowSize; index++) { - doubleValues[index] = ReadWriteIOUtils.readDouble(byteBuffer); - } - values[i] = doubleValues; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[] binaryValues = new Binary[rowSize]; - for (int index = 0; index < rowSize; index++) { - boolean isNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); - if (isNotNull) { - binaryValues[index] = ReadWriteIOUtils.readBinary(byteBuffer); - } else { - binaryValues[index] = Binary.EMPTY_VALUE; - } - } - values[i] = binaryValues; - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.tablet_client_type_not_supported", types[i])); - } + values[i] = Type.fromTsDataType(types[i]).deserializeArray(byteBuffer, rowSize); } } return values; From a3420576226231b3855872d901817cf1525351d0 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:44:59 +0800 Subject: [PATCH 005/131] add interface for equals --- .../read/common/type/AbstractIntType.java | 7 ++ .../read/common/type/AbstractLongType.java | 7 ++ .../tsfile/read/common/type/AbstractType.java | 20 ++++ .../read/common/type/AbstractVarcharType.java | 5 + .../tsfile/read/common/type/BlobType.java | 5 + .../tsfile/read/common/type/BooleanType.java | 7 ++ .../tsfile/read/common/type/DateType.java | 7 ++ .../tsfile/read/common/type/DoubleType.java | 7 ++ .../tsfile/read/common/type/FloatType.java | 7 ++ .../tsfile/read/common/type/ObjectType.java | 5 + .../apache/tsfile/read/common/type/Type.java | 5 + .../apache/tsfile/write/record/Tablet.java | 96 +------------------ 12 files changed, 85 insertions(+), 93 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index ec6341939..4d3227aad 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -106,6 +107,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((int[]) left, 0, rowSize, (int[]) right, 0, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 4dfeafefd..8933cf02a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -106,6 +107,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((long[]) left, 0, rowSize, (long[]) right, 0, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index c0bce11c3..0c95cb90d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -26,7 +26,9 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.lang.reflect.Array; import java.nio.ByteBuffer; +import java.util.Objects; public abstract class AbstractType implements Type { @@ -73,6 +75,24 @@ protected Object deserializeBinaryValues(ByteBuffer buffer, int rowSize) { return values; } + protected boolean hasEnoughLength(Object left, Object right, int rowSize) { + return Array.getLength(left) >= rowSize && Array.getLength(right) >= rowSize; + } + + protected boolean binaryArrayEquals(Object left, Object right, int rowSize) { + if (!hasEnoughLength(left, right, rowSize)) { + return false; + } + Binary[] leftValues = (Binary[]) left; + Binary[] rightValues = (Binary[]) right; + for (int i = 0; i < rowSize; i++) { + if (!Objects.equals(leftValues[i], rightValues[i])) { + return false; + } + } + return true; + } + @Override public String toString() { return getDisplayName(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index bf48a1811..549b7a2e6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -86,6 +86,11 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return binaryArrayEquals(left, right, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 4c694fb07..61da6dc6d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -89,6 +89,11 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return binaryArrayEquals(left, right, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 864b83b1a..655fe9deb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -28,6 +28,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -81,6 +82,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((boolean[]) left, 0, rowSize, (boolean[]) right, 0, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 3cc2787e3..c1a667d34 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.time.LocalDate; +import java.util.Arrays; public class DateType extends AbstractIntType { @@ -67,6 +68,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((LocalDate[]) left, 0, rowSize, (LocalDate[]) right, 0, rowSize); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index dceb76a95..c64d2eb42 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -110,6 +111,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((double[]) left, 0, rowSize, (double[]) right, 0, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 4f3783f86..cb7444220 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -110,6 +111,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((float[]) left, 0, rowSize, (float[]) right, 0, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 7bd38308a..3f2d6f0c7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -89,6 +89,11 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return binaryArrayEquals(left, right, rowSize); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index b0ed44a82..7fd898dbf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -131,6 +131,11 @@ default Object deserializeArray(ByteBuffer buffer, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns whether two array columns are equal in the first {@code rowSize} entries. */ + default boolean arrayEquals(Object left, Object right, int rowSize) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 260fdb8a6..6b1bd7311 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -36,7 +36,6 @@ import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -908,98 +907,9 @@ public boolean equals(Object o) { return false; } - switch (schemas.get(i).getType()) { - case INT32: - int[] thisIntValues = (int[]) values[i]; - int[] thatIntValues = (int[]) thatValues[i]; - if (thisIntValues.length < rowSize || thatIntValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisIntValues[j] != thatIntValues[j]) { - return false; - } - } - break; - case DATE: - LocalDate[] thisDateValues = (LocalDate[]) values[i]; - LocalDate[] thatDateValues = (LocalDate[]) thatValues[i]; - if (thisDateValues.length < rowSize || thatDateValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (!thisDateValues[j].equals(thatDateValues[j])) { - return false; - } - } - break; - case INT64: - case TIMESTAMP: - long[] thisLongValues = (long[]) values[i]; - long[] thatLongValues = (long[]) thatValues[i]; - if (thisLongValues.length < rowSize || thatLongValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisLongValues[j] != thatLongValues[j]) { - return false; - } - } - break; - case FLOAT: - float[] thisFloatValues = (float[]) values[i]; - float[] thatFloatValues = (float[]) thatValues[i]; - if (thisFloatValues.length < rowSize || thatFloatValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisFloatValues[j] != thatFloatValues[j]) { - return false; - } - } - break; - case DOUBLE: - double[] thisDoubleValues = (double[]) values[i]; - double[] thatDoubleValues = (double[]) thatValues[i]; - if (thisDoubleValues.length < rowSize || thatDoubleValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisDoubleValues[j] != thatDoubleValues[j]) { - return false; - } - } - break; - case BOOLEAN: - boolean[] thisBooleanValues = (boolean[]) values[i]; - boolean[] thatBooleanValues = (boolean[]) thatValues[i]; - if (thisBooleanValues.length < rowSize || thatBooleanValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisBooleanValues[j] != thatBooleanValues[j]) { - return false; - } - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[] thisBinaryValues = (Binary[]) values[i]; - Binary[] thatBinaryValues = (Binary[]) thatValues[i]; - if (thisBinaryValues.length < rowSize || thatBinaryValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (!thisBinaryValues[j].equals(thatBinaryValues[j])) { - return false; - } - } - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", schemas.get(i).getType())); + if (!Type.fromTsDataType(schemas.get(i).getType()) + .arrayEquals(values[i], thatValues[i], rowSize)) { + return false; } } From 671c97c584cbc38c9360102bc738863dfdc0ac14 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:50:17 +0800 Subject: [PATCH 006/131] add interface for getValue --- .../tsfile/read/common/type/AbstractType.java | 5 ++++ .../apache/tsfile/read/common/type/Type.java | 5 ++++ .../apache/tsfile/write/record/Tablet.java | 24 +------------------ 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 0c95cb90d..3a90a1230 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -93,6 +93,11 @@ protected boolean binaryArrayEquals(Object left, Object right, int rowSize) { return true; } + @Override + public Object getValue(Object column, int rowIndex) { + return Array.get(column, rowIndex); + } + @Override public String toString() { return getDisplayName(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 7fd898dbf..fbed7a4c6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -136,6 +136,11 @@ default boolean arrayEquals(Object left, Object right, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the value at {@code rowIndex} from an array column in Tablet. */ + default Object getValue(Object column, int rowIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 6b1bd7311..2df05e065 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -987,29 +987,7 @@ public Object getValue(int i, int j) { if (isNull(i, j)) { return null; } - switch (schemas.get(j).getType()) { - case BLOB: - case TEXT: - case STRING: - case OBJECT: - return ((Binary[]) values[j])[i]; - case INT32: - return ((int[]) values[j])[i]; - case FLOAT: - return ((float[]) values[j])[i]; - case DOUBLE: - return ((double[]) values[j])[i]; - case BOOLEAN: - return ((boolean[]) values[j])[i]; - case INT64: - case TIMESTAMP: - return ((long[]) values[j])[i]; - case DATE: - return ((LocalDate[]) values[j])[i]; - default: - throw new IllegalArgumentException( - Messages.format("error.write.tablet_unsupported_type", schemas.get(j).getType())); - } + return Type.fromTsDataType(schemas.get(j).getType()).getValue(values[j], i); } /** From 93eaa340e3c3afdf87aff95a51979cc84f5d48f6 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:56:28 +0800 Subject: [PATCH 007/131] add interface for ramBytesUsed --- .../read/common/type/AbstractIntType.java | 6 ++++ .../read/common/type/AbstractLongType.java | 6 ++++ .../tsfile/read/common/type/FloatType.java | 6 ++++ .../apache/tsfile/read/common/type/Type.java | 5 ++++ .../apache/tsfile/write/record/Tablet.java | 28 +------------------ 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 4d3227aad..d72ecfc2d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -113,6 +114,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { && Arrays.equals((int[]) left, 0, rowSize, (int[]) right, 0, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((int[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 8933cf02a..9a0dddf9a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -113,6 +114,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { && Arrays.equals((long[]) left, 0, rowSize, (long[]) right, 0, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((long[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index cb7444220..04ca47064 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -117,6 +118,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { && Arrays.equals((float[]) left, 0, rowSize, (float[]) right, 0, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((float[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index fbed7a4c6..b451fe983 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -141,6 +141,11 @@ default Object getValue(Object column, int rowIndex) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the retained memory size of an array column in Tablet. */ + default long arrayRamBytesUsed(Object array) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 2df05e065..f30c298fd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -1287,33 +1287,7 @@ public long ramBytesUsed() { if (values == null || values.length <= column) { continue; } - switch (tsDataType) { - case INT64: - case TIMESTAMP: - totalSizeInBytes += RamUsageEstimator.sizeOf((long[]) values[column]); - break; - case DATE: - totalSizeInBytes += RamUsageEstimator.sizeOf((LocalDate[]) values[column]); - break; - case INT32: - totalSizeInBytes += RamUsageEstimator.sizeOf((int[]) values[column]); - break; - case DOUBLE: - totalSizeInBytes += RamUsageEstimator.sizeOf((double[]) values[column]); - break; - case FLOAT: - totalSizeInBytes += RamUsageEstimator.sizeOf((float[]) values[column]); - break; - case BOOLEAN: - totalSizeInBytes += RamUsageEstimator.sizeOf((boolean[]) values[column]); - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - totalSizeInBytes += RamUsageEstimator.sizeOf((Binary[]) values[column]); - break; - } + totalSizeInBytes += Type.fromTsDataType(tsDataType).arrayRamBytesUsed(values[column]); } } From 3c76323c6bbf3a7e80300b24dd7501d2151159e2 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 16:56:33 +0800 Subject: [PATCH 008/131] add interface for ramBytesUsed --- .../apache/tsfile/read/common/type/AbstractVarcharType.java | 6 ++++++ .../java/org/apache/tsfile/read/common/type/BlobType.java | 6 ++++++ .../org/apache/tsfile/read/common/type/BooleanType.java | 6 ++++++ .../java/org/apache/tsfile/read/common/type/DateType.java | 6 ++++++ .../java/org/apache/tsfile/read/common/type/DoubleType.java | 6 ++++++ .../java/org/apache/tsfile/read/common/type/ObjectType.java | 6 ++++++ 6 files changed, 36 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 549b7a2e6..16dc3cdbe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; import java.io.DataOutputStream; import java.io.IOException; @@ -91,6 +92,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { return binaryArrayEquals(left, right, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((Binary[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 61da6dc6d..125627642 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; import java.io.DataOutputStream; import java.io.IOException; @@ -94,6 +95,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { return binaryArrayEquals(left, right, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((Binary[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 655fe9deb..aac98e2b3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -88,6 +89,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { && Arrays.equals((boolean[]) left, 0, rowSize, (boolean[]) right, 0, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((boolean[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index c1a667d34..f741f0566 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -74,6 +75,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { && Arrays.equals((LocalDate[]) left, 0, rowSize, (LocalDate[]) right, 0, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((LocalDate[]) array); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index c64d2eb42..00e28681f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -117,6 +118,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { && Arrays.equals((double[]) left, 0, rowSize, (double[]) right, 0, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((double[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 3f2d6f0c7..ff41116aa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; import java.io.DataOutputStream; import java.io.IOException; @@ -94,6 +95,11 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { return binaryArrayEquals(left, right, rowSize); } + @Override + public long arrayRamBytesUsed(Object array) { + return RamUsageEstimator.sizeOf((Binary[]) array); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); From b67b008f706d10cea8254e705876c566eb2b983e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 7 Jul 2026 17:14:51 +0800 Subject: [PATCH 009/131] add interface for createColumn --- .../tsfile/read/common/type/AbstractIntType.java | 12 ++++++++++++ .../tsfile/read/common/type/AbstractLongType.java | 8 ++++++++ .../read/common/type/AbstractVarcharType.java | 8 ++++++++ .../apache/tsfile/read/common/type/BlobType.java | 8 ++++++++ .../apache/tsfile/read/common/type/BooleanType.java | 8 ++++++++ .../apache/tsfile/read/common/type/DateType.java | 13 +++++++++++++ .../apache/tsfile/read/common/type/DoubleType.java | 8 ++++++++ .../apache/tsfile/read/common/type/FloatType.java | 8 ++++++++ .../apache/tsfile/read/common/type/ObjectType.java | 8 ++++++++ .../org/apache/tsfile/read/common/type/Type.java | 5 +++++ 10 files changed, 86 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index d72ecfc2d..413b7a1f5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,6 +33,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public abstract class AbstractIntType extends AbstractType { @@ -119,6 +122,15 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((int[]) array); } + @Override + public Column createColumn(int positionCount) { + return new IntColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.INT32); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 9a0dddf9a..cca984616 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,6 +32,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public abstract class AbstractLongType extends AbstractType { @@ -119,6 +121,12 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((long[]) array); } + @Override + public Column createColumn(int positionCount) { + return new LongColumn( + positionCount, Optional.of(new boolean[positionCount]), new long[positionCount]); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 16dc3cdbe..86982f542 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; @@ -32,6 +33,7 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; +import java.util.Optional; public abstract class AbstractVarcharType extends AbstractType { @Override @@ -97,6 +99,12 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((Binary[]) array); } + @Override + public Column createColumn(int positionCount) { + return new BinaryColumn( + positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 125627642..94995de58 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; @@ -32,6 +33,7 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; +import java.util.Optional; public class BlobType extends AbstractType { @@ -100,6 +102,12 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((Binary[]) array); } + @Override + public Column createColumn(int positionCount) { + return new BinaryColumn( + positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index aac98e2b3..14011ccf1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -32,6 +33,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public class BooleanType extends AbstractType { @@ -94,6 +96,12 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((boolean[]) array); } + @Override + public Column createColumn(int positionCount) { + return new BooleanColumn( + positionCount, Optional.of(new boolean[positionCount]), new boolean[positionCount]); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index f741f0566..142bfd882 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -19,6 +19,9 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -28,6 +31,7 @@ import java.nio.ByteBuffer; import java.time.LocalDate; import java.util.Arrays; +import java.util.Optional; public class DateType extends AbstractIntType { @@ -80,6 +84,15 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((LocalDate[]) array); } + @Override + public Column createColumn(int positionCount) { + return new IntColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.DATE); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 00e28681f..1003a1152 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,6 +32,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public class DoubleType extends AbstractType { @@ -123,6 +125,12 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((double[]) array); } + @Override + public Column createColumn(int positionCount) { + return new DoubleColumn( + positionCount, Optional.of(new boolean[positionCount]), new double[positionCount]); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 04ca47064..187e21bdb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,6 +32,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public class FloatType extends AbstractType { @@ -123,6 +125,12 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((float[]) array); } + @Override + public Column createColumn(int positionCount) { + return new FloatColumn( + positionCount, Optional.of(new boolean[positionCount]), new float[positionCount]); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index ff41116aa..1e2c974e7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; @@ -32,6 +33,7 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; +import java.util.Optional; public class ObjectType extends AbstractType { @@ -100,6 +102,12 @@ public long arrayRamBytesUsed(Object array) { return RamUsageEstimator.sizeOf((Binary[]) array); } + @Override + public Column createColumn(int positionCount) { + return new BinaryColumn( + positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index b451fe983..4f6bcd3b5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -146,6 +146,11 @@ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); } + /** Creates a column with {@code positionCount} capacity for this type. */ + default Column createColumn(int positionCount) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. From 46db79b636bd17e44f3eaadb932a24fe5654492e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 15:08:50 +0800 Subject: [PATCH 010/131] Add TypeServices --- .../type/service/BooleanTypeService.java | 49 +++++++++++++++++++ .../type/service/DoubleTypeService.java | 49 +++++++++++++++++++ .../common/type/service/FloatTypeService.java | 49 +++++++++++++++++++ .../common/type/service/IntTypeService.java | 49 +++++++++++++++++++ .../common/type/service/LongTypeService.java | 49 +++++++++++++++++++ .../read/common/type/service/TypeService.java | 49 +++++++++++++++++++ 6 files changed, 294 insertions(+) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/BooleanTypeService.java create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/DoubleTypeService.java create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/FloatTypeService.java create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/IntTypeService.java create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/LongTypeService.java create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/BooleanTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/BooleanTypeService.java new file mode 100644 index 000000000..22e3ce7fa --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/BooleanTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A BooleanTypeService wraps a switch(dataType) clause into a checked service class. + * Implementations should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns boolean only. + */ +public interface BooleanTypeService { + + boolean call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/DoubleTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/DoubleTypeService.java new file mode 100644 index 000000000..7f3af905d --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/DoubleTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A DoubleTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns double only. + */ +public interface DoubleTypeService { + + double call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/FloatTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/FloatTypeService.java new file mode 100644 index 000000000..2d19f6abc --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/FloatTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A FloatTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns float only. + */ +public interface FloatTypeService { + + float call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/IntTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/IntTypeService.java new file mode 100644 index 000000000..e0900b299 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/IntTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * An IntTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns int only. + */ +public interface IntTypeService { + + int call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/LongTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/LongTypeService.java new file mode 100644 index 000000000..c589fb060 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/LongTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A LongTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns long only. + */ +public interface LongTypeService { + + long call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java new file mode 100644 index 000000000..b732ad86b --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A TypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + * @param the return type of the service. + */ +public interface TypeService { + + T call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} From 138d70d9f88c19d21622145410c18fbd0da483c3 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 15:22:18 +0800 Subject: [PATCH 011/131] Add TypeServices --- .../tsfile/read/common/type/TypeFactory.java | 84 +++++++------------ 1 file changed, 30 insertions(+), 54 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java index bebefbdea..d0e87955c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java @@ -29,62 +29,38 @@ private TypeFactory() { } public static Type getType(TSDataType tsDataType) { - switch (tsDataType) { - case DATE: - return DateType.getInstance(); - case INT32: - return IntType.getInstance(); - case INT64: - return LongType.getInstance(); - case TIMESTAMP: - return TimestampType.getInstance(); - case FLOAT: - return FloatType.getInstance(); - case DOUBLE: - return DoubleType.getInstance(); - case BOOLEAN: - return BooleanType.getInstance(); - case TEXT: - return BinaryType.getInstance(); - case STRING: - return StringType.getInstance(); - case BLOB: - return BlobType.getInstance(); - case OBJECT: - return ObjectType.getInstance(); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.typefactory_invalid_tsdata_type", tsDataType)); - } + return switch (tsDataType) { + case DATE -> DateType.getInstance(); + case INT32 -> IntType.getInstance(); + case INT64 -> LongType.getInstance(); + case TIMESTAMP -> TimestampType.getInstance(); + case FLOAT -> FloatType.getInstance(); + case DOUBLE -> DoubleType.getInstance(); + case BOOLEAN -> BooleanType.getInstance(); + case TEXT -> BinaryType.getInstance(); + case STRING -> StringType.getInstance(); + case BLOB -> BlobType.getInstance(); + case OBJECT -> ObjectType.getInstance(); + default -> throw new UnsupportedOperationException( + Messages.format("error.read.typefactory_invalid_tsdata_type", tsDataType)); + }; } public static Type getType(TypeEnum typeEnum) { - switch (typeEnum) { - case INT32: - return IntType.getInstance(); - case INT64: - return LongType.getInstance(); - case FLOAT: - return FloatType.getInstance(); - case DOUBLE: - return DoubleType.getInstance(); - case BOOLEAN: - return BooleanType.getInstance(); - case TEXT: - return BinaryType.getInstance(); - case UNKNOWN: - return UnknownType.getInstance(); - case DATE: - return DateType.getInstance(); - case TIMESTAMP: - return TimestampType.getInstance(); - case BLOB: - return BlobType.getInstance(); - case STRING: - return StringType.getInstance(); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.typefactory_invalid_type_enum", typeEnum)); - } + return switch (typeEnum) { + case INT32 -> IntType.getInstance(); + case INT64 -> LongType.getInstance(); + case FLOAT -> FloatType.getInstance(); + case DOUBLE -> DoubleType.getInstance(); + case BOOLEAN -> BooleanType.getInstance(); + case TEXT -> BinaryType.getInstance(); + case UNKNOWN -> UnknownType.getInstance(); + case DATE -> DateType.getInstance(); + case TIMESTAMP -> TimestampType.getInstance(); + case BLOB -> BlobType.getInstance(); + case STRING -> StringType.getInstance(); + default -> throw new UnsupportedOperationException( + Messages.format("error.read.typefactory_invalid_type_enum", typeEnum)); + }; } } From 3ca5c85350ee97ec1eab1eff4951548165fc5fe2 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 16:01:21 +0800 Subject: [PATCH 012/131] Add getTsPrimitiveType --- .../apache/tsfile/utils/TsPrimitiveType.java | 67 ------------------- .../read/common/type/AbstractIntType.java | 11 +++ .../read/common/type/AbstractLongType.java | 11 +++ .../read/common/type/AbstractVarcharType.java | 12 ++++ .../tsfile/read/common/type/BlobType.java | 11 +++ .../tsfile/read/common/type/BooleanType.java | 11 +++ .../tsfile/read/common/type/DateType.java | 11 +++ .../tsfile/read/common/type/DoubleType.java | 11 +++ .../tsfile/read/common/type/FloatType.java | 11 +++ .../tsfile/read/common/type/ObjectType.java | 11 +++ .../tsfile/read/common/type/RowType.java | 11 +++ .../apache/tsfile/read/common/type/Type.java | 11 +++ .../tsfile/read/common/type/TypeFactory.java | 10 +-- .../tsfile/read/reader/TsFileLastReader.java | 14 ++-- .../read/reader/TsFileLastReaderTest.java | 12 ++-- .../tsfile/utils/TsPrimitiveTypeTest.java | 30 +++++++-- 16 files changed, 169 insertions(+), 86 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java index fa88e68c6..75d1b2c77 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java @@ -28,73 +28,6 @@ public abstract class TsPrimitiveType implements Serializable { - /** - * get tsPrimitiveType by resultDataType. - * - * @param dataType given TsDataType - */ - public static TsPrimitiveType getByType(TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return new TsPrimitiveType.TsBoolean(); - case INT32: - return new TsPrimitiveType.TsInt(); - case DATE: - return new TsPrimitiveType.TsInt(TSDataType.DATE); - case INT64: - case TIMESTAMP: - return new TsPrimitiveType.TsLong(); - case FLOAT: - return new TsPrimitiveType.TsFloat(); - case DOUBLE: - return new TsPrimitiveType.TsDouble(); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new TsPrimitiveType.TsBinary(); - case VECTOR: - return new TsPrimitiveType.TsVector(); - default: - throw new UnSupportedDataTypeException( - Messages.format("error.common.unsupported_data_type", dataType)); - } - } - - /** - * get tsPrimitiveType by resultDataType and initial value. - * - * @param dataType given TsDataType - * @param v initial value - */ - public static TsPrimitiveType getByType(TSDataType dataType, Object v) { - switch (dataType) { - case BOOLEAN: - return new TsPrimitiveType.TsBoolean((boolean) v); - case INT32: - return new TsPrimitiveType.TsInt((int) v); - case DATE: - return new TsPrimitiveType.TsInt((int) v, TSDataType.DATE); - case INT64: - case TIMESTAMP: - return new TsPrimitiveType.TsLong((long) v); - case FLOAT: - return new TsPrimitiveType.TsFloat((float) v); - case DOUBLE: - return new TsPrimitiveType.TsDouble((double) v); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new TsPrimitiveType.TsBinary((Binary) v); - case VECTOR: - return new TsPrimitiveType.TsVector((TsPrimitiveType[]) v); - default: - throw new UnSupportedDataTypeException( - Messages.format("error.common.unsupported_data_type", dataType)); - } - } - public boolean getBoolean() { throw new UnsupportedOperationException( Messages.format("error.common.subclass_op_not_supported", "getBoolean()")); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 413b7a1f5..92808c82e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -26,6 +26,7 @@ import org.apache.tsfile.read.common.block.column.IntColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -37,6 +38,16 @@ public abstract class AbstractIntType extends AbstractType { + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsInt(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsInt((int) value); + } + @Override public int getInt(Column c, int position) { return c.getInt(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index cca984616..892116e7d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -36,6 +37,16 @@ public abstract class AbstractLongType extends AbstractType { + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsLong(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsLong((long) value); + } + @Override public int getInt(Column c, int position) { return (int) c.getLong(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 86982f542..d3dfadacf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -27,6 +27,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -36,6 +37,17 @@ import java.util.Optional; public abstract class AbstractVarcharType extends AbstractType { + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsBinary(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsBinary((Binary) value); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 94995de58..154bf69c0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -27,6 +27,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -41,6 +42,16 @@ public class BlobType extends AbstractType { private BlobType() {} + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsBinary(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsBinary((Binary) value); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 14011ccf1..549861179 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -26,6 +26,7 @@ import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -41,6 +42,16 @@ public class BooleanType extends AbstractType { private BooleanType() {} + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsBoolean(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsBoolean((boolean) value); + } + @Override public boolean getBoolean(Column c, int position) { return c.getBoolean(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 142bfd882..a849ccbb4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -40,6 +41,16 @@ public class DateType extends AbstractIntType { private DateType() {} + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsInt(TSDataType.DATE); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsInt((int) value, TSDataType.DATE); + } + @Override public void addValue(int rowIndex, Object value, Object column) { checkValueType(value, LocalDate.class, "LocalDate"); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 1003a1152..666f1ae54 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -40,6 +41,16 @@ public class DoubleType extends AbstractType { private DoubleType() {} + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsDouble(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsDouble((double) value); + } + @Override public int getInt(Column c, int position) { return (int) c.getDouble(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 187e21bdb..c606495ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -40,6 +41,16 @@ public class FloatType extends AbstractType { private FloatType() {} + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsFloat(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsFloat((float) value); + } + @Override public int getInt(Column c, int position) { return (int) c.getFloat(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 1e2c974e7..8d729fa54 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -27,6 +27,7 @@ import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -41,6 +42,16 @@ public class ObjectType extends AbstractType { private ObjectType() {} + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsBinary(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsBinary((Binary) value); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java index 96bd3b380..455aaa639 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.utils.TsPrimitiveType; import java.util.ArrayList; import java.util.Arrays; @@ -73,6 +74,16 @@ public static RowType createWithTypeSignature(List fields) { return new RowType(fields); } + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsVector(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsVector((TsPrimitiveType[]) value); + } + public static Field field(String name, Type type) { return new Field(Optional.of(name), type); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 4f6bcd3b5..9608dd25c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -70,6 +71,16 @@ default Object getObject(Column c, int position) { return c.getObject(position); } + /** Creates an empty TsPrimitiveType for this type. */ + default TsPrimitiveType getTsPrimitiveType() { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Creates a TsPrimitiveType initialized with {@code value}. */ + default TsPrimitiveType getTsPrimitiveType(Object value) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java index d0e87955c..a768702ca 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java @@ -41,8 +41,9 @@ public static Type getType(TSDataType tsDataType) { case STRING -> StringType.getInstance(); case BLOB -> BlobType.getInstance(); case OBJECT -> ObjectType.getInstance(); - default -> throw new UnsupportedOperationException( - Messages.format("error.read.typefactory_invalid_tsdata_type", tsDataType)); + default -> + throw new UnsupportedOperationException( + Messages.format("error.read.typefactory_invalid_tsdata_type", tsDataType)); }; } @@ -59,8 +60,9 @@ public static Type getType(TypeEnum typeEnum) { case TIMESTAMP -> TimestampType.getInstance(); case BLOB -> BlobType.getInstance(); case STRING -> StringType.getInstance(); - default -> throw new UnsupportedOperationException( - Messages.format("error.read.typefactory_invalid_type_enum", typeEnum)); + default -> + throw new UnsupportedOperationException( + Messages.format("error.read.typefactory_invalid_type_enum", typeEnum)); }; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java index 83a454007..8700a850e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java @@ -32,6 +32,7 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.read.reader.page.ValuePageReader; import org.apache.tsfile.utils.Pair; @@ -231,11 +232,14 @@ private Pair convertToLastPoint( seriesMeta.getMeasurementId(), new TimeValuePair( seriesMeta.getStatistics().getEndTime(), - seriesMeta.getTsDataType() == TSDataType.VECTOR - ? TsPrimitiveType.getByType( - TSDataType.INT64, seriesMeta.getStatistics().getEndTime()) - : TsPrimitiveType.getByType( - seriesMeta.getTsDataType(), seriesMeta.getStatistics().getLastValue()))); + Type.fromTsDataType( + seriesMeta.getTsDataType() == TSDataType.VECTOR + ? TSDataType.INT64 + : seriesMeta.getTsDataType()) + .getTsPrimitiveType( + seriesMeta.getTsDataType() == TSDataType.VECTOR + ? seriesMeta.getStatistics().getEndTime() + : seriesMeta.getStatistics().getLastValue()))); } else { return readLastPoint(seriesMeta, isAligned); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java index bfc55868d..d8f25ea97 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java @@ -24,6 +24,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; @@ -332,13 +333,16 @@ public void testLastEmptyPage() throws Exception { alignedChunkWriter.write( 0, new TsPrimitiveType[] { - TsPrimitiveType.getByType(TSDataType.INT64, 0L), - TsPrimitiveType.getByType( - TSDataType.BLOB, new Binary("0".getBytes(StandardCharsets.UTF_8))) + Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(0L), + Type.fromTsDataType(TSDataType.BLOB) + .getTsPrimitiveType(new Binary("0".getBytes(StandardCharsets.UTF_8))) }); alignedChunkWriter.sealCurrentPage(); alignedChunkWriter.write( - 1, new TsPrimitiveType[] {TsPrimitiveType.getByType(TSDataType.INT64, 1L), null}); + 1, + new TsPrimitiveType[] { + Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(1L), null + }); alignedChunkWriter.writeToFileWriter(ioWriter); ioWriter.endChunkGroup(); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java index d74847b38..cdfd3a79a 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java @@ -21,6 +21,8 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.TsPrimitiveType.TsBinary; import org.apache.tsfile.utils.TsPrimitiveType.TsBoolean; import org.apache.tsfile.utils.TsPrimitiveType.TsDouble; @@ -35,32 +37,48 @@ public class TsPrimitiveTypeTest { @Test public void testNewAndGet() { - TsPrimitiveType intValue = TsPrimitiveType.getByType(TSDataType.INT32, 123); + TsPrimitiveType intValue = Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(123); Assert.assertEquals(new TsInt(123), intValue); Assert.assertEquals(123, intValue.getInt()); - TsPrimitiveType longValue = TsPrimitiveType.getByType(TSDataType.INT64, 456L); + TsPrimitiveType longValue = Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(456L); Assert.assertEquals(new TsLong(456), longValue); Assert.assertEquals(456L, longValue.getLong()); - TsPrimitiveType floatValue = TsPrimitiveType.getByType(TSDataType.FLOAT, 123f); + TsPrimitiveType floatValue = Type.fromTsDataType(TSDataType.FLOAT).getTsPrimitiveType(123f); Assert.assertEquals(new TsFloat(123), floatValue); Assert.assertEquals(123f, floatValue.getFloat(), 0.01); - TsPrimitiveType doubleValue = TsPrimitiveType.getByType(TSDataType.DOUBLE, 456d); + TsPrimitiveType doubleValue = Type.fromTsDataType(TSDataType.DOUBLE).getTsPrimitiveType(456d); Assert.assertEquals(new TsDouble(456), doubleValue); Assert.assertEquals(456d, doubleValue.getDouble(), 0.01); TsPrimitiveType textValue = - TsPrimitiveType.getByType(TSDataType.TEXT, new Binary("123", TSFileConfig.STRING_CHARSET)); + Type.fromTsDataType(TSDataType.TEXT) + .getTsPrimitiveType(new Binary("123", TSFileConfig.STRING_CHARSET)); Assert.assertEquals(new TsBinary(new Binary("123", TSFileConfig.STRING_CHARSET)), textValue); Assert.assertEquals(new Binary("123", TSFileConfig.STRING_CHARSET), textValue.getBinary()); - TsPrimitiveType booleanValue = TsPrimitiveType.getByType(TSDataType.BOOLEAN, true); + TsPrimitiveType booleanValue = Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(true); Assert.assertEquals(new TsBoolean(true), booleanValue); Assert.assertTrue(booleanValue.getBoolean()); } + @Test + public void testTypeSpecificCreation() { + TsPrimitiveType emptyDate = Type.fromTsDataType(TSDataType.DATE).getTsPrimitiveType(); + Assert.assertEquals(TSDataType.DATE, emptyDate.getDataType()); + + TsPrimitiveType date = Type.fromTsDataType(TSDataType.DATE).getTsPrimitiveType(20260713); + Assert.assertEquals(20260713, date.getInt()); + Assert.assertEquals(TSDataType.DATE, date.getDataType()); + + TsPrimitiveType[] values = {date, null}; + TsPrimitiveType vector = RowType.anonymousRow().getTsPrimitiveType(values); + Assert.assertSame(values, vector.getVector()); + Assert.assertNull(RowType.anonymousRow().getTsPrimitiveType().getVector()); + } + @Test public void testCompareWithNullValue() { TimeValuePair timeValuePair1 = From 79b6b21a9d6d4c2d891367d209183b16f2dfbf66 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 17:22:27 +0800 Subject: [PATCH 013/131] Add arePositionsEqual --- .../apache/tsfile/block/column/Column.java | 3 ++ .../common/block/column/BinaryColumn.java | 9 ++++++ .../common/block/column/BooleanColumn.java | 9 ++++++ .../common/block/column/DictionaryColumn.java | 5 ++++ .../common/block/column/DoubleColumn.java | 9 ++++++ .../read/common/block/column/FloatColumn.java | 9 ++++++ .../read/common/block/column/IntColumn.java | 9 ++++++ .../read/common/block/column/LongColumn.java | 9 ++++++ .../read/common/block/column/NullColumn.java | 5 ++++ .../block/column/RunLengthEncodedColumn.java | 5 ++++ .../read/common/block/column/TimeColumn.java | 5 ++++ .../apache/tsfile/read/common/ColumnTest.java | 29 +++++++++++++++++++ 12 files changed, 106 insertions(+) diff --git a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java index c9e30d200..3c53f077b 100644 --- a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java +++ b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java @@ -108,6 +108,9 @@ default TsPrimitiveType getTsPrimitiveType(int position) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns whether the values at the two positions are equal. */ + boolean arePositionsEqual(int pos1, int pos2); + /** * Is it possible the column may have a null value? If false, the column cannot contain a null, * but if true, the column may or may not have a null. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index 88f010381..f28b98d78 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -143,6 +143,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBinary(getBinary(position)); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + boolean pos1IsNull = isNull(pos1); + if (pos1IsNull) { + return isNull(pos2); + } + return !isNull(pos2) && getBinary(pos1).equals(getBinary(pos2)); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java index 318c450f7..2644a85cc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java @@ -125,6 +125,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBoolean(getBoolean(position)); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + boolean pos1IsNull = isNull(pos1); + if (pos1IsNull) { + return isNull(pos2); + } + return !isNull(pos2) && getBoolean(pos1) == getBoolean(pos2); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java index af7f22bef..089b24769 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java @@ -427,6 +427,11 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return dictionary.getTsPrimitiveType(position); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + return dictionary.arePositionsEqual(getId(pos1), getId(pos2)); + } + @Override public void reset() { throw new UnsupportedOperationException(getClass().getSimpleName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java index 4ac8d08ae..68ccb93ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java @@ -126,6 +126,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsDouble(getDouble(position)); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + boolean pos1IsNull = isNull(pos1); + if (pos1IsNull) { + return isNull(pos2); + } + return !isNull(pos2) && getDouble(pos1) == getDouble(pos2); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java index d8a31f110..0857c9994 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java @@ -141,6 +141,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsFloat(getFloat(position)); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + boolean pos1IsNull = isNull(pos1); + if (pos1IsNull) { + return isNull(pos2); + } + return !isNull(pos2) && getFloat(pos1) == getFloat(pos2); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index fbea95799..65c035e9e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -196,6 +196,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsInt(getInt(position), dataType); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + boolean pos1IsNull = isNull(pos1); + if (pos1IsNull) { + return isNull(pos2); + } + return !isNull(pos2) && getInt(pos1) == getInt(pos2); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 5bf01c752..0a6acf600 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -141,6 +141,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsLong(getLong(position)); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + boolean pos1IsNull = isNull(pos1); + if (pos1IsNull) { + return isNull(pos2); + } + return !isNull(pos2) && getLong(pos1) == getLong(pos2); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index 75c32ad61..4891fed3d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -71,6 +71,11 @@ public boolean isNull(int position) { return true; } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + return true; + } + @Override public boolean[] isNull() { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java index 4414e24de..125158f91 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java @@ -165,6 +165,11 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return value.getTsPrimitiveType(0); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + return value.arePositionsEqual(0, 0); + } + @Override public boolean mayHaveNull() { return value.mayHaveNull(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java index 45053b971..662f17cf6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java @@ -90,6 +90,11 @@ public Object getObject(int position) { return getLong(position); } + @Override + public boolean arePositionsEqual(int pos1, int pos2) { + return getLong(pos1) == getLong(pos2); + } + public boolean mayHaveNull() { return false; } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index f3a784324..a5f7f76fd 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -49,6 +49,35 @@ public class ColumnTest { + @Test + public void testArePositionsEqual() { + IntColumnBuilder builder = new IntColumnBuilder(null, 5); + builder.writeInt(1); + builder.writeInt(1); + builder.writeInt(2); + builder.appendNull(); + builder.appendNull(); + Column column = builder.build(); + + Assert.assertTrue(column.arePositionsEqual(0, 1)); + Assert.assertFalse(column.arePositionsEqual(0, 2)); + Assert.assertFalse(column.arePositionsEqual(0, 3)); + Assert.assertTrue(column.arePositionsEqual(3, 4)); + + Column dictionaryColumn = column.getPositions(new int[] {0, 2, 1}, 0, 3); + Assert.assertTrue(dictionaryColumn.arePositionsEqual(0, 2)); + Assert.assertFalse(dictionaryColumn.arePositionsEqual(0, 1)); + + Column runLengthEncodedColumn = new RunLengthEncodedColumn(column.getRegion(0, 1), 3); + Assert.assertTrue(runLengthEncodedColumn.arePositionsEqual(0, 2)); + + Column nanColumn = new DoubleColumn(1, Optional.empty(), new double[] {Double.NaN}); + Column nanRunLengthEncodedColumn = new RunLengthEncodedColumn(nanColumn, 2); + Assert.assertFalse(nanRunLengthEncodedColumn.arePositionsEqual(0, 1)); + + Assert.assertTrue(new NullColumn(2).arePositionsEqual(0, 1)); + } + @Test public void timeColumnSubColumnTest() { TimeColumnBuilder columnBuilder = new TimeColumnBuilder(null, 10); From dfe73df4658cb4efe1bf0b6773d128aa4c3a430c Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 17:31:51 +0800 Subject: [PATCH 014/131] Add arePositionsEqual --- .../apache/tsfile/block/column/Column.java | 11 ++++++++++- .../common/block/column/BinaryColumn.java | 10 +++++----- .../common/block/column/BooleanColumn.java | 10 +++++----- .../common/block/column/DictionaryColumn.java | 4 ++-- .../common/block/column/DoubleColumn.java | 10 +++++----- .../read/common/block/column/FloatColumn.java | 10 +++++----- .../read/common/block/column/IntColumn.java | 10 +++++----- .../read/common/block/column/LongColumn.java | 10 +++++----- .../read/common/block/column/NullColumn.java | 4 ++-- .../block/column/RunLengthEncodedColumn.java | 4 ++-- .../read/common/block/column/TimeColumn.java | 4 ++-- .../apache/tsfile/read/common/ColumnTest.java | 19 ++++++++++++++++++- 12 files changed, 66 insertions(+), 40 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java index 3c53f077b..93e346dc5 100644 --- a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java +++ b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java @@ -109,7 +109,16 @@ default TsPrimitiveType getTsPrimitiveType(int position) { } /** Returns whether the values at the two positions are equal. */ - boolean arePositionsEqual(int pos1, int pos2); + default boolean arePositionsEqual(int pos1, int pos2) { + return arePositionsEqual(pos1, this, pos2); + } + + /** + * Returns whether this value and the value in another column are equal. + * + *

For non-null values, this column and {@code that} must provide compatible value accessors. + */ + boolean arePositionsEqual(int thisPos, Column that, int thatPos); /** * Is it possible the column may have a null value? If false, the column cannot contain a null, diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index f28b98d78..5208d29b6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -144,12 +144,12 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - boolean pos1IsNull = isNull(pos1); - if (pos1IsNull) { - return isNull(pos2); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); } - return !isNull(pos2) && getBinary(pos1).equals(getBinary(pos2)); + return !that.isNull(thatPos) && getBinary(thisPos).equals(that.getBinary(thatPos)); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java index 2644a85cc..e40ff4e3b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java @@ -126,12 +126,12 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - boolean pos1IsNull = isNull(pos1); - if (pos1IsNull) { - return isNull(pos2); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); } - return !isNull(pos2) && getBoolean(pos1) == getBoolean(pos2); + return !that.isNull(thatPos) && getBoolean(thisPos) == that.getBoolean(thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java index 089b24769..e26f42c09 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java @@ -428,8 +428,8 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - return dictionary.arePositionsEqual(getId(pos1), getId(pos2)); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return dictionary.arePositionsEqual(getId(thisPos), that, thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java index 68ccb93ae..c12c200a7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java @@ -127,12 +127,12 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - boolean pos1IsNull = isNull(pos1); - if (pos1IsNull) { - return isNull(pos2); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); } - return !isNull(pos2) && getDouble(pos1) == getDouble(pos2); + return !that.isNull(thatPos) && getDouble(thisPos) == that.getDouble(thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java index 0857c9994..3755aef1e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java @@ -142,12 +142,12 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - boolean pos1IsNull = isNull(pos1); - if (pos1IsNull) { - return isNull(pos2); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); } - return !isNull(pos2) && getFloat(pos1) == getFloat(pos2); + return !that.isNull(thatPos) && getFloat(thisPos) == that.getFloat(thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index 65c035e9e..b54d1575b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -197,12 +197,12 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - boolean pos1IsNull = isNull(pos1); - if (pos1IsNull) { - return isNull(pos2); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); } - return !isNull(pos2) && getInt(pos1) == getInt(pos2); + return !that.isNull(thatPos) && getInt(thisPos) == that.getInt(thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 0a6acf600..49d8183db 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -142,12 +142,12 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - boolean pos1IsNull = isNull(pos1); - if (pos1IsNull) { - return isNull(pos2); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); } - return !isNull(pos2) && getLong(pos1) == getLong(pos2); + return !that.isNull(thatPos) && getLong(thisPos) == that.getLong(thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index 4891fed3d..30092aeb3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -72,8 +72,8 @@ public boolean isNull(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - return true; + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return that.isNull(thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java index 125158f91..548cfab81 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java @@ -166,8 +166,8 @@ public TsPrimitiveType getTsPrimitiveType(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - return value.arePositionsEqual(0, 0); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return value.arePositionsEqual(0, that, thatPos); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java index 662f17cf6..2c9ddc964 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java @@ -91,8 +91,8 @@ public Object getObject(int position) { } @Override - public boolean arePositionsEqual(int pos1, int pos2) { - return getLong(pos1) == getLong(pos2); + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return !that.isNull(thatPos) && getLong(thisPos) == that.getLong(thatPos); } public boolean mayHaveNull() { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index a5f7f76fd..6d6fe91a1 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -64,18 +64,35 @@ public void testArePositionsEqual() { Assert.assertFalse(column.arePositionsEqual(0, 3)); Assert.assertTrue(column.arePositionsEqual(3, 4)); + IntColumnBuilder thatBuilder = new IntColumnBuilder(null, 3); + thatBuilder.writeInt(1); + thatBuilder.writeInt(2); + thatBuilder.appendNull(); + Column that = thatBuilder.build(); + Assert.assertTrue(column.arePositionsEqual(0, that, 0)); + Assert.assertFalse(column.arePositionsEqual(0, that, 1)); + Assert.assertFalse(column.arePositionsEqual(0, that, 2)); + Assert.assertTrue(column.arePositionsEqual(3, that, 2)); + Column dictionaryColumn = column.getPositions(new int[] {0, 2, 1}, 0, 3); Assert.assertTrue(dictionaryColumn.arePositionsEqual(0, 2)); Assert.assertFalse(dictionaryColumn.arePositionsEqual(0, 1)); + Assert.assertTrue(dictionaryColumn.arePositionsEqual(0, that, 0)); + Assert.assertTrue(that.arePositionsEqual(0, dictionaryColumn, 2)); Column runLengthEncodedColumn = new RunLengthEncodedColumn(column.getRegion(0, 1), 3); Assert.assertTrue(runLengthEncodedColumn.arePositionsEqual(0, 2)); + Assert.assertTrue(runLengthEncodedColumn.arePositionsEqual(0, that, 0)); + Assert.assertTrue(that.arePositionsEqual(0, runLengthEncodedColumn, 2)); Column nanColumn = new DoubleColumn(1, Optional.empty(), new double[] {Double.NaN}); Column nanRunLengthEncodedColumn = new RunLengthEncodedColumn(nanColumn, 2); Assert.assertFalse(nanRunLengthEncodedColumn.arePositionsEqual(0, 1)); - Assert.assertTrue(new NullColumn(2).arePositionsEqual(0, 1)); + Column nullColumn = new NullColumn(2); + Assert.assertTrue(nullColumn.arePositionsEqual(0, 1)); + Assert.assertTrue(nullColumn.arePositionsEqual(0, that, 2)); + Assert.assertFalse(nullColumn.arePositionsEqual(0, that, 0)); } @Test From 82cba6d372a97d8928ed902935d5b0b5e970c572 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 17:53:15 +0800 Subject: [PATCH 015/131] Add getDefaultEncoding --- .../tsfile/common/conf/TSFileConfig.java | 22 ++--------------- .../read/common/type/AbstractIntType.java | 7 ++++++ .../read/common/type/AbstractLongType.java | 7 ++++++ .../read/common/type/AbstractVarcharType.java | 6 +++++ .../tsfile/read/common/type/BlobType.java | 6 +++++ .../tsfile/read/common/type/BooleanType.java | 7 ++++++ .../tsfile/read/common/type/DoubleType.java | 7 ++++++ .../tsfile/read/common/type/FloatType.java | 7 ++++++ .../tsfile/read/common/type/ObjectType.java | 6 +++++ .../apache/tsfile/read/common/type/Type.java | 7 ++++++ .../common/conf/TSFileDescriptorTest.java | 24 +++++++++++++++++++ 11 files changed, 86 insertions(+), 20 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index 764eda5bd..2084c7000 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@ -24,6 +24,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.FSUtils; import java.io.Serializable; @@ -370,26 +371,7 @@ public String getValueEncoder() { } public TSEncoding getValueEncoder(TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return booleanEncoding; - case INT32: - case DATE: - return int32Encoding; - case INT64: - case TIMESTAMP: - return int64Encoding; - case FLOAT: - return floatEncoding; - case DOUBLE: - return doubleEncoding; - case STRING: - case BLOB: - case TEXT: - case OBJECT: - default: - return textEncoding; - } + return Type.fromTsDataType(dataType).getDefaultEncoding(this); } public CompressionType getCompressor(TSDataType dataType) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 92808c82e..1a4f63af6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -21,7 +21,9 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; @@ -38,6 +40,11 @@ public abstract class AbstractIntType extends AbstractType { + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getInt32Encoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 892116e7d..a9ac7192c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; @@ -37,6 +39,11 @@ public abstract class AbstractLongType extends AbstractType { + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getInt64Encoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index d3dfadacf..d19fd7542 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; @@ -38,6 +39,11 @@ public abstract class AbstractVarcharType extends AbstractType { + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getTextEncoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 154bf69c0..aaa19ab4a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; @@ -42,6 +43,11 @@ public class BlobType extends AbstractType { private BlobType() {} + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getTextEncoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 549861179..192f64494 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -42,6 +44,11 @@ public class BooleanType extends AbstractType { private BooleanType() {} + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getBooleanEncoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 666f1ae54..6f9def3cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; @@ -41,6 +43,11 @@ public class DoubleType extends AbstractType { private DoubleType() {} + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getDoubleEncoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index c606495ae..03f479960 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -21,6 +21,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; import org.apache.tsfile.utils.RamUsageEstimator; @@ -41,6 +43,11 @@ public class FloatType extends AbstractType { private FloatType() {} + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getFloatEncoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 8d729fa54..127c0a281 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; @@ -42,6 +43,11 @@ public class ObjectType extends AbstractType { private ObjectType() {} + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getTextEncoding()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 9608dd25c..888fec5b3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -21,7 +21,9 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -162,6 +164,11 @@ default Column createColumn(int positionCount) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the default encoding configured for this type. */ + default TSEncoding getDefaultEncoding(TSFileConfig config) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java index 1dc6214d7..a90f8d8a1 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java @@ -19,6 +19,8 @@ package org.apache.tsfile.common.conf; import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.Assert; import org.junit.Before; @@ -82,4 +84,26 @@ public void testGetInstanceOverWriteFromFile() { Assert.assertEquals(tsFileConfig1.getMaxStringLength(), tsFileConfig2.getMaxStringLength()); } + + @Test + public void testGetValueEncoder() { + tsFileConfig1.setBooleanEncoding(TSEncoding.PLAIN.name()); + tsFileConfig1.setInt32Encoding(TSEncoding.RLE.name()); + tsFileConfig1.setInt64Encoding(TSEncoding.GORILLA.name()); + tsFileConfig1.setFloatEncoding(TSEncoding.TS_2DIFF.name()); + tsFileConfig1.setDoubleEncoding(TSEncoding.RLE.name()); + tsFileConfig1.setTextEncoding(TSEncoding.DICTIONARY.name()); + + Assert.assertEquals(TSEncoding.PLAIN, tsFileConfig1.getValueEncoder(TSDataType.BOOLEAN)); + Assert.assertEquals(TSEncoding.RLE, tsFileConfig1.getValueEncoder(TSDataType.INT32)); + Assert.assertEquals(TSEncoding.RLE, tsFileConfig1.getValueEncoder(TSDataType.DATE)); + Assert.assertEquals(TSEncoding.GORILLA, tsFileConfig1.getValueEncoder(TSDataType.INT64)); + Assert.assertEquals(TSEncoding.GORILLA, tsFileConfig1.getValueEncoder(TSDataType.TIMESTAMP)); + Assert.assertEquals(TSEncoding.TS_2DIFF, tsFileConfig1.getValueEncoder(TSDataType.FLOAT)); + Assert.assertEquals(TSEncoding.RLE, tsFileConfig1.getValueEncoder(TSDataType.DOUBLE)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.TEXT)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.STRING)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.BLOB)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.OBJECT)); + } } From 0e84e7055efc82d9ef407b82238a49ab3adf8438 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 18:15:32 +0800 Subject: [PATCH 016/131] Add toBytes --- .../read/common/type/AbstractIntType.java | 6 ++ .../read/common/type/AbstractLongType.java | 6 ++ .../tsfile/read/common/type/AbstractType.java | 7 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 6 ++ .../tsfile/read/common/type/FloatType.java | 6 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 5 ++ .../tsfile/read/common/type/TypeTest.java | 74 +++++++++++++++++++ 11 files changed, 130 insertions(+) create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 1a4f63af6..747483322 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -26,6 +26,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -40,6 +41,11 @@ public abstract class AbstractIntType extends AbstractType { + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.intToBytes(value.getInt(), valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index a9ac7192c..a8d2280cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -39,6 +40,11 @@ public abstract class AbstractLongType extends AbstractType { + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.longToBytes(value.getLong(), valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 3a90a1230..8839a3722 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -23,6 +23,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; import java.io.DataOutputStream; import java.io.IOException; @@ -32,6 +33,12 @@ public abstract class AbstractType implements Type { + protected void binaryToBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + Binary binary = value.getBinary(); + BytesUtils.intToBytes(binary.getLength(), valueBytes, offset); + System.arraycopy(binary.getValues(), 0, valueBytes, offset + Integer.BYTES, binary.getLength()); + } + protected void checkValueType(Object value, Class expectedClass, String expectedType) { if (value != null && !expectedClass.isInstance(value)) { throw new IllegalArgumentException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index d19fd7542..96939ed2c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -39,6 +39,11 @@ public abstract class AbstractVarcharType extends AbstractType { + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + binaryToBytes(value, valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index aaa19ab4a..e37c099ce 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -43,6 +43,11 @@ public class BlobType extends AbstractType { private BlobType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + binaryToBytes(value, valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 192f64494..ac31369e7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -44,6 +44,11 @@ public class BooleanType extends AbstractType { private BooleanType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.boolToBytes(value.getBoolean(), valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 6f9def3cd..2e7f89f85 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -43,6 +44,11 @@ public class DoubleType extends AbstractType { private DoubleType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.doubleToBytes(value.getDouble(), valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 03f479960..b9fce3b11 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -43,6 +44,11 @@ public class FloatType extends AbstractType { private FloatType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.floatToBytes(value.getFloat(), valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 127c0a281..06d7409ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -43,6 +43,11 @@ public class ObjectType extends AbstractType { private ObjectType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + binaryToBytes(value, valueBytes, offset); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 888fec5b3..2a6cebd14 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -83,6 +83,11 @@ default TsPrimitiveType getTsPrimitiveType(Object value) { throw new UnsupportedOperationException(getClass().getName()); } + /** Serializes {@code value} into {@code valueBytes} starting at {@code offset}. */ + default void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java new file mode 100644 index 000000000..79b4e618f --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.common.type; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.TsPrimitiveType; + +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +public class TypeTest { + + private static final int OFFSET = 2; + + @Test + public void testToBytes() { + byte[] valueBytes = new byte[16]; + + Type.fromTsDataType(TSDataType.BOOLEAN) + .toBytes(new TsPrimitiveType.TsBoolean(true), valueBytes, OFFSET); + Assert.assertTrue(BytesUtils.bytesToBool(valueBytes, OFFSET)); + + Type.fromTsDataType(TSDataType.DATE) + .toBytes(new TsPrimitiveType.TsInt(20260713, TSDataType.DATE), valueBytes, OFFSET); + Assert.assertEquals(20260713, BytesUtils.bytesToInt(valueBytes, OFFSET)); + + Type.fromTsDataType(TSDataType.TIMESTAMP) + .toBytes(new TsPrimitiveType.TsLong(123456789L), valueBytes, OFFSET); + Assert.assertEquals( + 123456789L, BytesUtils.bytesToLongFromOffset(valueBytes, Long.BYTES, OFFSET)); + + Type.fromTsDataType(TSDataType.FLOAT) + .toBytes(new TsPrimitiveType.TsFloat(1.25F), valueBytes, OFFSET); + Assert.assertEquals(1.25F, BytesUtils.bytesToFloat(valueBytes, OFFSET), 0); + + Type.fromTsDataType(TSDataType.DOUBLE) + .toBytes(new TsPrimitiveType.TsDouble(2.5D), valueBytes, OFFSET); + Assert.assertEquals(2.5D, BytesUtils.bytesToDouble(valueBytes, OFFSET), 0); + + Binary binary = new Binary("test", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType) + .toBytes(new TsPrimitiveType.TsBinary(binary), valueBytes, OFFSET); + Assert.assertEquals(binary.getLength(), BytesUtils.bytesToInt(valueBytes, OFFSET)); + Assert.assertArrayEquals( + binary.getValues(), + Arrays.copyOfRange( + valueBytes, OFFSET + Integer.BYTES, OFFSET + Integer.BYTES + binary.getLength())); + } + } +} From dc087c815bd34829f678627707cde8edebe27536 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 18:32:35 +0800 Subject: [PATCH 017/131] Add getCompressor --- .../tsfile/common/conf/TSFileConfig.java | 62 +++++++++---------- .../read/common/type/AbstractIntType.java | 6 ++ .../read/common/type/AbstractLongType.java | 6 ++ .../read/common/type/AbstractVarcharType.java | 6 ++ .../tsfile/read/common/type/BlobType.java | 6 ++ .../tsfile/read/common/type/BooleanType.java | 6 ++ .../tsfile/read/common/type/DoubleType.java | 6 ++ .../tsfile/read/common/type/FloatType.java | 6 ++ .../tsfile/read/common/type/ObjectType.java | 6 ++ .../apache/tsfile/read/common/type/Type.java | 6 ++ .../common/conf/TSFileDescriptorTest.java | 32 ++++++++++ 11 files changed, 115 insertions(+), 33 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index 2084c7000..6c79d6567 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@ -375,39 +375,7 @@ public TSEncoding getValueEncoder(TSDataType dataType) { } public CompressionType getCompressor(TSDataType dataType) { - CompressionType compressionType; - switch (dataType) { - case BOOLEAN: - compressionType = booleanCompression; - break; - case INT32: - case DATE: - compressionType = int32Compression; - break; - case INT64: - case TIMESTAMP: - compressionType = int64Compression; - break; - case FLOAT: - compressionType = floatCompression; - break; - case DOUBLE: - compressionType = doubleCompression; - break; - case STRING: - case BLOB: - case TEXT: - case OBJECT: - compressionType = textCompression; - break; - default: - compressionType = null; - } - - if (compressionType == null) { - compressionType = compressor; - } - return compressionType; + return Type.fromTsDataType(dataType).getDefaultCompressor(this); } public void setValueEncoder(String valueEncoder) { @@ -514,6 +482,34 @@ public CompressionType getCompressor() { return compressor; } + public CompressionType getBooleanCompressor() { + return getCompressorOrDefault(booleanCompression); + } + + public CompressionType getInt32Compressor() { + return getCompressorOrDefault(int32Compression); + } + + public CompressionType getInt64Compressor() { + return getCompressorOrDefault(int64Compression); + } + + public CompressionType getFloatCompressor() { + return getCompressorOrDefault(floatCompression); + } + + public CompressionType getDoubleCompressor() { + return getCompressorOrDefault(doubleCompression); + } + + public CompressionType getTextCompressor() { + return getCompressorOrDefault(textCompression); + } + + private CompressionType getCompressorOrDefault(CompressionType compressionType) { + return compressionType == null ? compressor : compressionType; + } + public void setCompressor(String compressor) { this.compressor = CompressionType.valueOf(compressor); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 747483322..20383d451 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; @@ -51,6 +52,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getInt32Compressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index a8d2280cd..8b45965dd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; @@ -50,6 +51,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getInt64Compressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 96939ed2c..c3292fded 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumn; @@ -49,6 +50,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getTextCompressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index e37c099ce..2c66c5638 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumn; @@ -53,6 +54,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getTextCompressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index ac31369e7..839e6dc2b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; @@ -54,6 +55,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getBooleanCompressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 2e7f89f85..811fdadc9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; @@ -54,6 +55,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getDoubleCompressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index b9fce3b11..c1d1cf89d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; @@ -54,6 +55,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getFloatCompressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 06d7409ae..ed025b7cc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.BinaryColumn; @@ -53,6 +54,11 @@ public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); } + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getTextCompressor(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 2a6cebd14..c01ef0716 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -174,6 +175,11 @@ default TSEncoding getDefaultEncoding(TSFileConfig config) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the default compressor configured for this type. */ + default CompressionType getDefaultCompressor(TSFileConfig config) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java index a90f8d8a1..668fee698 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java @@ -20,6 +20,7 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.Assert; @@ -106,4 +107,35 @@ public void testGetValueEncoder() { Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.BLOB)); Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.OBJECT)); } + + @Test + public void testGetCompressor() { + tsFileConfig1.setCompressor(CompressionType.ZSTD.name()); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.BOOLEAN)); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.INT32)); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.TEXT)); + + tsFileConfig1.setBooleanCompression(CompressionType.UNCOMPRESSED.name()); + tsFileConfig1.setInt32Compression(CompressionType.SNAPPY.name()); + tsFileConfig1.setInt64Compression(CompressionType.GZIP.name()); + tsFileConfig1.setFloatCompression(CompressionType.LZ4.name()); + tsFileConfig1.setDoubleCompression(CompressionType.ZSTD.name()); + tsFileConfig1.setTextCompression(CompressionType.LZMA2.name()); + + Assert.assertEquals( + CompressionType.UNCOMPRESSED, tsFileConfig1.getCompressor(TSDataType.BOOLEAN)); + Assert.assertEquals(CompressionType.SNAPPY, tsFileConfig1.getCompressor(TSDataType.INT32)); + Assert.assertEquals(CompressionType.SNAPPY, tsFileConfig1.getCompressor(TSDataType.DATE)); + Assert.assertEquals(CompressionType.GZIP, tsFileConfig1.getCompressor(TSDataType.INT64)); + Assert.assertEquals(CompressionType.GZIP, tsFileConfig1.getCompressor(TSDataType.TIMESTAMP)); + Assert.assertEquals(CompressionType.LZ4, tsFileConfig1.getCompressor(TSDataType.FLOAT)); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.DOUBLE)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.TEXT)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.STRING)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.BLOB)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.OBJECT)); + + tsFileConfig1.setInt32Compression(null); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.INT32)); + } } From b95cfad41ad71690509259fd89ef0c5feb777b00 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 13 Jul 2026 19:00:24 +0800 Subject: [PATCH 018/131] Add calcTypeSize --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 6 +++ .../tsfile/read/common/type/AbstractType.java | 4 ++ .../read/common/type/AbstractVarcharType.java | 6 +++ .../tsfile/read/common/type/BlobType.java | 6 +++ .../tsfile/read/common/type/BooleanType.java | 6 +++ .../tsfile/read/common/type/DoubleType.java | 6 +++ .../tsfile/read/common/type/FloatType.java | 6 +++ .../tsfile/read/common/type/ObjectType.java | 6 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 41 +++++++++++++++++++ 11 files changed, 97 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 20383d451..af164d930 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -47,6 +47,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { BytesUtils.intToBytes(value.getInt(), valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return Integer.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 8b45965dd..cf5346cc9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.LongColumn; @@ -46,6 +47,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { BytesUtils.longToBytes(value.getLong(), valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return Long.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 8839a3722..8bae60b3c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -39,6 +39,10 @@ protected void binaryToBytes(TsPrimitiveType value, byte[] valueBytes, int offse System.arraycopy(binary.getValues(), 0, valueBytes, offset + Integer.BYTES, binary.getLength()); } + protected int binaryTypeSize(TsPrimitiveType value) { + return Integer.BYTES + value.getBinary().getLength(); + } + protected void checkValueType(Object value, Class expectedClass, String expectedType) { if (value != null && !expectedClass.isInstance(value)) { throw new IllegalArgumentException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index c3292fded..18d6a9822 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; @@ -45,6 +46,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { binaryToBytes(value, valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return binaryTypeSize(value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 2c66c5638..e67d6c86e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; @@ -49,6 +50,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { binaryToBytes(value, valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return binaryTypeSize(value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 839e6dc2b..794ae0f12 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.BooleanColumn; @@ -50,6 +51,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { BytesUtils.boolToBytes(value.getBoolean(), valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return Byte.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 811fdadc9..a71b5018e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.DoubleColumn; @@ -50,6 +51,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { BytesUtils.doubleToBytes(value.getDouble(), valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return Double.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index c1d1cf89d..9f6bdf4bd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.column.FloatColumn; @@ -50,6 +51,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { BytesUtils.floatToBytes(value.getFloat(), valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return Float.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index ed025b7cc..215b03386 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; @@ -49,6 +50,11 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { binaryToBytes(value, valueBytes, offset); } + @Override + public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + return binaryTypeSize(value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index c01ef0716..ac3625938 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -89,6 +89,11 @@ default void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the number of bytes required to serialize {@code value}. */ + default int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 79b4e618f..41bcad617 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -71,4 +71,45 @@ public void testToBytes() { valueBytes, OFFSET + Integer.BYTES, OFFSET + Integer.BYTES + binary.getLength())); } } + + @Test + public void testCalcTypeSize() { + Assert.assertEquals( + Byte.BYTES, + Type.fromTsDataType(TSDataType.BOOLEAN) + .calcTypeSize(TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true))); + Assert.assertEquals( + Integer.BYTES, + Type.fromTsDataType(TSDataType.INT32) + .calcTypeSize(TSDataType.INT32, new TsPrimitiveType.TsInt(1))); + Assert.assertEquals( + Integer.BYTES, + Type.fromTsDataType(TSDataType.DATE) + .calcTypeSize(TSDataType.DATE, new TsPrimitiveType.TsInt(20260713, TSDataType.DATE))); + Assert.assertEquals( + Long.BYTES, + Type.fromTsDataType(TSDataType.INT64) + .calcTypeSize(TSDataType.INT64, new TsPrimitiveType.TsLong(1L))); + Assert.assertEquals( + Long.BYTES, + Type.fromTsDataType(TSDataType.TIMESTAMP) + .calcTypeSize(TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(1L))); + Assert.assertEquals( + Float.BYTES, + Type.fromTsDataType(TSDataType.FLOAT) + .calcTypeSize(TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.0F))); + Assert.assertEquals( + Double.BYTES, + Type.fromTsDataType(TSDataType.DOUBLE) + .calcTypeSize(TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(1.0D))); + + Binary binary = new Binary("test", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Assert.assertEquals( + Integer.BYTES + binary.getLength(), + Type.fromTsDataType(dataType) + .calcTypeSize(dataType, new TsPrimitiveType.TsBinary(binary))); + } + } } From 657e9e1f868321416ce29eaac0ee692f019c2034 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 09:46:41 +0800 Subject: [PATCH 019/131] Add getDecoder --- .../tsfile/encoding/decoder/Decoder.java | 144 +----------------- .../read/common/type/AbstractIntType.java | 25 +++ .../read/common/type/AbstractLongType.java | 25 +++ .../tsfile/read/common/type/AbstractType.java | 21 +++ .../tsfile/read/common/type/BooleanType.java | 11 ++ .../tsfile/read/common/type/DoubleType.java | 23 +++ .../tsfile/read/common/type/FloatType.java | 21 +++ .../apache/tsfile/read/common/type/Type.java | 6 + .../tsfile/read/common/type/TypeEnum.java | 3 +- .../tsfile/read/common/type/TypeFactory.java | 3 + .../tsfile/read/common/type/VectorType.java | 62 ++++++++ .../tsfile/encoding/decoder/DecoderTest.java | 107 +++++++++++++ 12 files changed, 308 insertions(+), 143 deletions(-) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/DecoderTest.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java index 340316bb1..bfa1218ab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java @@ -23,6 +23,7 @@ import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import java.io.IOException; @@ -31,8 +32,6 @@ public abstract class Decoder { - private static final String ERROR_MSG_KEY = "error.encoding.decoder_not_found"; - private TSEncoding type; public Decoder(TSEncoding type) { @@ -48,146 +47,7 @@ public TSEncoding getType() { } public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) { - switch (encoding) { - case PLAIN: - return new PlainDecoder(); - case RLE: - switch (dataType) { - case BOOLEAN: - case INT32: - case DATE: - return new IntRleDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new LongRleDecoder(); - case FLOAT: - case DOUBLE: - return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case TS_2DIFF: - switch (dataType) { - case INT32: - case DATE: - return new DeltaBinaryDecoder.IntDeltaDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new DeltaBinaryDecoder.LongDeltaDecoder(); - case FLOAT: - case DOUBLE: - return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case GORILLA_V1: - switch (dataType) { - case FLOAT: - return new SinglePrecisionDecoderV1(); - case DOUBLE: - return new DoublePrecisionDecoderV1(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case REGULAR: - switch (dataType) { - case INT32: - case DATE: - return new RegularDataDecoder.IntRegularDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new RegularDataDecoder.LongRegularDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case GORILLA: - switch (dataType) { - case FLOAT: - return new SinglePrecisionDecoderV2(); - case DOUBLE: - return new DoublePrecisionDecoderV2(); - case INT32: - case DATE: - return new IntGorillaDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new LongGorillaDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case DICTIONARY: - return new DictionaryDecoder(); - case ZIGZAG: - switch (dataType) { - case INT32: - case DATE: - return new IntZigzagDecoder(); - case INT64: - case TIMESTAMP: - return new LongZigzagDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case CHIMP: - switch (dataType) { - case FLOAT: - return new SinglePrecisionChimpDecoder(); - case DOUBLE: - return new DoublePrecisionChimpDecoder(); - case INT32: - case DATE: - return new IntChimpDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new LongChimpDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case SPRINTZ: - switch (dataType) { - case INT32: - case DATE: - return new IntSprintzDecoder(); - case INT64: - case TIMESTAMP: - return new LongSprintzDecoder(); - case FLOAT: - return new FloatSprintzDecoder(); - case DOUBLE: - return new DoubleSprintzDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case RLBE: - switch (dataType) { - case INT32: - case DATE: - return new IntRLBEDecoder(); - case INT64: - case TIMESTAMP: - return new LongRLBEDecoder(); - case FLOAT: - return new FloatRLBEDecoder(); - case DOUBLE: - return new DoubleRLBEDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case CAMEL: - switch (dataType) { - case DOUBLE: - return new CamelDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } + return Type.fromTsDataType(dataType).getDecoder(encoding); } public int readInt(ByteBuffer buffer) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index af164d930..e29579709 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -22,6 +22,15 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; +import org.apache.tsfile.encoding.decoder.IntChimpDecoder; +import org.apache.tsfile.encoding.decoder.IntGorillaDecoder; +import org.apache.tsfile.encoding.decoder.IntRLBEDecoder; +import org.apache.tsfile.encoding.decoder.IntRleDecoder; +import org.apache.tsfile.encoding.decoder.IntSprintzDecoder; +import org.apache.tsfile.encoding.decoder.IntZigzagDecoder; +import org.apache.tsfile.encoding.decoder.RegularDataDecoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -62,6 +71,22 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getInt32Compressor(); } + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new IntRleDecoder(); + case TS_2DIFF -> new DeltaBinaryDecoder.IntDeltaDecoder(); + case REGULAR -> new RegularDataDecoder.IntRegularDecoder(); + case GORILLA -> new IntGorillaDecoder(); + case ZIGZAG -> new IntZigzagDecoder(); + case CHIMP -> new IntChimpDecoder(); + case SPRINTZ -> new IntSprintzDecoder(); + case RLBE -> new IntRLBEDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index cf5346cc9..32970cc93 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -22,6 +22,15 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; +import org.apache.tsfile.encoding.decoder.LongChimpDecoder; +import org.apache.tsfile.encoding.decoder.LongGorillaDecoder; +import org.apache.tsfile.encoding.decoder.LongRLBEDecoder; +import org.apache.tsfile.encoding.decoder.LongRleDecoder; +import org.apache.tsfile.encoding.decoder.LongSprintzDecoder; +import org.apache.tsfile.encoding.decoder.LongZigzagDecoder; +import org.apache.tsfile.encoding.decoder.RegularDataDecoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -62,6 +71,22 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getInt64Compressor(); } + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new LongRleDecoder(); + case TS_2DIFF -> new DeltaBinaryDecoder.LongDeltaDecoder(); + case REGULAR -> new RegularDataDecoder.LongRegularDecoder(); + case GORILLA -> new LongGorillaDecoder(); + case ZIGZAG -> new LongZigzagDecoder(); + case CHIMP -> new LongChimpDecoder(); + case SPRINTZ -> new LongSprintzDecoder(); + case RLBE -> new LongRLBEDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 8bae60b3c..ae86cd235 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -19,6 +19,11 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DictionaryDecoder; +import org.apache.tsfile.encoding.decoder.PlainDecoder; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; @@ -33,6 +38,22 @@ public abstract class AbstractType implements Type { + private static final String DECODER_NOT_FOUND_ERROR_KEY = "error.encoding.decoder_not_found"; + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN -> new PlainDecoder(); + case DICTIONARY -> new DictionaryDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + protected TsFileDecodingException decoderNotFound(TSEncoding encoding) { + return new TsFileDecodingException( + Messages.format(DECODER_NOT_FOUND_ERROR_KEY, encoding, getTypeEnum())); + } + protected void binaryToBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { Binary binary = value.getBinary(); BytesUtils.intToBytes(binary.getLength(), valueBytes, offset); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 794ae0f12..41734a2a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -22,6 +22,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.IntRleDecoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -66,6 +68,15 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getBooleanCompressor(); } + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new IntRleDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index a71b5018e..e20f21778 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -22,6 +22,14 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.CamelDecoder; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DoublePrecisionChimpDecoder; +import org.apache.tsfile.encoding.decoder.DoublePrecisionDecoderV1; +import org.apache.tsfile.encoding.decoder.DoublePrecisionDecoderV2; +import org.apache.tsfile.encoding.decoder.DoubleRLBEDecoder; +import org.apache.tsfile.encoding.decoder.DoubleSprintzDecoder; +import org.apache.tsfile.encoding.decoder.FloatDecoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -66,6 +74,21 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getDoubleCompressor(); } + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE, TS_2DIFF -> new FloatDecoder(encoding, TSDataType.DOUBLE); + case GORILLA_V1 -> new DoublePrecisionDecoderV1(); + case GORILLA -> new DoublePrecisionDecoderV2(); + case CHIMP -> new DoublePrecisionChimpDecoder(); + case SPRINTZ -> new DoubleSprintzDecoder(); + case RLBE -> new DoubleRLBEDecoder(); + case CAMEL -> new CamelDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 9f6bdf4bd..4fc36af44 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -22,6 +22,13 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.FloatDecoder; +import org.apache.tsfile.encoding.decoder.FloatRLBEDecoder; +import org.apache.tsfile.encoding.decoder.FloatSprintzDecoder; +import org.apache.tsfile.encoding.decoder.SinglePrecisionChimpDecoder; +import org.apache.tsfile.encoding.decoder.SinglePrecisionDecoderV1; +import org.apache.tsfile.encoding.decoder.SinglePrecisionDecoderV2; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -66,6 +73,20 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getFloatCompressor(); } + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE, TS_2DIFF -> new FloatDecoder(encoding, TSDataType.FLOAT); + case GORILLA_V1 -> new SinglePrecisionDecoderV1(); + case GORILLA -> new SinglePrecisionDecoderV2(); + case CHIMP -> new SinglePrecisionChimpDecoder(); + case SPRINTZ -> new FloatSprintzDecoder(); + case RLBE -> new FloatRLBEDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index ac3625938..9c68fa552 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -185,6 +186,11 @@ default CompressionType getDefaultCompressor(TSFileConfig config) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns a decoder for the specified encoding. */ + default Decoder getDecoder(TSEncoding encoding) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java index 52db17a39..8bcab3d10 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java @@ -39,5 +39,6 @@ public enum TypeEnum { DATE, BLOB, STRING, - OBJECT + OBJECT, + VECTOR } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java index a768702ca..7e4d542f0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java @@ -41,6 +41,8 @@ public static Type getType(TSDataType tsDataType) { case STRING -> StringType.getInstance(); case BLOB -> BlobType.getInstance(); case OBJECT -> ObjectType.getInstance(); + case VECTOR -> VectorType.getInstance(); + case UNKNOWN -> UnknownType.getInstance(); default -> throw new UnsupportedOperationException( Messages.format("error.read.typefactory_invalid_tsdata_type", tsDataType)); @@ -60,6 +62,7 @@ public static Type getType(TypeEnum typeEnum) { case TIMESTAMP -> TimestampType.getInstance(); case BLOB -> BlobType.getInstance(); case STRING -> StringType.getInstance(); + case VECTOR -> VectorType.getInstance(); default -> throw new UnsupportedOperationException( Messages.format("error.read.typefactory_invalid_type_enum", typeEnum)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java new file mode 100644 index 000000000..51d07fe70 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.common.type; + +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; +import org.apache.tsfile.encoding.decoder.LongChimpDecoder; +import org.apache.tsfile.encoding.decoder.LongGorillaDecoder; +import org.apache.tsfile.encoding.decoder.LongRleDecoder; +import org.apache.tsfile.encoding.decoder.RegularDataDecoder; +import org.apache.tsfile.file.metadata.enums.TSEncoding; + +public class VectorType extends AbstractLongType { + + public static final VectorType VECTOR = new VectorType(); + + private VectorType() {} + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new LongRleDecoder(); + case TS_2DIFF -> new DeltaBinaryDecoder.LongDeltaDecoder(); + case REGULAR -> new RegularDataDecoder.LongRegularDecoder(); + case GORILLA -> new LongGorillaDecoder(); + case CHIMP -> new LongChimpDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + @Override + public TypeEnum getTypeEnum() { + return TypeEnum.VECTOR; + } + + @Override + public String getDisplayName() { + return "VECTOR"; + } + + public static VectorType getInstance() { + return VECTOR; + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/DecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/DecoderTest.java new file mode 100644 index 000000000..000e08206 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/DecoderTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.encoding.decoder; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; + +import org.junit.Assert; +import org.junit.Test; + +public class DecoderTest { + + @Test + public void testGetDecoderByType() { + Object[][] testCases = { + {TSEncoding.PLAIN, TSDataType.UNKNOWN, PlainDecoder.class}, + {TSEncoding.DICTIONARY, TSDataType.BLOB, DictionaryDecoder.class}, + {TSEncoding.RLE, TSDataType.BOOLEAN, IntRleDecoder.class}, + {TSEncoding.RLE, TSDataType.INT32, IntRleDecoder.class}, + {TSEncoding.RLE, TSDataType.DATE, IntRleDecoder.class}, + {TSEncoding.RLE, TSDataType.INT64, LongRleDecoder.class}, + {TSEncoding.RLE, TSDataType.VECTOR, LongRleDecoder.class}, + {TSEncoding.RLE, TSDataType.TIMESTAMP, LongRleDecoder.class}, + {TSEncoding.RLE, TSDataType.FLOAT, FloatDecoder.class}, + {TSEncoding.RLE, TSDataType.DOUBLE, FloatDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.INT32, DeltaBinaryDecoder.IntDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.DATE, DeltaBinaryDecoder.IntDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.INT64, DeltaBinaryDecoder.LongDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.VECTOR, DeltaBinaryDecoder.LongDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.TIMESTAMP, DeltaBinaryDecoder.LongDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.FLOAT, FloatDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.DOUBLE, FloatDecoder.class}, + {TSEncoding.GORILLA_V1, TSDataType.FLOAT, SinglePrecisionDecoderV1.class}, + {TSEncoding.GORILLA_V1, TSDataType.DOUBLE, DoublePrecisionDecoderV1.class}, + {TSEncoding.REGULAR, TSDataType.INT32, RegularDataDecoder.IntRegularDecoder.class}, + {TSEncoding.REGULAR, TSDataType.VECTOR, RegularDataDecoder.LongRegularDecoder.class}, + {TSEncoding.GORILLA, TSDataType.INT32, IntGorillaDecoder.class}, + {TSEncoding.GORILLA, TSDataType.VECTOR, LongGorillaDecoder.class}, + {TSEncoding.GORILLA, TSDataType.FLOAT, SinglePrecisionDecoderV2.class}, + {TSEncoding.GORILLA, TSDataType.DOUBLE, DoublePrecisionDecoderV2.class}, + {TSEncoding.ZIGZAG, TSDataType.INT32, IntZigzagDecoder.class}, + {TSEncoding.ZIGZAG, TSDataType.INT64, LongZigzagDecoder.class}, + {TSEncoding.CHIMP, TSDataType.INT32, IntChimpDecoder.class}, + {TSEncoding.CHIMP, TSDataType.VECTOR, LongChimpDecoder.class}, + {TSEncoding.CHIMP, TSDataType.FLOAT, SinglePrecisionChimpDecoder.class}, + {TSEncoding.CHIMP, TSDataType.DOUBLE, DoublePrecisionChimpDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.INT32, IntSprintzDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.INT64, LongSprintzDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.FLOAT, FloatSprintzDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.DOUBLE, DoubleSprintzDecoder.class}, + {TSEncoding.RLBE, TSDataType.INT32, IntRLBEDecoder.class}, + {TSEncoding.RLBE, TSDataType.INT64, LongRLBEDecoder.class}, + {TSEncoding.RLBE, TSDataType.FLOAT, FloatRLBEDecoder.class}, + {TSEncoding.RLBE, TSDataType.DOUBLE, DoubleRLBEDecoder.class}, + {TSEncoding.CAMEL, TSDataType.DOUBLE, CamelDecoder.class} + }; + + for (Object[] testCase : testCases) { + Assert.assertEquals( + testCase[2], + Decoder.getDecoderByType((TSEncoding) testCase[0], (TSDataType) testCase[1]).getClass()); + } + } + + @Test + public void testUnsupportedDecoder() { + Object[][] testCases = { + {TSEncoding.RLE, TSDataType.TEXT}, + {TSEncoding.TS_2DIFF, TSDataType.BOOLEAN}, + {TSEncoding.GORILLA_V1, TSDataType.INT32}, + {TSEncoding.REGULAR, TSDataType.FLOAT}, + {TSEncoding.GORILLA, TSDataType.BOOLEAN}, + {TSEncoding.ZIGZAG, TSDataType.VECTOR}, + {TSEncoding.CHIMP, TSDataType.TEXT}, + {TSEncoding.SPRINTZ, TSDataType.VECTOR}, + {TSEncoding.RLBE, TSDataType.VECTOR}, + {TSEncoding.CAMEL, TSDataType.FLOAT} + }; + + for (Object[] testCase : testCases) { + try { + Decoder.getDecoderByType((TSEncoding) testCase[0], (TSDataType) testCase[1]); + Assert.fail("Expected TsFileDecodingException"); + } catch (TsFileDecodingException ignored) { + // Expected. + } + } + } +} From 0050db8c4f8007a57377849a4572e81fe2ad29e2 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 09:55:22 +0800 Subject: [PATCH 020/131] Add getOneItemMaxSize --- .../tsfile/encoding/encoder/PlainEncoder.java | 25 +----------- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/VectorType.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 38 +++++++++++++++++++ 12 files changed, 90 insertions(+), 23 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java index 0f9fd8206..f2b518c61 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java @@ -19,11 +19,11 @@ package org.apache.tsfile.encoding.encoder; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.encoding.TsFileEncodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -112,28 +112,7 @@ public void flush(ByteArrayOutputStream out) { @Override public int getOneItemMaxSize() { - switch (dataType) { - case BOOLEAN: - return 1; - case INT32: - case DATE: - return 4; - case INT64: - case TIMESTAMP: - return 8; - case FLOAT: - return 4; - case DOUBLE: - return 8; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - // refer to encode(Binary,ByteArrayOutputStream) - return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength; - default: - throw new UnsupportedOperationException(dataType.toString()); - } + return Type.fromTsDataType(dataType).getOneItemMaxSize(maxStringLength); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index e29579709..8a2a59d0d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -87,6 +87,11 @@ public Decoder getDecoder(TSEncoding encoding) { }; } + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 32970cc93..284167de1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -87,6 +87,11 @@ public Decoder getDecoder(TSEncoding encoding) { }; } + @Override + public int getOneItemMaxSize(int valveLength) { + return Long.BYTES; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 18d6a9822..6c568cfcf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -61,6 +61,11 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getTextCompressor(); } + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index e67d6c86e..621a776f1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -65,6 +65,11 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getTextCompressor(); } + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 41734a2a3..87e169541 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -77,6 +77,11 @@ public Decoder getDecoder(TSEncoding encoding) { }; } + @Override + public int getOneItemMaxSize(int valveLength) { + return Byte.BYTES; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index e20f21778..54e17f0f6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -89,6 +89,11 @@ public Decoder getDecoder(TSEncoding encoding) { }; } + @Override + public int getOneItemMaxSize(int valveLength) { + return Double.BYTES; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 4fc36af44..13bfcffe3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -87,6 +87,11 @@ public Decoder getDecoder(TSEncoding encoding) { }; } + @Override + public int getOneItemMaxSize(int valveLength) { + return Float.BYTES; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 215b03386..350bfc652 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -65,6 +65,11 @@ public CompressionType getDefaultCompressor(TSFileConfig config) { return config.getTextCompressor(); } + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 9c68fa552..0215724b5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -191,6 +191,11 @@ default Decoder getDecoder(TSEncoding encoding) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the maximum encoded size of one item. */ + default int getOneItemMaxSize(int valveLength) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 51d07fe70..836f2c0bc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -46,6 +46,11 @@ public Decoder getDecoder(TSEncoding encoding) { }; } + @Override + public int getOneItemMaxSize(int valveLength) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.VECTOR; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 41bcad617..b83f82fd1 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -19,6 +19,8 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.encoder.PlainEncoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; @@ -33,6 +35,7 @@ public class TypeTest { private static final int OFFSET = 2; + private static final int VALUE_LENGTH = 10; @Test public void testToBytes() { @@ -112,4 +115,39 @@ public void testCalcTypeSize() { .calcTypeSize(dataType, new TsPrimitiveType.TsBinary(binary))); } } + + @Test + public void testGetOneItemMaxSize() { + Object[][] testCases = { + {TSDataType.BOOLEAN, Byte.BYTES}, + {TSDataType.INT32, Integer.BYTES}, + {TSDataType.DATE, Integer.BYTES}, + {TSDataType.INT64, Long.BYTES}, + {TSDataType.TIMESTAMP, Long.BYTES}, + {TSDataType.FLOAT, Float.BYTES}, + {TSDataType.DOUBLE, Double.BYTES}, + {TSDataType.TEXT, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH}, + {TSDataType.STRING, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH}, + {TSDataType.BLOB, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH}, + {TSDataType.OBJECT, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + int expectedSize = (int) testCase[1]; + Assert.assertEquals( + expectedSize, Type.fromTsDataType(dataType).getOneItemMaxSize(VALUE_LENGTH)); + Assert.assertEquals( + expectedSize, new PlainEncoder(dataType, VALUE_LENGTH).getOneItemMaxSize()); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + new PlainEncoder(dataType, VALUE_LENGTH).getOneItemMaxSize(); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } } From 7111b7ce9a1675eef4e981e1fe2aef91b20f6ac7 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 10:26:09 +0800 Subject: [PATCH 021/131] Add updateStatistics --- .../tsfile/read/TsFileSequenceReader.java | 89 ++----------------- .../read/common/type/AbstractIntType.java | 12 +++ .../read/common/type/AbstractLongType.java | 12 +++ .../read/common/type/AbstractVarcharType.java | 12 +++ .../tsfile/read/common/type/BlobType.java | 12 +++ .../tsfile/read/common/type/BooleanType.java | 12 +++ .../tsfile/read/common/type/DoubleType.java | 12 +++ .../tsfile/read/common/type/FloatType.java | 12 +++ .../tsfile/read/common/type/ObjectType.java | 12 +++ .../apache/tsfile/read/common/type/Type.java | 12 +++ .../tsfile/read/common/type/VectorType.java | 13 +++ .../tsfile/read/common/type/TypeTest.java | 41 +++++++++ 12 files changed, 168 insertions(+), 83 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 97dcddc0d..396a75576 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -66,6 +66,7 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.tsfile.read.reader.TsFileInput; @@ -2457,6 +2458,7 @@ public long selfCheck( chunkHeader.getCompressionType()); measurementSchemaList.add(measurementSchema); dataType = chunkHeader.getDataType(); + Type valueType = Type.fromTsDataType(dataType); Statistics chunkStatistics = Statistics.getStatsByType(dataType); @@ -2534,34 +2536,7 @@ public long selfCheck( continue; } long timeStamp = timeBatch.get(timeBatchIndex)[i]; - switch (dataType) { - case INT32: - case DATE: - chunkStatistics.update(timeStamp, value.getInt()); - break; - case INT64: - case TIMESTAMP: - chunkStatistics.update(timeStamp, value.getLong()); - break; - case FLOAT: - chunkStatistics.update(timeStamp, value.getFloat()); - break; - case DOUBLE: - chunkStatistics.update(timeStamp, value.getDouble()); - break; - case BOOLEAN: - chunkStatistics.update(timeStamp, value.getBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkStatistics.update(timeStamp, value.getBinary()); - break; - default: - throw new IOException( - Messages.format("error.read.unexpected_type", dataType)); - } + valueType.update(chunkStatistics, timeStamp, value); } } @@ -2575,34 +2550,7 @@ public long selfCheck( timeDecoder); BatchData batchData = reader.getAllSatisfiedPageData(); while (batchData.hasCurrent()) { - switch (dataType) { - case INT32: - case DATE: - chunkStatistics.update(batchData.currentTime(), batchData.getInt()); - break; - case INT64: - case TIMESTAMP: - chunkStatistics.update(batchData.currentTime(), batchData.getLong()); - break; - case FLOAT: - chunkStatistics.update(batchData.currentTime(), batchData.getFloat()); - break; - case DOUBLE: - chunkStatistics.update(batchData.currentTime(), batchData.getDouble()); - break; - case BOOLEAN: - chunkStatistics.update(batchData.currentTime(), batchData.getBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkStatistics.update(batchData.currentTime(), batchData.getBinary()); - break; - default: - throw new IOException( - Messages.format("error.read.unexpected_type", dataType)); - } + valueType.update(chunkStatistics, batchData); batchData.next(); } } @@ -2781,6 +2729,7 @@ public long checkChunkAndPagesStatistics(IChunkMetadata chunkMetadata) throws IO byte marker = this.readMarker(); ChunkHeader chunkHeader = this.readChunkHeader(marker); TSDataType dataType = chunkHeader.getDataType(); + Type valueType = Type.fromTsDataType(dataType); Statistics chunkStatistics = Statistics.getStatsByType(dataType); int dataSize = chunkHeader.getDataSize(); if (((byte) (chunkHeader.getChunkType() & 0x3F)) == MetaMarker.CHUNK_HEADER) { @@ -2808,33 +2757,7 @@ public long checkChunkAndPagesStatistics(IChunkMetadata chunkMetadata) throws IO pageHeader, pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder); BatchData batchData = reader.getAllSatisfiedPageData(); while (batchData.hasCurrent()) { - switch (dataType) { - case INT32: - case DATE: - chunkStatistics.update(batchData.currentTime(), batchData.getInt()); - break; - case INT64: - case TIMESTAMP: - chunkStatistics.update(batchData.currentTime(), batchData.getLong()); - break; - case FLOAT: - chunkStatistics.update(batchData.currentTime(), batchData.getFloat()); - break; - case DOUBLE: - chunkStatistics.update(batchData.currentTime(), batchData.getDouble()); - break; - case BOOLEAN: - chunkStatistics.update(batchData.currentTime(), batchData.getBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkStatistics.update(batchData.currentTime(), batchData.getBinary()); - break; - default: - throw new IOException(Messages.format("error.read.unexpected_type", dataType)); - } + valueType.update(chunkStatistics, batchData); batchData.next(); } chunkHeader.increasePageNums(1); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 8a2a59d0d..df6a15ce9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -34,6 +34,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -92,6 +94,16 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getInt()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getInt()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 284167de1..499aeea82 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -34,6 +34,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -92,6 +94,16 @@ public int getOneItemMaxSize(int valveLength) { return Long.BYTES; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getLong()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getLong()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 6c568cfcf..f2d5682cb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -25,7 +25,9 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -66,6 +68,16 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBinary()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBinary()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 621a776f1..41da8f06f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -25,7 +25,9 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -70,6 +72,16 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBinary()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBinary()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 87e169541..4b9b75586 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -27,6 +27,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -82,6 +84,16 @@ public int getOneItemMaxSize(int valveLength) { return Byte.BYTES; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBoolean()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBoolean()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 54e17f0f6..cbc074a49 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -33,6 +33,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -94,6 +96,16 @@ public int getOneItemMaxSize(int valveLength) { return Double.BYTES; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getDouble()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getDouble()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 13bfcffe3..de5b2ee58 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -32,6 +32,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -92,6 +94,16 @@ public int getOneItemMaxSize(int valveLength) { return Float.BYTES; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getFloat()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getFloat()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 350bfc652..bd0cae2e8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -25,7 +25,9 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -70,6 +72,16 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBinary()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBinary()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 0215724b5..91cf7a733 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -26,6 +26,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -196,6 +198,16 @@ default int getOneItemMaxSize(int valveLength) { throw new UnsupportedOperationException(getClass().getName()); } + /** Updates statistics with a timestamp-value pair. */ + default void update(Statistics stats, long timestamp, TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Updates statistics with the current timestamp-value pair in {@code batchData}. */ + default void update(Statistics stats, BatchData batchData) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 836f2c0bc..a9b293f8a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -26,6 +26,9 @@ import org.apache.tsfile.encoding.decoder.LongRleDecoder; import org.apache.tsfile.encoding.decoder.RegularDataDecoder; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.utils.TsPrimitiveType; public class VectorType extends AbstractLongType { @@ -51,6 +54,16 @@ public int getOneItemMaxSize(int valveLength) { throw new UnsupportedOperationException(getDisplayName()); } + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.VECTOR; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index b83f82fd1..afad64c26 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -22,6 +22,8 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.encoder.PlainEncoder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -150,4 +152,43 @@ public void testGetOneItemMaxSize() { } } } + + @Test + public void testUpdateStatistics() { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260714, TSDataType.DATE)}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.0F)}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(1.0D)}, + {TSDataType.TEXT, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.STRING, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.BLOB, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.OBJECT, new TsPrimitiveType.TsBinary(binary)} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + TsPrimitiveType value = (TsPrimitiveType) testCase[1]; + Type type = Type.fromTsDataType(dataType); + Statistics statistics = Statistics.getStatsByType(dataType); + type.update(statistics, 100L, value); + Assert.assertEquals(1, statistics.getCount()); + Assert.assertEquals(100L, statistics.getStartTime()); + Assert.assertEquals(100L, statistics.getEndTime()); + Assert.assertFalse(statistics.isEmpty()); + + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(200L, value.getValue()); + Statistics batchStatistics = Statistics.getStatsByType(dataType); + type.update(batchStatistics, batchData); + Assert.assertEquals(1, batchStatistics.getCount()); + Assert.assertEquals(200L, batchStatistics.getStartTime()); + Assert.assertEquals(200L, batchStatistics.getEndTime()); + Assert.assertFalse(batchStatistics.isEmpty()); + } + } } From 59a928070e7c00bdf004e67623ae67e67ddc1d81 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 10:47:44 +0800 Subject: [PATCH 022/131] Add getCurrentValue --- .../apache/tsfile/read/common/BatchData.java | 25 ++----------------- .../read/common/type/AbstractIntType.java | 5 ++++ .../read/common/type/AbstractLongType.java | 5 ++++ .../read/common/type/AbstractVarcharType.java | 5 ++++ .../tsfile/read/common/type/BlobType.java | 5 ++++ .../tsfile/read/common/type/BooleanType.java | 5 ++++ .../tsfile/read/common/type/DoubleType.java | 5 ++++ .../tsfile/read/common/type/FloatType.java | 5 ++++ .../tsfile/read/common/type/ObjectType.java | 5 ++++ .../apache/tsfile/read/common/type/Type.java | 5 ++++ .../tsfile/read/common/type/UnknownType.java | 6 +++++ .../tsfile/read/common/type/VectorType.java | 5 ++++ .../tsfile/read/common/type/TypeTest.java | 12 +++++++++ 13 files changed, 70 insertions(+), 23 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index 9fbda97f1..f361b6535 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -23,6 +23,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -129,29 +130,7 @@ public long currentTime() { * @return current value */ public Object currentValue() { - switch (dataType) { - case INT32: - case DATE: - return getInt(); - case INT64: - case TIMESTAMP: - return getLong(); - case FLOAT: - return getFloat(); - case DOUBLE: - return getDouble(); - case BOOLEAN: - return getBoolean(); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return getBinary(); - case VECTOR: - return getVector(); - default: - return null; - } + return Type.fromTsDataType(dataType).getCurrentValue(this); } public TsPrimitiveType currentTsPrimitiveType() { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index df6a15ce9..b4c22e057 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -104,6 +104,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getInt()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getInt(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 499aeea82..87fd18c64 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -104,6 +104,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getLong()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getLong(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index f2d5682cb..547d78b27 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -78,6 +78,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBinary()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBinary(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 41da8f06f..6d083bc2c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -82,6 +82,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBinary()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBinary(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 4b9b75586..7c45ec378 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -94,6 +94,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBoolean()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBoolean(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index cbc074a49..e4f40a8b1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -106,6 +106,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getDouble()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getDouble(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index de5b2ee58..f68d679ee 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -104,6 +104,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getFloat()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getFloat(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index bd0cae2e8..a51e55a29 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -82,6 +82,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBinary()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBinary(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 91cf7a733..d1871d4bc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -208,6 +208,11 @@ default void update(Statistics stats, BatchData batchData) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the current value in {@code batchData}. */ + default Object getCurrentValue(BatchData batchData) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java index 802165dc8..7e80a8b44 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import java.util.Collections; @@ -51,6 +52,11 @@ public boolean getBoolean(Column column, int position) { return false; } + @Override + public Object getCurrentValue(BatchData batchData) { + return null; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index a9b293f8a..629ca9cf5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -64,6 +64,11 @@ public void update(Statistics stats, BatchData batchData) { throw new UnsupportedOperationException(getDisplayName()); } + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getVector(); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.VECTOR; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index afad64c26..1358d2a77 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -183,6 +183,8 @@ public void testUpdateStatistics() { BatchData batchData = new BatchData(dataType); batchData.putAnObject(200L, value.getValue()); + Assert.assertEquals(value.getValue(), type.getCurrentValue(batchData)); + Assert.assertEquals(value.getValue(), batchData.currentValue()); Statistics batchStatistics = Statistics.getStatsByType(dataType); type.update(batchStatistics, batchData); Assert.assertEquals(1, batchStatistics.getCount()); @@ -191,4 +193,14 @@ public void testUpdateStatistics() { Assert.assertFalse(batchStatistics.isEmpty()); } } + + @Test + public void testGetCurrentValueForVectorAndUnknown() { + TsPrimitiveType[] vector = {new TsPrimitiveType.TsLong(1L)}; + BatchData batchData = new BatchData(TSDataType.VECTOR); + batchData.putVector(1L, vector); + Assert.assertSame(vector, Type.fromTsDataType(TSDataType.VECTOR).getCurrentValue(batchData)); + Assert.assertSame(vector, batchData.currentValue()); + Assert.assertNull(Type.fromTsDataType(TSDataType.UNKNOWN).getCurrentValue(null)); + } } From f0672889302504187d4d1e3cd1d8037d938836a5 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 11:17:20 +0800 Subject: [PATCH 023/131] refactor calcTypeSize --- .../read/common/type/AbstractIntType.java | 2 +- .../read/common/type/AbstractLongType.java | 3 +-- .../read/common/type/AbstractVarcharType.java | 3 +-- .../tsfile/read/common/type/BlobType.java | 3 +-- .../tsfile/read/common/type/BooleanType.java | 3 +-- .../tsfile/read/common/type/DoubleType.java | 2 +- .../tsfile/read/common/type/FloatType.java | 2 +- .../tsfile/read/common/type/ObjectType.java | 3 +-- .../apache/tsfile/read/common/type/Type.java | 2 +- .../tsfile/read/common/type/TypeTest.java | 23 +++++++------------ 10 files changed, 17 insertions(+), 29 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index b4c22e057..81a50b5e3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -59,7 +59,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return Integer.BYTES; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 87fd18c64..a3306386d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -31,7 +31,6 @@ import org.apache.tsfile.encoding.decoder.LongSprintzDecoder; import org.apache.tsfile.encoding.decoder.LongZigzagDecoder; import org.apache.tsfile.encoding.decoder.RegularDataDecoder; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -59,7 +58,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return Long.BYTES; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 547d78b27..c4b413618 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -22,7 +22,6 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -49,7 +48,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return binaryTypeSize(value); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 6d083bc2c..97799cd11 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -22,7 +22,6 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -53,7 +52,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return binaryTypeSize(value); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 7c45ec378..1f8628512 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -24,7 +24,6 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.encoding.decoder.IntRleDecoder; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -56,7 +55,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return Byte.BYTES; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index e4f40a8b1..09954a59f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -62,7 +62,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return Double.BYTES; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index f68d679ee..b107a2b56 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -61,7 +61,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return Float.BYTES; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index a51e55a29..9d12b84dc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -22,7 +22,6 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -53,7 +52,7 @@ public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } @Override - public int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + public int calcTypeSize(TsPrimitiveType value) { return binaryTypeSize(value); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index d1871d4bc..cbfd6f697 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -93,7 +93,7 @@ default void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { } /** Returns the number of bytes required to serialize {@code value}. */ - default int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { + default int calcTypeSize(TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 1358d2a77..d90adf94f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -81,40 +81,33 @@ public void testToBytes() { public void testCalcTypeSize() { Assert.assertEquals( Byte.BYTES, - Type.fromTsDataType(TSDataType.BOOLEAN) - .calcTypeSize(TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true))); + Type.fromTsDataType(TSDataType.BOOLEAN).calcTypeSize(new TsPrimitiveType.TsBoolean(true))); Assert.assertEquals( Integer.BYTES, - Type.fromTsDataType(TSDataType.INT32) - .calcTypeSize(TSDataType.INT32, new TsPrimitiveType.TsInt(1))); + Type.fromTsDataType(TSDataType.INT32).calcTypeSize(new TsPrimitiveType.TsInt(1))); Assert.assertEquals( Integer.BYTES, Type.fromTsDataType(TSDataType.DATE) - .calcTypeSize(TSDataType.DATE, new TsPrimitiveType.TsInt(20260713, TSDataType.DATE))); + .calcTypeSize(new TsPrimitiveType.TsInt(20260713, TSDataType.DATE))); Assert.assertEquals( Long.BYTES, - Type.fromTsDataType(TSDataType.INT64) - .calcTypeSize(TSDataType.INT64, new TsPrimitiveType.TsLong(1L))); + Type.fromTsDataType(TSDataType.INT64).calcTypeSize(new TsPrimitiveType.TsLong(1L))); Assert.assertEquals( Long.BYTES, - Type.fromTsDataType(TSDataType.TIMESTAMP) - .calcTypeSize(TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(1L))); + Type.fromTsDataType(TSDataType.TIMESTAMP).calcTypeSize(new TsPrimitiveType.TsLong(1L))); Assert.assertEquals( Float.BYTES, - Type.fromTsDataType(TSDataType.FLOAT) - .calcTypeSize(TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.0F))); + Type.fromTsDataType(TSDataType.FLOAT).calcTypeSize(new TsPrimitiveType.TsFloat(1.0F))); Assert.assertEquals( Double.BYTES, - Type.fromTsDataType(TSDataType.DOUBLE) - .calcTypeSize(TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(1.0D))); + Type.fromTsDataType(TSDataType.DOUBLE).calcTypeSize(new TsPrimitiveType.TsDouble(1.0D))); Binary binary = new Binary("test", StandardCharsets.UTF_8); for (TSDataType dataType : new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { Assert.assertEquals( Integer.BYTES + binary.getLength(), - Type.fromTsDataType(dataType) - .calcTypeSize(dataType, new TsPrimitiveType.TsBinary(binary))); + Type.fromTsDataType(dataType).calcTypeSize(new TsPrimitiveType.TsBinary(binary))); } } From f384a1eb2beca962c8d1aabe0047b0d05eea7000 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 11:39:41 +0800 Subject: [PATCH 024/131] refactor calcTypeSize --- .../read/common/type/AbstractIntType.java | 5 ++++ .../read/common/type/AbstractLongType.java | 5 ++++ .../tsfile/read/common/type/AbstractType.java | 4 +++ .../read/common/type/AbstractVarcharType.java | 5 ++++ .../tsfile/read/common/type/BlobType.java | 5 ++++ .../tsfile/read/common/type/BooleanType.java | 5 ++++ .../tsfile/read/common/type/DoubleType.java | 5 ++++ .../tsfile/read/common/type/FloatType.java | 5 ++++ .../tsfile/read/common/type/ObjectType.java | 5 ++++ .../apache/tsfile/read/common/type/Type.java | 5 ++++ .../tsfile/read/common/type/TypeTest.java | 25 +++++++++++++++++++ 11 files changed, 74 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 81a50b5e3..aa993730a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -63,6 +63,11 @@ public int calcTypeSize(TsPrimitiveType value) { return Integer.BYTES; } + @Override + public int calcTypeSize(Object value) { + return Integer.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index a3306386d..469ab4efc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -62,6 +62,11 @@ public int calcTypeSize(TsPrimitiveType value) { return Long.BYTES; } + @Override + public int calcTypeSize(Object value) { + return Long.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index ae86cd235..82ab72d7f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -64,6 +64,10 @@ protected int binaryTypeSize(TsPrimitiveType value) { return Integer.BYTES + value.getBinary().getLength(); } + protected int binaryTypeSize(Object value) { + return Integer.BYTES + ((Binary) value).getLength(); + } + protected void checkValueType(Object value, Class expectedClass, String expectedType) { if (value != null && !expectedClass.isInstance(value)) { throw new IllegalArgumentException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index c4b413618..72721eca1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -52,6 +52,11 @@ public int calcTypeSize(TsPrimitiveType value) { return binaryTypeSize(value); } + @Override + public int calcTypeSize(Object value) { + return binaryTypeSize(value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 97799cd11..0120e7c77 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -56,6 +56,11 @@ public int calcTypeSize(TsPrimitiveType value) { return binaryTypeSize(value); } + @Override + public int calcTypeSize(Object value) { + return binaryTypeSize(value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 1f8628512..d8c22362a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -59,6 +59,11 @@ public int calcTypeSize(TsPrimitiveType value) { return Byte.BYTES; } + @Override + public int calcTypeSize(Object value) { + return Byte.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 09954a59f..2dc028022 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -66,6 +66,11 @@ public int calcTypeSize(TsPrimitiveType value) { return Double.BYTES; } + @Override + public int calcTypeSize(Object value) { + return Double.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index b107a2b56..0362dcb88 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -65,6 +65,11 @@ public int calcTypeSize(TsPrimitiveType value) { return Float.BYTES; } + @Override + public int calcTypeSize(Object value) { + return Float.BYTES; + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 9d12b84dc..720ce0aed 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -56,6 +56,11 @@ public int calcTypeSize(TsPrimitiveType value) { return binaryTypeSize(value); } + @Override + public int calcTypeSize(Object value) { + return binaryTypeSize(value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index cbfd6f697..dca823429 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -97,6 +97,11 @@ default int calcTypeSize(TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the number of bytes required to serialize {@code value}. */ + default int calcTypeSize(Object value) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index d90adf94f..47dad3dbc 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -111,6 +111,31 @@ public void testCalcTypeSize() { } } + @Test + public void testCalcTypeSizeFromObject() { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true, Byte.BYTES}, + {TSDataType.INT32, 1, Integer.BYTES}, + {TSDataType.DATE, 20260713, Integer.BYTES}, + {TSDataType.INT64, 1L, Long.BYTES}, + {TSDataType.TIMESTAMP, 1L, Long.BYTES}, + {TSDataType.FLOAT, 1.0F, Float.BYTES}, + {TSDataType.DOUBLE, 1.0D, Double.BYTES}, + {TSDataType.TEXT, binary, Integer.BYTES + binary.getLength()}, + {TSDataType.STRING, binary, Integer.BYTES + binary.getLength()}, + {TSDataType.BLOB, binary, Integer.BYTES + binary.getLength()}, + {TSDataType.OBJECT, binary, Integer.BYTES + binary.getLength()} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Object value = testCase[1]; + int expectedSize = (int) testCase[2]; + Assert.assertEquals(expectedSize, Type.fromTsDataType(dataType).calcTypeSize(value)); + } + } + @Test public void testGetOneItemMaxSize() { Object[][] testCases = { From a403a13a29d357f66a6a0e3d4a2fb7ed9ac9162a Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 11:43:52 +0800 Subject: [PATCH 025/131] refactor currentTsPrimitiveType --- .../apache/tsfile/read/common/BatchData.java | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index f361b6535..4abed19c6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -134,29 +134,8 @@ public Object currentValue() { } public TsPrimitiveType currentTsPrimitiveType() { - switch (dataType) { - case INT32: - case DATE: - return new TsInt(getInt()); - case INT64: - case TIMESTAMP: - return new TsLong(getLong()); - case FLOAT: - return new TsFloat(getFloat()); - case DOUBLE: - return new TsDouble(getDouble()); - case BOOLEAN: - return new TsBoolean(getBoolean()); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new TsBinary(getBinary()); - case VECTOR: - return new TsVector(getVector()); - default: - return null; - } + Type type = Type.fromTsDataType(dataType); + return type.getTsPrimitiveType(type.getCurrentValue(this)); } public TSDataType getDataType() { From 61b44e7b999cdf0ec653657849cbc8e20db68d0b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 12:23:13 +0800 Subject: [PATCH 026/131] refactor BatchData.init --- .../apache/tsfile/read/common/BatchData.java | 80 +++++++++---------- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 6 ++ .../tsfile/read/common/type/VectorType.java | 5 ++ 11 files changed, 87 insertions(+), 44 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index 4abed19c6..bdd299ee9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -27,13 +27,6 @@ import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.utils.TsPrimitiveType.TsBinary; -import org.apache.tsfile.utils.TsPrimitiveType.TsBoolean; -import org.apache.tsfile.utils.TsPrimitiveType.TsDouble; -import org.apache.tsfile.utils.TsPrimitiveType.TsFloat; -import org.apache.tsfile.utils.TsPrimitiveType.TsInt; -import org.apache.tsfile.utils.TsPrimitiveType.TsLong; -import org.apache.tsfile.utils.TsPrimitiveType.TsVector; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.DataOutputStream; @@ -166,43 +159,42 @@ public void init(TSDataType type) { timeRet.add(new long[capacity]); count = 0; - switch (dataType) { - case BOOLEAN: - booleanRet = new ArrayList<>(); - booleanRet.add(new boolean[capacity]); - break; - case INT32: - case DATE: - intRet = new ArrayList<>(); - intRet.add(new int[capacity]); - break; - case INT64: - case TIMESTAMP: - longRet = new ArrayList<>(); - longRet.add(new long[capacity]); - break; - case FLOAT: - floatRet = new ArrayList<>(); - floatRet.add(new float[capacity]); - break; - case DOUBLE: - doubleRet = new ArrayList<>(); - doubleRet.add(new double[capacity]); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - binaryRet = new ArrayList<>(); - binaryRet.add(new Binary[capacity]); - break; - case VECTOR: - vectorRet = new ArrayList<>(); - vectorRet.add(new TsPrimitiveType[capacity][]); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).init(this); + } + + public void initBooleanValues() { + booleanRet = new ArrayList<>(); + booleanRet.add(new boolean[capacity]); + } + + public void initIntValues() { + intRet = new ArrayList<>(); + intRet.add(new int[capacity]); + } + + public void initLongValues() { + longRet = new ArrayList<>(); + longRet.add(new long[capacity]); + } + + public void initFloatValues() { + floatRet = new ArrayList<>(); + floatRet.add(new float[capacity]); + } + + public void initDoubleValues() { + doubleRet = new ArrayList<>(); + doubleRet.add(new double[capacity]); + } + + public void initBinaryValues() { + binaryRet = new ArrayList<>(); + binaryRet.add(new Binary[capacity]); + } + + public void initVectorValues() { + vectorRet = new ArrayList<>(); + vectorRet.add(new TsPrimitiveType[capacity][]); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index aa993730a..3ebe4daf3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -68,6 +68,11 @@ public int calcTypeSize(Object value) { return Integer.BYTES; } + @Override + public void init(BatchData batchData) { + batchData.initIntValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 469ab4efc..1bd70bd60 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -67,6 +67,11 @@ public int calcTypeSize(Object value) { return Long.BYTES; } + @Override + public void init(BatchData batchData) { + batchData.initLongValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 72721eca1..61fbff51c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -57,6 +57,11 @@ public int calcTypeSize(Object value) { return binaryTypeSize(value); } + @Override + public void init(BatchData batchData) { + batchData.initBinaryValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 0120e7c77..bb31c8338 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -61,6 +61,11 @@ public int calcTypeSize(Object value) { return binaryTypeSize(value); } + @Override + public void init(BatchData batchData) { + batchData.initBinaryValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index d8c22362a..d4806ce02 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -64,6 +64,11 @@ public int calcTypeSize(Object value) { return Byte.BYTES; } + @Override + public void init(BatchData batchData) { + batchData.initBooleanValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 2dc028022..05809d031 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -71,6 +71,11 @@ public int calcTypeSize(Object value) { return Double.BYTES; } + @Override + public void init(BatchData batchData) { + batchData.initDoubleValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 0362dcb88..ce6ecfa25 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -70,6 +70,11 @@ public int calcTypeSize(Object value) { return Float.BYTES; } + @Override + public void init(BatchData batchData) { + batchData.initFloatValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 720ce0aed..155c41cd1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -61,6 +61,11 @@ public int calcTypeSize(Object value) { return binaryTypeSize(value); } + @Override + public void init(BatchData batchData) { + batchData.initBinaryValues(); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index dca823429..8519908a7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -30,6 +30,7 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.DataOutputStream; import java.io.IOException; @@ -102,6 +103,11 @@ default int calcTypeSize(Object value) { throw new UnsupportedOperationException(getClass().getName()); } + /** Initializes the value storage in {@code batchData} for this type. */ + default void init(BatchData batchData) { + throw new UnSupportedDataTypeException(String.valueOf(batchData.getDataType())); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 629ca9cf5..e14737894 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -36,6 +36,11 @@ public class VectorType extends AbstractLongType { private VectorType() {} + @Override + public void init(BatchData batchData) { + batchData.initVectorValues(); + } + @Override public Decoder getDecoder(TSEncoding encoding) { return switch (encoding) { From baa472265b043a5c66caff7cdb53fe363a105a95 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 14:07:49 +0800 Subject: [PATCH 027/131] refactor BatchData.putAnObject --- .../apache/tsfile/read/common/BatchData.java | 32 +------------------ .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/VectorType.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 2 +- 12 files changed, 52 insertions(+), 32 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index bdd299ee9..4e0bc430e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -27,7 +27,6 @@ import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.DataOutputStream; import java.io.IOException; @@ -508,36 +507,7 @@ public void setTime(long v) { * @param v object */ public void putAnObject(long t, Object v) { - switch (dataType) { - case BOOLEAN: - putBoolean(t, (boolean) v); - break; - case INT32: - case DATE: - putInt(t, (int) v); - break; - case INT64: - case TIMESTAMP: - putLong(t, (long) v); - break; - case FLOAT: - putFloat(t, (float) v); - break; - case DOUBLE: - putDouble(t, (double) v); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - putBinary(t, (Binary) v); - break; - case VECTOR: - putVector(t, (TsPrimitiveType[]) v); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).put(this, t, v); } public int length() { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 3ebe4daf3..2761b1d87 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -73,6 +73,11 @@ public void init(BatchData batchData) { batchData.initIntValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putInt(timestamp, (int) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 1bd70bd60..9f2a5e991 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -72,6 +72,11 @@ public void init(BatchData batchData) { batchData.initLongValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putLong(timestamp, (long) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 61fbff51c..dddedf154 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -62,6 +62,11 @@ public void init(BatchData batchData) { batchData.initBinaryValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBinary(timestamp, (Binary) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index bb31c8338..3e0acbb08 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -66,6 +66,11 @@ public void init(BatchData batchData) { batchData.initBinaryValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBinary(timestamp, (Binary) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index d4806ce02..c4254ab70 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -69,6 +69,11 @@ public void init(BatchData batchData) { batchData.initBooleanValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBoolean(timestamp, (boolean) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 05809d031..8a69cd92a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -76,6 +76,11 @@ public void init(BatchData batchData) { batchData.initDoubleValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putDouble(timestamp, (double) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index ce6ecfa25..ff69b4366 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -75,6 +75,11 @@ public void init(BatchData batchData) { batchData.initFloatValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putFloat(timestamp, (float) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 155c41cd1..528900b98 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -66,6 +66,11 @@ public void init(BatchData batchData) { batchData.initBinaryValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBinary(timestamp, (Binary) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 8519908a7..43c2adce5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -108,6 +108,11 @@ default void init(BatchData batchData) { throw new UnSupportedDataTypeException(String.valueOf(batchData.getDataType())); } + /** Adds a timestamp-value pair to {@code batchData}. */ + default void put(BatchData batchData, long timestamp, Object value) { + throw new UnSupportedDataTypeException(String.valueOf(batchData.getDataType())); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index e14737894..1d7398a93 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -41,6 +41,11 @@ public void init(BatchData batchData) { batchData.initVectorValues(); } + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putVector(timestamp, (TsPrimitiveType[]) value); + } + @Override public Decoder getDecoder(TSEncoding encoding) { return switch (encoding) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 47dad3dbc..370d06fea 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -216,7 +216,7 @@ public void testUpdateStatistics() { public void testGetCurrentValueForVectorAndUnknown() { TsPrimitiveType[] vector = {new TsPrimitiveType.TsLong(1L)}; BatchData batchData = new BatchData(TSDataType.VECTOR); - batchData.putVector(1L, vector); + batchData.putAnObject(1L, vector); Assert.assertSame(vector, Type.fromTsDataType(TSDataType.VECTOR).getCurrentValue(batchData)); Assert.assertSame(vector, batchData.currentValue()); Assert.assertNull(Type.fromTsDataType(TSDataType.UNKNOWN).getCurrentValue(null)); From 27b8bbb7d677b405c6c90e53f496a73b7d08d18f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 14:46:31 +0800 Subject: [PATCH 028/131] refactor BatchData.serializeData --- .../apache/tsfile/read/common/BatchData.java | 94 +-------------- .../read/common/type/AbstractIntType.java | 13 ++ .../read/common/type/AbstractLongType.java | 13 ++ .../read/common/type/AbstractVarcharType.java | 17 +++ .../tsfile/read/common/type/BlobType.java | 17 +++ .../tsfile/read/common/type/BooleanType.java | 13 ++ .../tsfile/read/common/type/DoubleType.java | 13 ++ .../tsfile/read/common/type/FloatType.java | 13 ++ .../tsfile/read/common/type/ObjectType.java | 17 +++ .../apache/tsfile/read/common/type/Type.java | 13 ++ .../tsfile/read/common/type/VectorType.java | 28 +++++ .../tsfile/read/common/BatchDataTest.java | 111 ++++++++++++++++++ 12 files changed, 269 insertions(+), 93 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index 4e0bc430e..9dd8de27f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -602,99 +602,7 @@ public IBatchDataIterator getBatchDataIterator(int subIndex) { * writing, otherwise after deserializing the sequence will be reversed */ public void serializeData(DataOutputStream outputStream) throws IOException { - switch (dataType) { - case BOOLEAN: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeBoolean(getBooleanByIndex(i)); - } - break; - case DOUBLE: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeDouble(getDoubleByIndex(i)); - } - break; - case FLOAT: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeFloat(getFloatByIndex(i)); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - Binary binary = getBinaryByIndex(i); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeLong(getLongByIndex(i)); - } - break; - case INT32: - case DATE: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeInt(getIntByIndex(i)); - } - break; - case VECTOR: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - TsPrimitiveType[] values = getVectorByIndex(i); - outputStream.writeInt(values.length); - for (TsPrimitiveType value : values) { - if (value == null) { - outputStream.write(0); - } else { - outputStream.write(1); - outputStream.write(value.getDataType().serialize()); - switch (value.getDataType()) { - case BOOLEAN: - outputStream.writeBoolean(value.getBoolean()); - break; - case DOUBLE: - outputStream.writeDouble(value.getDouble()); - break; - case FLOAT: - outputStream.writeFloat(value.getFloat()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary binary = value.getBinary(); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - break; - case INT64: - case TIMESTAMP: - outputStream.writeLong(value.getLong()); - break; - case INT32: - case DATE: - outputStream.writeInt(value.getInt()); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.batch_data_unknown_type", dataType)); - } - } - } - } - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.batch_data_unknown_type", dataType)); - } + Type.fromTsDataType(dataType).serialize(this, outputStream); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 2761b1d87..8e9a3d490 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -78,6 +78,19 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putInt(timestamp, (int) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + outputStream.writeInt(batchData.getIntByIndex(i)); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeInt(value.getInt()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 9f2a5e991..ebc05adb5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -77,6 +77,19 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putLong(timestamp, (long) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + outputStream.writeLong(batchData.getLongByIndex(i)); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeLong(value.getLong()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index dddedf154..dc7ff2753 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -67,6 +67,23 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putBinary(timestamp, (Binary) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + Binary binary = batchData.getBinaryByIndex(i); + outputStream.writeInt(binary.getLength()); + outputStream.write(binary.getValues()); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + Binary binary = value.getBinary(); + stream.writeInt(binary.getLength()); + stream.write(binary.getValues()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 3e0acbb08..ec7ab940d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -71,6 +71,23 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putBinary(timestamp, (Binary) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + Binary binary = batchData.getBinaryByIndex(i); + outputStream.writeInt(binary.getLength()); + outputStream.write(binary.getValues()); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + Binary binary = value.getBinary(); + stream.writeInt(binary.getLength()); + stream.write(binary.getValues()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index c4254ab70..1a58cd253 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -74,6 +74,19 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putBoolean(timestamp, (boolean) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + outputStream.writeBoolean(batchData.getBooleanByIndex(i)); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeBoolean(value.getBoolean()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 8a69cd92a..ed1a42b11 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -81,6 +81,19 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putDouble(timestamp, (double) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + outputStream.writeDouble(batchData.getDoubleByIndex(i)); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeDouble(value.getDouble()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index ff69b4366..f91fe27f7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -80,6 +80,19 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putFloat(timestamp, (float) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + outputStream.writeFloat(batchData.getFloatByIndex(i)); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeFloat(value.getFloat()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 528900b98..e71528d36 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -71,6 +71,23 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putBinary(timestamp, (Binary) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + Binary binary = batchData.getBinaryByIndex(i); + outputStream.writeInt(binary.getLength()); + outputStream.write(binary.getValues()); + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + Binary binary = value.getBinary(); + stream.writeInt(binary.getLength()); + stream.write(binary.getValues()); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 43c2adce5..921000628 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -27,6 +27,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -113,6 +114,18 @@ default void put(BatchData batchData, long timestamp, Object value) { throw new UnSupportedDataTypeException(String.valueOf(batchData.getDataType())); } + /** Serializes all timestamp-value pairs in {@code batchData}. */ + default void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + throw new IllegalArgumentException( + Messages.format("error.read.batch_data_unknown_type", batchData.getDataType())); + } + + /** Serializes {@code value} to {@code stream}. */ + default void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + throw new IllegalArgumentException( + Messages.format("error.read.batch_data_unknown_type", value.getDataType())); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 1d7398a93..489566e40 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -27,9 +27,13 @@ import org.apache.tsfile.encoding.decoder.RegularDataDecoder; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; + public class VectorType extends AbstractLongType { public static final VectorType VECTOR = new VectorType(); @@ -46,6 +50,30 @@ public void put(BatchData batchData, long timestamp, Object value) { batchData.putVector(timestamp, (TsPrimitiveType[]) value); } + @Override + public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + for (int i = 0; i < batchData.length(); i++) { + outputStream.writeLong(batchData.getTimeByIndex(i)); + TsPrimitiveType[] values = batchData.getVectorByIndex(i); + outputStream.writeInt(values.length); + for (TsPrimitiveType value : values) { + if (value == null) { + outputStream.write(0); + continue; + } + outputStream.write(1); + outputStream.write(value.getDataType().serialize()); + Type.fromTsDataType(value.getDataType()).serialize(value, outputStream); + } + } + } + + @Override + public void serialize(TsPrimitiveType value, DataOutputStream stream) { + throw new IllegalArgumentException( + Messages.format("error.read.batch_data_unknown_type", value.getDataType())); + } + @Override public Decoder getDecoder(TSEncoding encoding) { return switch (encoding) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java index e388fa51e..055109f4d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java @@ -22,11 +22,19 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.TsPrimitiveType; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -81,4 +89,107 @@ public void testSignal() { } fail(); } + + @Test + public void testSerializeData() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260714}, + {TSDataType.INT64, 1L}, + {TSDataType.TIMESTAMP, 2L}, + {TSDataType.FLOAT, 1.0F}, + {TSDataType.DOUBLE, 2.0D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Object value = testCase[1]; + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(100L, value); + + try (DataInputStream inputStream = serialize(batchData)) { + assertEquals(100L, inputStream.readLong()); + assertSerializedValue(inputStream, dataType, value); + assertEquals(-1, inputStream.read()); + } + } + } + + @Test + public void testSerializeVectorData() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + TsPrimitiveType[] values = { + null, + new TsPrimitiveType.TsBoolean(true), + new TsPrimitiveType.TsInt(1), + new TsPrimitiveType.TsInt(20260714, TSDataType.DATE), + new TsPrimitiveType.TsLong(1L), + new TsPrimitiveType.TsFloat(1.0F), + new TsPrimitiveType.TsDouble(2.0D), + new TsPrimitiveType.TsBinary(binary) + }; + BatchData batchData = new BatchData(TSDataType.VECTOR); + batchData.putAnObject(100L, values); + + try (DataInputStream inputStream = serialize(batchData)) { + assertEquals(100L, inputStream.readLong()); + assertEquals(values.length, inputStream.readInt()); + for (TsPrimitiveType value : values) { + if (value == null) { + assertEquals(0, inputStream.readByte()); + continue; + } + assertEquals(1, inputStream.readByte()); + assertEquals(value.getDataType().serialize(), inputStream.readByte()); + assertSerializedValue(inputStream, value.getDataType(), value.getValue()); + } + assertEquals(-1, inputStream.read()); + } + } + + private DataInputStream serialize(BatchData batchData) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (DataOutputStream outputStream = new DataOutputStream(byteStream)) { + batchData.serializeData(outputStream); + } + return new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())); + } + + private void assertSerializedValue( + DataInputStream inputStream, TSDataType dataType, Object expectedValue) throws IOException { + switch (dataType) { + case BOOLEAN: + assertEquals(expectedValue, inputStream.readBoolean()); + break; + case INT32: + case DATE: + assertEquals(expectedValue, inputStream.readInt()); + break; + case INT64: + case TIMESTAMP: + assertEquals(expectedValue, inputStream.readLong()); + break; + case FLOAT: + assertEquals((float) expectedValue, inputStream.readFloat(), 0); + break; + case DOUBLE: + assertEquals((double) expectedValue, inputStream.readDouble(), 0); + break; + case TEXT: + case STRING: + case BLOB: + case OBJECT: + Binary binary = (Binary) expectedValue; + assertArrayEquals(binary.getValues(), inputStream.readNBytes(inputStream.readInt())); + break; + default: + fail("Unexpected data type: " + dataType); + } + } } From f7158769426ff54ff280143fe1732a3099efc55b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 14:50:11 +0800 Subject: [PATCH 029/131] refactor DescReadWriteBatchData.new --- .../read/common/DescReadWriteBatchData.java | 39 +------------------ 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java index b667c537d..15e31135c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -49,43 +50,7 @@ public DescReadWriteBatchData(TSDataType dataType) { timeRet.add(new long[capacity]); count = 0; - switch (dataType) { - case BOOLEAN: - booleanRet = new LinkedList<>(); - booleanRet.add(new boolean[capacity]); - break; - case INT32: - case DATE: - intRet = new LinkedList<>(); - intRet.add(new int[capacity]); - break; - case INT64: - case TIMESTAMP: - longRet = new LinkedList<>(); - longRet.add(new long[capacity]); - break; - case FLOAT: - floatRet = new LinkedList<>(); - floatRet.add(new float[capacity]); - break; - case DOUBLE: - doubleRet = new LinkedList<>(); - doubleRet.add(new double[capacity]); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - binaryRet = new LinkedList<>(); - binaryRet.add(new Binary[capacity]); - break; - case VECTOR: - vectorRet = new LinkedList<>(); - vectorRet.add(new TsPrimitiveType[capacity][]); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).init(this); } /** From d27a487cc419bfac63afaef245ce87e1fbbc3f45 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 15:00:15 +0800 Subject: [PATCH 030/131] refactor DescReadWriteBatchData.serializeData --- .../apache/tsfile/read/common/BatchData.java | 2 +- .../read/common/DescReadWriteBatchData.java | 92 +------------------ .../read/common/type/AbstractIntType.java | 8 +- .../read/common/type/AbstractLongType.java | 8 +- .../read/common/type/AbstractVarcharType.java | 8 +- .../tsfile/read/common/type/BlobType.java | 8 +- .../tsfile/read/common/type/BooleanType.java | 8 +- .../tsfile/read/common/type/DoubleType.java | 8 +- .../tsfile/read/common/type/FloatType.java | 8 +- .../tsfile/read/common/type/ObjectType.java | 8 +- .../apache/tsfile/read/common/type/Type.java | 3 +- .../tsfile/read/common/type/VectorType.java | 8 +- 12 files changed, 49 insertions(+), 120 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index 9dd8de27f..7633a62fd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -602,7 +602,7 @@ public IBatchDataIterator getBatchDataIterator(int subIndex) { * writing, otherwise after deserializing the sequence will be reversed */ public void serializeData(DataOutputStream outputStream) throws IOException { - Type.fromTsDataType(dataType).serialize(this, outputStream); + Type.fromTsDataType(dataType).serialize(this, outputStream, false); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java index 15e31135c..96abf1ce4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java @@ -384,97 +384,7 @@ public TsPrimitiveType[] getVectorByIndex(int idx) { @Override public void serializeData(DataOutputStream outputStream) throws IOException { - switch (dataType) { - case BOOLEAN: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeBoolean(getBooleanByIndex(i)); - } - break; - case DOUBLE: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeDouble(getDoubleByIndex(i)); - } - break; - case FLOAT: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeFloat(getFloatByIndex(i)); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - Binary binary = getBinaryByIndex(i); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - } - break; - case INT64: - case TIMESTAMP: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeLong(getLongByIndex(i)); - } - break; - case INT32: - case DATE: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeInt(getIntByIndex(i)); - } - break; - case VECTOR: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - TsPrimitiveType[] values = getVectorByIndex(i); - outputStream.writeInt(values.length); - for (TsPrimitiveType value : values) { - if (value == null) { - outputStream.write(0); - } else { - outputStream.write(1); - outputStream.write(value.getDataType().serialize()); - switch (value.getDataType()) { - case BOOLEAN: - outputStream.writeBoolean(value.getBoolean()); - break; - case DOUBLE: - outputStream.writeDouble(value.getDouble()); - break; - case FLOAT: - outputStream.writeFloat(value.getFloat()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary binary = value.getBinary(); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - break; - case INT64: - case TIMESTAMP: - outputStream.writeLong(value.getLong()); - break; - case INT32: - case DATE: - outputStream.writeInt(value.getInt()); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } - } - } - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).serialize(this, outputStream, true); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 8e9a3d490..d2c9dc5d5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -79,10 +79,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - outputStream.writeInt(batchData.getIntByIndex(i)); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeInt(batchData.getIntByIndex(index)); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index ebc05adb5..851b727f9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -78,10 +78,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - outputStream.writeLong(batchData.getLongByIndex(i)); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeLong(batchData.getLongByIndex(index)); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index dc7ff2753..b6ed50a0e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -68,10 +68,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - Binary binary = batchData.getBinaryByIndex(i); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + Binary binary = batchData.getBinaryByIndex(index); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index ec7ab940d..6ce7a3563 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -72,10 +72,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - Binary binary = batchData.getBinaryByIndex(i); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + Binary binary = batchData.getBinaryByIndex(index); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 1a58cd253..665270e5a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -75,10 +75,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - outputStream.writeBoolean(batchData.getBooleanByIndex(i)); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeBoolean(batchData.getBooleanByIndex(index)); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index ed1a42b11..3675444dd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -82,10 +82,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - outputStream.writeDouble(batchData.getDoubleByIndex(i)); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeDouble(batchData.getDoubleByIndex(index)); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index f91fe27f7..b29686bae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -81,10 +81,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - outputStream.writeFloat(batchData.getFloatByIndex(i)); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeFloat(batchData.getFloatByIndex(index)); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index e71528d36..b6c08d661 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -72,10 +72,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - Binary binary = batchData.getBinaryByIndex(i); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + Binary binary = batchData.getBinaryByIndex(index); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 921000628..de271806f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -115,7 +115,8 @@ default void put(BatchData batchData, long timestamp, Object value) { } /** Serializes all timestamp-value pairs in {@code batchData}. */ - default void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + default void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { throw new IllegalArgumentException( Messages.format("error.read.batch_data_unknown_type", batchData.getDataType())); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 489566e40..b07c2460c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -51,10 +51,12 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream) throws IOException { + public void serialize( + BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { - outputStream.writeLong(batchData.getTimeByIndex(i)); - TsPrimitiveType[] values = batchData.getVectorByIndex(i); + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + TsPrimitiveType[] values = batchData.getVectorByIndex(index); outputStream.writeInt(values.length); for (TsPrimitiveType value : values) { if (value == null) { From 789ea033281bab16b36d67af7928121cef9819b2 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 15:38:53 +0800 Subject: [PATCH 031/131] refactor Field.getStringValue --- .../read/common/DescReadWriteBatchData.java | 1 - .../org/apache/tsfile/read/common/Field.java | 31 +--------------- .../read/common/type/AbstractIntType.java | 10 ++++- .../read/common/type/AbstractLongType.java | 10 ++++- .../read/common/type/AbstractVarcharType.java | 10 ++++- .../tsfile/read/common/type/BlobType.java | 11 +++++- .../tsfile/read/common/type/BooleanType.java | 10 ++++- .../tsfile/read/common/type/DoubleType.java | 10 ++++- .../tsfile/read/common/type/FloatType.java | 10 ++++- .../tsfile/read/common/type/ObjectType.java | 12 +++++- .../apache/tsfile/read/common/type/Type.java | 10 ++++- .../tsfile/read/common/type/VectorType.java | 11 +++++- .../tsfile/read/common/type/TypeTest.java | 37 +++++++++++++++++++ 13 files changed, 123 insertions(+), 50 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java index 96abf1ce4..78ec4ed3c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java @@ -23,7 +23,6 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.DataOutputStream; import java.io.IOException; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java index 1e8a3afcc..c262c7b3f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java @@ -22,16 +22,14 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NullFieldException; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.time.LocalDate; -import static org.apache.tsfile.utils.BytesUtils.parseObjectByteArrayToString; - /** * Field is component of one {@code RowRecord} which stores a value in specific data type. The value * type of Field is primitive(int long, float, double, binary, boolean). @@ -147,9 +145,6 @@ public void setDoubleV(double doubleV) { public Binary getBinaryV() { if (dataType == null) { throw new NullFieldException(); - } else if (dataType == TSDataType.OBJECT) { - throw new UnsupportedOperationException( - Messages.get("error.read.field_object_type_no_binary")); } return binaryV; } @@ -174,29 +169,7 @@ public String getStringValue() { if (dataType == null) { return "null"; } - switch (dataType) { - case BOOLEAN: - return String.valueOf(boolV); - case INT32: - case DATE: - return String.valueOf(intV); - case INT64: - case TIMESTAMP: - return String.valueOf(longV); - case FLOAT: - return String.valueOf(floatV); - case DOUBLE: - return String.valueOf(doubleV); - case TEXT: - case STRING: - return binaryV.toString(); - case OBJECT: - return parseObjectByteArrayToString(binaryV.getValues()); - case BLOB: - return BytesUtils.parseBlobByteArrayToString(binaryV.getValues()); - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + return Type.fromTsDataType(dataType).toString(this); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index d2c9dc5d5..13d21e26f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -36,6 +36,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -79,8 +80,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -139,6 +140,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getInt(); } + @Override + public String toString(Field field) { + return String.valueOf(field.getIntV()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 851b727f9..59dccdeca 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -78,8 +79,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -138,6 +139,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getLong(); } + @Override + public String toString(Field field) { + return String.valueOf(field.getLongV()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index b6ed50a0e..c5a7ae821 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -27,6 +27,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -68,8 +69,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -116,6 +117,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getBinary(); } + @Override + public String toString(Field field) { + return field.getBinaryV().toString(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 6ce7a3563..a4cb8221a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -27,9 +27,11 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -72,8 +74,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -120,6 +122,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getBinary(); } + @Override + public String toString(Field field) { + return BytesUtils.parseBlobByteArrayToString(field.getBinaryV().getValues()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 665270e5a..86fc5d018 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -28,6 +28,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -75,8 +76,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -128,6 +129,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getBoolean(); } + @Override + public String toString(Field field) { + return String.valueOf(field.getBoolV()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 3675444dd..8b61811b2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -82,8 +83,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -141,6 +142,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getDouble(); } + @Override + public String toString(Field field) { + return String.valueOf(field.getDoubleV()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index b29686bae..8d9cd5128 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -34,6 +34,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; import org.apache.tsfile.utils.BytesUtils; @@ -81,8 +82,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -139,6 +140,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getFloat(); } + @Override + public String toString(Field field) { + return String.valueOf(field.getFloatV()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index b6c08d661..e3da6d29f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -27,6 +27,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; @@ -40,6 +41,8 @@ import java.util.List; import java.util.Optional; +import static org.apache.tsfile.utils.BytesUtils.parseObjectByteArrayToString; + public class ObjectType extends AbstractType { public static final ObjectType OBJECT = new ObjectType(); @@ -72,8 +75,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -120,6 +123,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getBinary(); } + @Override + public String toString(Field field) { + return parseObjectByteArrayToString(field.getBinaryV().getValues()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index de271806f..c11c08a97 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -29,6 +29,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -115,8 +116,8 @@ default void put(BatchData batchData, long timestamp, Object value) { } /** Serializes all timestamp-value pairs in {@code batchData}. */ - default void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + default void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { throw new IllegalArgumentException( Messages.format("error.read.batch_data_unknown_type", batchData.getDataType())); } @@ -243,6 +244,11 @@ default Object getCurrentValue(BatchData batchData) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the string representation of the value in {@code field}. */ + default String toString(Field field) { + throw new UnSupportedDataTypeException(field.getDataType().toString()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index b07c2460c..820abd9b6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -29,7 +29,9 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.DataOutputStream; import java.io.IOException; @@ -51,8 +53,8 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize( - BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); @@ -109,6 +111,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getVector(); } + @Override + public String toString(Field field) { + throw new UnSupportedDataTypeException(field.getDataType().toString()); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.VECTOR; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 370d06fea..799f931e5 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -24,9 +24,11 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; import org.junit.Assert; import org.junit.Test; @@ -111,6 +113,41 @@ public void testCalcTypeSize() { } } + @Test + public void testToString() { + Object[][] testCases = { + {TSDataType.BOOLEAN, true, "true"}, + {TSDataType.INT32, 1, "1"}, + {TSDataType.DATE, 20260714, "20260714"}, + {TSDataType.INT64, 2L, "2"}, + {TSDataType.TIMESTAMP, 3L, "3"}, + {TSDataType.FLOAT, 1.25F, "1.25"}, + {TSDataType.DOUBLE, 2.5D, "2.5"}, + {TSDataType.TEXT, new Binary("text", StandardCharsets.UTF_8), "text"}, + {TSDataType.STRING, new Binary("string", StandardCharsets.UTF_8), "string"}, + {TSDataType.BLOB, new Binary(new byte[] {0x01, 0x23, (byte) 0xFF}), "0x0123ff"}, + {TSDataType.OBJECT, new Binary(BytesUtils.longToBytes(1L)), "(Object) 1 B"} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Field field = Field.getField(testCase[1], dataType); + Assert.assertEquals(testCase[2], Type.fromTsDataType(dataType).toString(field)); + Assert.assertEquals(testCase[2], field.getStringValue()); + } + + Assert.assertEquals("null", new Field(null).getStringValue()); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + new Field(dataType).getStringValue(); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + @Test public void testCalcTypeSizeFromObject() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From 670514d47003c273640b06aa05ab3bd88bdfd6b0 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 15:45:02 +0800 Subject: [PATCH 032/131] add Type.estimateArraySize --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DateType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 31 +++++++++++++++++++ 11 files changed, 81 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 13d21e26f..58557fdf6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -206,6 +206,11 @@ public Object createArray(int capacity) { return new int[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfIntArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Integer.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 59dccdeca..520811a49 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -205,6 +205,11 @@ public Object createArray(int capacity) { return new long[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfLongArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Long.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index c5a7ae821..391382a7d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -169,6 +169,11 @@ public Object createArray(int capacity) { return new Binary[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return serializedSizeOfBinaryValues(column, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index a4cb8221a..ce84391ce 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -172,6 +172,11 @@ public Object createArray(int capacity) { return new Binary[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return serializedSizeOfBinaryValues(column, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 86fc5d018..a5809dd36 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -165,6 +165,11 @@ public Object createArray(int capacity) { return new boolean[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfBooleanArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return rowSize; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index a849ccbb4..4bbbecdd5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -62,6 +62,11 @@ public Object createArray(int capacity) { return new LocalDate[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 8b61811b2..31bfd9299 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -208,6 +208,11 @@ public Object createArray(int capacity) { return new double[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfDoubleArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Double.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 8d9cd5128..3631e3135 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -206,6 +206,11 @@ public Object createArray(int capacity) { return new float[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfFloatArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Float.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index e3da6d29f..dd160e8ba 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -173,6 +173,11 @@ public Object createArray(int capacity) { return new Binary[capacity]; } + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + @Override public int serializedSize(Object column, int rowSize) { return serializedSizeOfBinaryValues(column, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index c11c08a97..acaa755a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -173,6 +173,11 @@ default Object createArray(int capacity) { throw new UnsupportedOperationException(getClass().getName()); } + /** Estimates the retained memory size of an array with {@code size} entries. */ + default long estimateArraySize(int size) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Returns the serialized byte size of the array column with {@code rowSize} entries. */ default int serializedSize(Object column, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 799f931e5..ae749a584 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -173,6 +173,37 @@ public void testCalcTypeSizeFromObject() { } } + @Test + public void testEstimateArraySize() { + int size = 10; + Object[][] testCases = { + {TSDataType.BOOLEAN, RamUsageEstimator.sizeOfBooleanArray(size)}, + {TSDataType.INT32, RamUsageEstimator.sizeOfIntArray(size)}, + {TSDataType.DATE, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.INT64, RamUsageEstimator.sizeOfLongArray(size)}, + {TSDataType.TIMESTAMP, RamUsageEstimator.sizeOfLongArray(size)}, + {TSDataType.FLOAT, RamUsageEstimator.sizeOfFloatArray(size)}, + {TSDataType.DOUBLE, RamUsageEstimator.sizeOfDoubleArray(size)}, + {TSDataType.TEXT, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.STRING, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.BLOB, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.OBJECT, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.VECTOR, RamUsageEstimator.sizeOfLongArray(size)} + }; + + for (Object[] testCase : testCases) { + Assert.assertEquals( + testCase[1], Type.fromTsDataType((TSDataType) testCase[0]).estimateArraySize(size)); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).estimateArraySize(size); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + @Test public void testGetOneItemMaxSize() { Object[][] testCases = { From 65cc017dab2a918eccab3c75c5c26b45afbb563e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 15:49:10 +0800 Subject: [PATCH 033/131] add Type.estimateArraySize --- .../test/java/org/apache/tsfile/read/common/type/TypeTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index ae749a584..620285af7 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -27,6 +27,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; From 1b572576d6687ad15c1bcc36f7a5d8e62458a9c7 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 15:57:31 +0800 Subject: [PATCH 034/131] add Type.estimateValueSize --- .../read/common/type/AbstractIntType.java | 5 ++++ .../read/common/type/AbstractLongType.java | 5 ++++ .../read/common/type/AbstractVarcharType.java | 5 ++++ .../tsfile/read/common/type/BlobType.java | 5 ++++ .../tsfile/read/common/type/BooleanType.java | 5 ++++ .../tsfile/read/common/type/DoubleType.java | 5 ++++ .../tsfile/read/common/type/FloatType.java | 5 ++++ .../tsfile/read/common/type/ObjectType.java | 5 ++++ .../apache/tsfile/read/common/type/Type.java | 5 ++++ .../tsfile/read/common/type/VectorType.java | 6 ++++ .../tsfile/read/common/type/TypeTest.java | 30 +++++++++++++++++++ 11 files changed, 81 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 58557fdf6..fa515dab7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -211,6 +211,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfIntArray(size); } + @Override + public long estimateValueSize() { + return Integer.BYTES; + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Integer.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 520811a49..9127e141d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -210,6 +210,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfLongArray(size); } + @Override + public long estimateValueSize() { + return Long.BYTES; + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Long.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 391382a7d..7e2614b2b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -174,6 +174,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfObjectArray(size); } + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + @Override public int serializedSize(Object column, int rowSize) { return serializedSizeOfBinaryValues(column, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index ce84391ce..99558ac32 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -177,6 +177,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfObjectArray(size); } + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + @Override public int serializedSize(Object column, int rowSize) { return serializedSizeOfBinaryValues(column, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index a5809dd36..3a5958753 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -170,6 +170,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfBooleanArray(size); } + @Override + public long estimateValueSize() { + return Byte.BYTES; + } + @Override public int serializedSize(Object column, int rowSize) { return rowSize; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 31bfd9299..9b5a8bbdd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -213,6 +213,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfDoubleArray(size); } + @Override + public long estimateValueSize() { + return Double.BYTES; + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Double.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 3631e3135..3c205e095 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -211,6 +211,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfFloatArray(size); } + @Override + public long estimateValueSize() { + return Float.BYTES; + } + @Override public int serializedSize(Object column, int rowSize) { return Math.multiplyExact(Float.BYTES, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index dd160e8ba..fb58ab96c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -178,6 +178,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfObjectArray(size); } + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + @Override public int serializedSize(Object column, int rowSize) { return serializedSizeOfBinaryValues(column, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index acaa755a3..9ffaebf03 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -178,6 +178,11 @@ default long estimateArraySize(int size) { throw new UnsupportedOperationException(getClass().getName()); } + /** Estimates the retained memory size of a value of this type. */ + default long estimateValueSize() { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Returns the serialized byte size of the array column with {@code rowSize} entries. */ default int serializedSize(Object column, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 820abd9b6..862a8cb87 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -30,6 +30,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -116,6 +117,11 @@ public String toString(Field field) { throw new UnSupportedDataTypeException(field.getDataType().toString()); } + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.VECTOR; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 620285af7..8a34f33cb 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -205,6 +205,36 @@ public void testEstimateArraySize() { } } + @Test + public void testEstimateValueSize() { + Object[][] testCases = { + {TSDataType.BOOLEAN, (long) Byte.BYTES}, + {TSDataType.INT32, (long) Integer.BYTES}, + {TSDataType.DATE, (long) Integer.BYTES}, + {TSDataType.INT64, (long) Long.BYTES}, + {TSDataType.TIMESTAMP, (long) Long.BYTES}, + {TSDataType.FLOAT, (long) Float.BYTES}, + {TSDataType.DOUBLE, (long) Double.BYTES}, + {TSDataType.TEXT, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.STRING, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.BLOB, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.OBJECT, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.VECTOR, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF} + }; + + for (Object[] testCase : testCases) { + Assert.assertEquals( + testCase[1], Type.fromTsDataType((TSDataType) testCase[0]).estimateValueSize()); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).estimateValueSize(); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + @Test public void testGetOneItemMaxSize() { Object[][] testCases = { From 3487c73129d97dda7cc5709015eda7d3b80dc38f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 16:38:07 +0800 Subject: [PATCH 035/131] add Field.getObjectValue --- .../org/apache/tsfile/read/common/Field.java | 24 +--------- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DateType.java | 6 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/VectorType.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 45 +++++++++++++++++++ 13 files changed, 102 insertions(+), 23 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java index c262c7b3f..d5500511b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java @@ -181,29 +181,7 @@ public Object getObjectValue(TSDataType dataType) { if (this.dataType == null) { return null; } - switch (dataType) { - case DOUBLE: - return getDoubleV(); - case FLOAT: - return getFloatV(); - case INT64: - case TIMESTAMP: - return getLongV(); - case INT32: - return getIntV(); - case DATE: - return getDateV(); - case BOOLEAN: - return getBoolV(); - case TEXT: - case BLOB: - case STRING: - return getBinaryV(); - case OBJECT: - return getStringValue(); - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + return Type.fromTsDataType(dataType).getValue(this); } public static Field getField(Object value, TSDataType dataType) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index fa515dab7..b3aa127a8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -145,6 +145,11 @@ public String toString(Field field) { return String.valueOf(field.getIntV()); } + @Override + public Object getValue(Field field) { + return field.getIntV(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 9127e141d..abc30eab2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -144,6 +144,11 @@ public String toString(Field field) { return String.valueOf(field.getLongV()); } + @Override + public Object getValue(Field field) { + return field.getLongV(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 7e2614b2b..798a96505 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -122,6 +122,11 @@ public String toString(Field field) { return field.getBinaryV().toString(); } + @Override + public Object getValue(Field field) { + return field.getBinaryV(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 99558ac32..57a3fdbcc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -127,6 +127,11 @@ public String toString(Field field) { return BytesUtils.parseBlobByteArrayToString(field.getBinaryV().getValues()); } + @Override + public Object getValue(Field field) { + return field.getBinaryV(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 3a5958753..9fae87035 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -134,6 +134,11 @@ public String toString(Field field) { return String.valueOf(field.getBoolV()); } + @Override + public Object getValue(Field field) { + return field.getBoolV(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 4bbbecdd5..51dc4104c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -67,6 +68,11 @@ public long estimateArraySize(int size) { return RamUsageEstimator.sizeOfObjectArray(size); } + @Override + public Object getValue(Field field) { + return field.getDateV(); + } + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 9b5a8bbdd..2c7354500 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -147,6 +147,11 @@ public String toString(Field field) { return String.valueOf(field.getDoubleV()); } + @Override + public Object getValue(Field field) { + return field.getDoubleV(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 3c205e095..7e3d8df5a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -145,6 +145,11 @@ public String toString(Field field) { return String.valueOf(field.getFloatV()); } + @Override + public Object getValue(Field field) { + return field.getFloatV(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index fb58ab96c..86265c972 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -128,6 +128,11 @@ public String toString(Field field) { return parseObjectByteArrayToString(field.getBinaryV().getValues()); } + @Override + public Object getValue(Field field) { + return field.getStringValue(); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 9ffaebf03..98b5000dc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -209,6 +209,11 @@ default Object getValue(Object column, int rowIndex) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the value in {@code field}. */ + default Object getValue(Field field) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 862a8cb87..6f7e8b0c1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -117,6 +117,11 @@ public String toString(Field field) { throw new UnSupportedDataTypeException(field.getDataType().toString()); } + @Override + public Object getValue(Field field) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + @Override public long estimateValueSize() { return RamUsageEstimator.NUM_BYTES_OBJECT_REF; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 8a34f33cb..215a9050d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -35,6 +35,7 @@ import org.junit.Test; import java.nio.charset.StandardCharsets; +import java.time.LocalDate; import java.util.Arrays; public class TypeTest { @@ -149,6 +150,50 @@ public void testToString() { } } + @Test + public void testGetValueFromField() { + Object[][] testCases = { + {TSDataType.BOOLEAN, true, true}, + {TSDataType.INT32, 1, 1}, + {TSDataType.DATE, 20260714, LocalDate.of(2026, 7, 14)}, + {TSDataType.INT64, 2L, 2L}, + {TSDataType.TIMESTAMP, 3L, 3L}, + {TSDataType.FLOAT, 1.25F, 1.25F}, + {TSDataType.DOUBLE, 2.5D, 2.5D}, + { + TSDataType.TEXT, + new Binary("text", StandardCharsets.UTF_8), + new Binary("text", StandardCharsets.UTF_8) + }, + { + TSDataType.STRING, + new Binary("string", StandardCharsets.UTF_8), + new Binary("string", StandardCharsets.UTF_8) + }, + {TSDataType.BLOB, new Binary(new byte[] {0x01, 0x23}), new Binary(new byte[] {0x01, 0x23})}, + {TSDataType.OBJECT, new Binary(BytesUtils.longToBytes(1L)), "(Object) 1 B"} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Field field = Field.getField(testCase[1], dataType); + Assert.assertEquals(testCase[2], Type.fromTsDataType(dataType).getValue(field)); + Assert.assertEquals(testCase[2], field.getObjectValue(dataType)); + } + + Assert.assertNull(new Field(null).getObjectValue(TSDataType.INT32)); + Assert.assertEquals("1", Field.getField(1, TSDataType.INT32).getObjectValue(TSDataType.OBJECT)); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + new Field(dataType).getObjectValue(dataType); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + @Test public void testCalcTypeSizeFromObject() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From e74c2ce2fb68f75135dabdc13a47e880e661fd09 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 17:05:58 +0800 Subject: [PATCH 036/131] add Field.getField --- .../org/apache/tsfile/read/common/Field.java | 30 +------------------ .../tsfile/read/common/type/BinaryType.java | 11 +++++++ .../tsfile/read/common/type/BlobType.java | 8 +++++ .../tsfile/read/common/type/BooleanType.java | 8 +++++ .../tsfile/read/common/type/DateType.java | 7 +++++ .../tsfile/read/common/type/DoubleType.java | 7 +++++ .../tsfile/read/common/type/FloatType.java | 7 +++++ .../tsfile/read/common/type/IntType.java | 10 +++++++ .../tsfile/read/common/type/LongType.java | 10 +++++++ .../tsfile/read/common/type/ObjectType.java | 8 +++++ .../tsfile/read/common/type/StringType.java | 11 +++++++ .../read/common/type/TimestampType.java | 10 +++++++ .../apache/tsfile/read/common/type/Type.java | 5 ++++ .../tsfile/read/common/type/VectorType.java | 5 ++++ .../tsfile/read/common/type/TypeTest.java | 16 +++++++++- 15 files changed, 123 insertions(+), 30 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java index d5500511b..a86faeea5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java @@ -188,35 +188,7 @@ public static Field getField(Object value, TSDataType dataType) { if (value == null) { return null; } - Field field = new Field(dataType); - switch (dataType) { - case INT32: - case DATE: - field.setIntV((int) value); - break; - case INT64: - case TIMESTAMP: - field.setLongV((long) value); - break; - case FLOAT: - field.setFloatV((float) value); - break; - case DOUBLE: - field.setDoubleV((double) value); - break; - case BOOLEAN: - field.setBoolV((boolean) value); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - field.setBinaryV((Binary) value); - break; - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } - return field; + return Type.fromTsDataType(dataType).getField(value); } public static void setTsPrimitiveValue(TsPrimitiveType value, Field field) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java index 68c874998..ece9ba52b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java @@ -19,11 +19,22 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.utils.Binary; + public class BinaryType extends AbstractVarcharType { public static final BinaryType TEXT = new BinaryType(); private BinaryType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.TEXT); + field.setBinaryV((Binary) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.TEXT; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 57a3fdbcc..16421658e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -132,6 +133,13 @@ public Object getValue(Field field) { return field.getBinaryV(); } + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.BLOB); + field.setBinaryV((Binary) value); + return field; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 9fae87035..8d88bb502 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -24,6 +24,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.encoding.decoder.IntRleDecoder; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -139,6 +140,13 @@ public Object getValue(Field field) { return field.getBoolV(); } + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.BOOLEAN); + field.setBoolV((boolean) value); + return field; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 51dc4104c..613a8abfa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -73,6 +73,13 @@ public Object getValue(Field field) { return field.getDateV(); } + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.DATE); + field.setIntV((int) value); + return field; + } + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 2c7354500..f77c160bb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -152,6 +152,13 @@ public Object getValue(Field field) { return field.getDoubleV(); } + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.DOUBLE); + field.setDoubleV((double) value); + return field; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 7e3d8df5a..c39de48a6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -150,6 +150,13 @@ public Object getValue(Field field) { return field.getFloatV(); } + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.FLOAT); + field.setFloatV((float) value); + return field; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java index e4d36c2a3..32e958193 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java @@ -19,12 +19,22 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; + public class IntType extends AbstractIntType { public static final IntType INT32 = new IntType(); private IntType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.INT32); + field.setIntV((int) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.INT32; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java index a1ed2a1c9..77c3d0520 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java @@ -19,12 +19,22 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; + public class LongType extends AbstractLongType { public static final LongType INT64 = new LongType(); private LongType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.INT64); + field.setLongV((long) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.INT64; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 86265c972..990111917 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -133,6 +134,13 @@ public Object getValue(Field field) { return field.getStringValue(); } + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.OBJECT); + field.setBinaryV((Binary) value); + return field; + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java index 725ae9e3b..58dac7f22 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java @@ -19,12 +19,23 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.utils.Binary; + public class StringType extends AbstractVarcharType { public static final StringType STRING = new StringType(); private StringType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.STRING); + field.setBinaryV((Binary) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.STRING; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java index c4e5e8434..210058970 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java @@ -19,12 +19,22 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; + public class TimestampType extends AbstractLongType { public static final TimestampType TIMESTAMP = new TimestampType(); private TimestampType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.TIMESTAMP); + field.setLongV((long) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.TIMESTAMP; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 98b5000dc..71abf56b6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -214,6 +214,11 @@ default Object getValue(Field field) { throw new UnSupportedDataTypeException(getTypeEnum().toString()); } + /** Creates a field initialized with {@code value}. */ + default Field getField(Object value) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 6f7e8b0c1..af86ce537 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -122,6 +122,11 @@ public Object getValue(Field field) { throw new UnSupportedDataTypeException(getTypeEnum().toString()); } + @Override + public Field getField(Object value) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + @Override public long estimateValueSize() { return RamUsageEstimator.NUM_BYTES_OBJECT_REF; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 215a9050d..bb274e95a 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -176,11 +176,18 @@ public void testGetValueFromField() { for (Object[] testCase : testCases) { TSDataType dataType = (TSDataType) testCase[0]; + Type type = Type.fromTsDataType(dataType); + Field typeField = type.getField(testCase[1]); + Assert.assertEquals(dataType, typeField.getDataType()); + Assert.assertEquals(testCase[2], type.getValue(typeField)); + Field field = Field.getField(testCase[1], dataType); - Assert.assertEquals(testCase[2], Type.fromTsDataType(dataType).getValue(field)); + Assert.assertEquals(dataType, field.getDataType()); + Assert.assertEquals(testCase[2], type.getValue(field)); Assert.assertEquals(testCase[2], field.getObjectValue(dataType)); } + Assert.assertNull(Field.getField(null, TSDataType.INT32)); Assert.assertNull(new Field(null).getObjectValue(TSDataType.INT32)); Assert.assertEquals("1", Field.getField(1, TSDataType.INT32).getObjectValue(TSDataType.OBJECT)); @@ -191,6 +198,13 @@ public void testGetValueFromField() { } catch (UnSupportedDataTypeException ignored) { // Expected. } + + try { + Type.fromTsDataType(dataType).getField(new Object()); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } } } From 83672066aa1d05137a51519b565171c27b74b060 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 17:30:36 +0800 Subject: [PATCH 037/131] add Field.setTsPrimitiveValue --- .../org/apache/tsfile/read/common/Field.java | 30 +------- .../read/common/type/AbstractIntType.java | 12 +++- .../read/common/type/AbstractLongType.java | 12 +++- .../read/common/type/AbstractVarcharType.java | 12 +++- .../tsfile/read/common/type/BlobType.java | 12 +++- .../tsfile/read/common/type/BooleanType.java | 12 +++- .../tsfile/read/common/type/DateType.java | 7 +- .../tsfile/read/common/type/DoubleType.java | 12 +++- .../tsfile/read/common/type/FloatType.java | 12 +++- .../tsfile/read/common/type/ObjectType.java | 12 +++- .../apache/tsfile/read/common/type/Type.java | 13 +++- .../tsfile/read/common/type/VectorType.java | 6 ++ .../tsfile/read/common/type/TypeTest.java | 70 +++++++++++++++++++ 13 files changed, 183 insertions(+), 39 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java index a86faeea5..33980c6f3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java @@ -21,7 +21,6 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NullFieldException; -import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; @@ -192,33 +191,6 @@ public static Field getField(Object value, TSDataType dataType) { } public static void setTsPrimitiveValue(TsPrimitiveType value, Field field) { - switch (value.getDataType()) { - case BOOLEAN: - field.setBoolV(value.getBoolean()); - break; - case INT32: - case DATE: - field.setIntV(value.getInt()); - break; - case INT64: - case TIMESTAMP: - field.setLongV(value.getLong()); - break; - case FLOAT: - field.setFloatV(value.getFloat()); - break; - case DOUBLE: - field.setDoubleV(value.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - field.setBinaryV(value.getBinary()); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.common.unsupported_data_type", value.getDataType())); - } + Type.fromTsDataType(value.getDataType()).setTo(value, field); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index b3aa127a8..cdfa3730f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -150,6 +150,11 @@ public Object getValue(Field field) { return field.getIntV(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setIntV(from.getInt()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); @@ -256,7 +261,7 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new IntColumn( positionCount, Optional.of(new boolean[positionCount]), @@ -264,6 +269,11 @@ public Column createColumn(int positionCount) { TSDataType.INT32); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new IntColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index abc30eab2..6db132561 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -149,6 +149,11 @@ public Object getValue(Field field) { return field.getLongV(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setLongV(from.getLong()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); @@ -255,11 +260,16 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new LongColumn( positionCount, Optional.of(new boolean[positionCount]), new long[positionCount]); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new LongColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 798a96505..fb383ff9c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -127,6 +127,11 @@ public Object getValue(Field field) { return field.getBinaryV(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBinaryV(from.getBinary()); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); @@ -211,11 +216,16 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new BinaryColumn( positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BinaryColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 16421658e..b615ea54d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -133,6 +133,11 @@ public Object getValue(Field field) { return field.getBinaryV(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBinaryV(from.getBinary()); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BLOB); @@ -222,11 +227,16 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new BinaryColumn( positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BinaryColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 8d88bb502..d6b42991f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -140,6 +140,11 @@ public Object getValue(Field field) { return field.getBoolV(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBoolV(from.getBoolean()); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BOOLEAN); @@ -223,11 +228,16 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new BooleanColumn( positionCount, Optional.of(new boolean[positionCount]), new boolean[positionCount]); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BooleanColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 613a8abfa..946f39357 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -114,7 +114,7 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new IntColumn( positionCount, Optional.of(new boolean[positionCount]), @@ -122,6 +122,11 @@ public Column createColumn(int positionCount) { TSDataType.DATE); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new IntColumn(positionCount, TSDataType.DATE); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index f77c160bb..2153f65c3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -152,6 +152,11 @@ public Object getValue(Field field) { return field.getDoubleV(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setDoubleV(from.getDouble()); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.DOUBLE); @@ -265,11 +270,16 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new DoubleColumn( positionCount, Optional.of(new boolean[positionCount]), new double[positionCount]); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new DoubleColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index c39de48a6..e6252602d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -150,6 +150,11 @@ public Object getValue(Field field) { return field.getFloatV(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setFloatV(from.getFloat()); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.FLOAT); @@ -263,11 +268,16 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new FloatColumn( positionCount, Optional.of(new boolean[positionCount]), new float[positionCount]); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new FloatColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 990111917..7c68bd3a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -134,6 +134,11 @@ public Object getValue(Field field) { return field.getStringValue(); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBinaryV(from.getBinary()); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.OBJECT); @@ -223,11 +228,16 @@ public long arrayRamBytesUsed(Object array) { } @Override - public Column createColumn(int positionCount) { + public Column createColumnWithMaxPosition(int positionCount) { return new BinaryColumn( positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); } + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BinaryColumn(positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 71abf56b6..4ce1dbbd7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -219,13 +219,24 @@ default Field getField(Object value) { throw new UnSupportedDataTypeException(getTypeEnum().toString()); } + /** Copies the value from {@code from} to {@code to}. */ + default void setTo(TsPrimitiveType from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); } /** Creates a column with {@code positionCount} capacity for this type. */ - default Column createColumn(int positionCount) { + default Column createColumnWithMaxPosition(int positionCount) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Creates a column with {@code positionCount} capacity for this type. */ + default Column createColumnWithZeroPosition(int positionCount) { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index af86ce537..8ae6f80d3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -127,6 +127,12 @@ public Field getField(Object value) { throw new UnSupportedDataTypeException(getTypeEnum().toString()); } + @Override + public void setTo(TsPrimitiveType from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + @Override public long estimateValueSize() { return RamUsageEstimator.NUM_BYTES_OBJECT_REF; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index bb274e95a..304996bba 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -208,6 +208,76 @@ public void testGetValueFromField() { } } + @Test + public void testSetToField() { + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true), true}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1), 1}, + { + TSDataType.DATE, + new TsPrimitiveType.TsInt(20260714, TSDataType.DATE), + LocalDate.of(2026, 7, 14) + }, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L), 2L}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L), 3L}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F), 1.25F}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D), 2.5D}, + { + TSDataType.TEXT, + new TsPrimitiveType.TsBinary(new Binary("text", StandardCharsets.UTF_8)), + new Binary("text", StandardCharsets.UTF_8) + }, + { + TSDataType.STRING, + new TsPrimitiveType.TsBinary(new Binary("string", StandardCharsets.UTF_8)), + new Binary("string", StandardCharsets.UTF_8) + }, + { + TSDataType.BLOB, + new TsPrimitiveType.TsBinary(new Binary(new byte[] {0x01, 0x23})), + new Binary(new byte[] {0x01, 0x23}) + }, + { + TSDataType.OBJECT, + new TsPrimitiveType.TsBinary(new Binary(BytesUtils.longToBytes(1L))), + "(Object) 1 B" + } + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + TsPrimitiveType from = (TsPrimitiveType) testCase[1]; + Type type = Type.fromTsDataType(dataType); + + Field directTarget = new Field(dataType); + type.setTo(from, directTarget); + Assert.assertEquals(testCase[2], type.getValue(directTarget)); + + if (from.getDataType() == dataType) { + Field factoryTarget = new Field(dataType); + Field.setTsPrimitiveValue(from, factoryTarget); + Assert.assertEquals(testCase[2], type.getValue(factoryTarget)); + } + } + + TsPrimitiveType vector = new TsPrimitiveType.TsVector(new TsPrimitiveType[0]); + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).setTo(vector, new Field(dataType)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + try { + Field.setTsPrimitiveValue(vector, new Field(TSDataType.VECTOR)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + @Test public void testCalcTypeSizeFromObject() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From 560f0821c129e784a56eef618f81062b5f82867a Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 18:02:40 +0800 Subject: [PATCH 038/131] add Type.deserializeColumn --- .../common/block/column/BinaryColumn.java | 2 +- .../common/block/column/ColumnFactory.java | 27 +++---------------- .../read/common/block/column/IntColumn.java | 5 ++-- .../read/common/block/column/LongColumn.java | 2 +- .../read/common/type/AbstractIntType.java | 17 ++++++++++++ .../read/common/type/AbstractLongType.java | 17 ++++++++++++ .../tsfile/read/common/type/AbstractType.java | 18 ++++++++++++- .../tsfile/read/common/type/BlobType.java | 6 +++++ .../tsfile/read/common/type/DateType.java | 10 +++++-- .../apache/tsfile/read/common/type/Type.java | 5 ++++ 10 files changed, 78 insertions(+), 31 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index 5208d29b6..b2b3285e5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -60,7 +60,7 @@ public BinaryColumn(int positionCount, Optional valueIsNull, Binary[] this(0, positionCount, valueIsNull.orElse(null), values); } - BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) { + public BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java index ed5b853ff..0f73c9b0e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java @@ -21,32 +21,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; public class ColumnFactory { public static Column create(TSDataType dataType, int initialCapacity) { - switch (dataType) { - case INT64: - case TIMESTAMP: - return new LongColumn(initialCapacity); - case DOUBLE: - return new DoubleColumn(initialCapacity); - case FLOAT: - return new FloatColumn(initialCapacity); - case TEXT: - case STRING: - case BLOB: - case OBJECT: - return new BinaryColumn(initialCapacity); - case INT32: - return new IntColumn(initialCapacity, TSDataType.INT32); - case DATE: - return new IntColumn(initialCapacity, TSDataType.DATE); - case BOOLEAN: - return new BooleanColumn(initialCapacity); - default: - throw new IllegalArgumentException( - Messages.format("error.read.column_factory_unsupported_type", dataType)); - } + Type type = Type.fromTsDataType(dataType); + return type.createColumnWithZeroPosition(initialCapacity); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index b54d1575b..de05f1f4d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -72,7 +72,7 @@ public IntColumn( this.dataType = dataType; } - IntColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, int[] values) { + public IntColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, int[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } @@ -111,8 +111,9 @@ public TSDataType getDataType() { return dataType; } - public void modifyDataType(TSDataType dataType) { + public IntColumn modifyDataType(TSDataType dataType) { this.dataType = dataType; + return this; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 49d8183db..8edd31d5a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -59,7 +59,7 @@ public LongColumn(int positionCount, Optional valueIsNull, long[] val this(0, positionCount, valueIsNull.orElse(null), values); } - LongColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, long[] values) { + public LongColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, long[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index cdfa3730f..46443a47f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -249,6 +249,23 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + int[] intValues = new int[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + intValues[i] = buffer.getInt(); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + intValues[i] = buffer.getInt(); + } + } + } + return new IntColumn(0, rowSize, nullIndicators, intValues); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return hasEnoughLength(left, right, rowSize) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 6db132561..942871f7c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -248,6 +248,23 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + long[] intValues = new long[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + intValues[i] = buffer.getLong(); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + intValues[i] = buffer.getLong(); + } + } + } + return new LongColumn(0, rowSize, nullIndicators, intValues); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return hasEnoughLength(left, right, rowSize) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 82ab72d7f..d9d429c5b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -102,7 +102,7 @@ protected void serializeBinaryValues(Object array, int rowSize, DataOutputStream } } - protected Object deserializeBinaryValues(ByteBuffer buffer, int rowSize) { + protected Binary[] deserializeBinaryValues(ByteBuffer buffer, int rowSize) { Binary[] values = new Binary[rowSize]; for (int i = 0; i < rowSize; i++) { boolean isNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); @@ -111,6 +111,22 @@ protected Object deserializeBinaryValues(ByteBuffer buffer, int rowSize) { return values; } + protected Binary[] deserializeBinaryValues(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] values = new Binary[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readBinary(buffer); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = ReadWriteIOUtils.readBinary(buffer); + } + } + } + return values; + } + protected boolean hasEnoughLength(Object left, Object right, int rowSize) { return Array.getLength(left) >= rowSize && Array.getLength(right) >= rowSize; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index b615ea54d..094729539 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -216,6 +216,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] binaries = deserializeBinaryValues(buffer, rowSize, nullIndicators); + return new BinaryColumn(0, rowSize, nullIndicators, binaries); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return binaryArrayEquals(left, right, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 946f39357..6ceceaf10 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -34,6 +34,7 @@ import java.time.LocalDate; import java.util.Arrays; import java.util.Optional; +import org.apache.tsfile.utils.TsPrimitiveType.TsInt; public class DateType extends AbstractIntType { @@ -44,12 +45,12 @@ private DateType() {} @Override public TsPrimitiveType getTsPrimitiveType() { - return new TsPrimitiveType.TsInt(TSDataType.DATE); + return new TsInt(TSDataType.DATE); } @Override public TsPrimitiveType getTsPrimitiveType(Object value) { - return new TsPrimitiveType.TsInt((int) value, TSDataType.DATE); + return new TsInt((int) value, TSDataType.DATE); } @Override @@ -102,6 +103,11 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + return ((IntColumn) super.deserializeColumn(buffer, rowSize, nullIndicators)).modifyDataType(TSDataType.DATE); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return hasEnoughLength(left, right, rowSize) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 4ce1dbbd7..ccd7f983f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -199,6 +199,11 @@ default Object deserializeArray(ByteBuffer buffer, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } + /** Deserializes an array with {@code rowSize} entries from {@code buffer}. */ + default Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Returns whether two array columns are equal in the first {@code rowSize} entries. */ default boolean arrayEquals(Object left, Object right, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); From b431e1cd0ef5a8821da0376fd67f399cc013332b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 18:32:56 +0800 Subject: [PATCH 039/131] add Type.deserializeColumn --- .../block/column/Int32ArrayColumnEncoder.java | 45 +++------- .../tsfile/read/common/type/AbstractType.java | 3 +- .../read/common/type/AbstractVarcharType.java | 6 ++ .../tsfile/read/common/type/BooleanType.java | 26 ++++++ .../tsfile/read/common/type/DateType.java | 5 +- .../tsfile/read/common/type/DoubleType.java | 17 ++++ .../tsfile/read/common/type/FloatType.java | 17 ++++ .../tsfile/read/common/type/ObjectType.java | 6 ++ .../tsfile/read/common/type/VectorType.java | 6 ++ .../block/Int32ArrayColumnEncoderTest.java | 7 ++ .../tsfile/read/common/type/TypeTest.java | 88 +++++++++++++++++++ 11 files changed, 188 insertions(+), 38 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java index 3c6e9b1af..22335915a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import java.io.DataOutputStream; import java.io.IOException; @@ -39,42 +40,16 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun // | byte | list[byte] | list[int32] | // +---------------+-----------------+-------------+ - boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); - - switch (dataType) { - case INT32: - case DATE: - int[] intValues = new int[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - intValues[i] = input.getInt(); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - intValues[i] = input.getInt(); - } - } - } - return new IntColumn(0, positionCount, nullIndicators, intValues); - case FLOAT: - float[] floatValues = new float[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - floatValues[i] = Float.intBitsToFloat(input.getInt()); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - floatValues[i] = Float.intBitsToFloat(input.getInt()); - } - } - } - return new FloatColumn(0, positionCount, nullIndicators, floatValues); - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT32 + && dataType != TSDataType.DATE + && dataType != TSDataType.FLOAT) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + + boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); + return (Column) + Type.fromTsDataType(dataType).deserializeColumn(input, positionCount, nullIndicators); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index d9d429c5b..27f0b0573 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -111,7 +111,8 @@ protected Binary[] deserializeBinaryValues(ByteBuffer buffer, int rowSize) { return values; } - protected Binary[] deserializeBinaryValues(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + protected Binary[] deserializeBinaryValues( + ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { Binary[] values = new Binary[rowSize]; if (nullIndicators == null) { for (int i = 0; i < rowSize; i++) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index fb383ff9c..22669d3f4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -205,6 +205,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] values = deserializeBinaryValues(buffer, rowSize, nullIndicators); + return new BinaryColumn(0, rowSize, nullIndicators, values); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return binaryArrayEquals(left, right, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index d6b42991f..e293af2dc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -32,6 +32,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.ColumnEncoder; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -216,6 +217,31 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + if (nullIndicators == null) { + boolean[] values = ColumnEncoder.deserializeBooleanArray(buffer, rowSize); + return new BooleanColumn(rowSize, Optional.empty(), values); + } + + int nonNullCount = 0; + for (boolean isNull : nullIndicators) { + if (!isNull) { + nonNullCount++; + } + } + + boolean[] nonNullValues = ColumnEncoder.deserializeBooleanArray(buffer, nonNullCount); + boolean[] values = new boolean[rowSize]; + int nonNullIndex = 0; + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = nonNullValues[nonNullIndex++]; + } + } + return new BooleanColumn(rowSize, Optional.of(nullIndicators), values); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return hasEnoughLength(left, right, rowSize) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 6ceceaf10..a0829798e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -27,6 +27,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.utils.TsPrimitiveType.TsInt; import java.io.DataOutputStream; import java.io.IOException; @@ -34,7 +35,6 @@ import java.time.LocalDate; import java.util.Arrays; import java.util.Optional; -import org.apache.tsfile.utils.TsPrimitiveType.TsInt; public class DateType extends AbstractIntType { @@ -105,7 +105,8 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { - return ((IntColumn) super.deserializeColumn(buffer, rowSize, nullIndicators)).modifyDataType(TSDataType.DATE); + return ((IntColumn) super.deserializeColumn(buffer, rowSize, nullIndicators)) + .modifyDataType(TSDataType.DATE); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 2153f65c3..55b9f1f23 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -258,6 +258,23 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + double[] values = new double[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + values[i] = Double.longBitsToDouble(buffer.getLong()); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = Double.longBitsToDouble(buffer.getLong()); + } + } + } + return new DoubleColumn(rowSize, Optional.ofNullable(nullIndicators), values); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return hasEnoughLength(left, right, rowSize) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index e6252602d..5a69b5178 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -256,6 +256,23 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + float[] values = new float[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + values[i] = Float.intBitsToFloat(buffer.getInt()); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = Float.intBitsToFloat(buffer.getInt()); + } + } + } + return new FloatColumn(rowSize, Optional.ofNullable(nullIndicators), values); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return hasEnoughLength(left, right, rowSize) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 7c68bd3a4..00276b810 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -217,6 +217,12 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] values = deserializeBinaryValues(buffer, rowSize, nullIndicators); + return new BinaryColumn(0, rowSize, nullIndicators, values); + } + @Override public boolean arrayEquals(Object left, Object right, int rowSize) { return binaryArrayEquals(left, right, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 8ae6f80d3..afd538693 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -36,6 +36,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; public class VectorType extends AbstractLongType { @@ -112,6 +113,11 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getVector(); } + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public String toString(Field field) { throw new UnSupportedDataTypeException(field.getDataType().toString()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java index 8c82a11dc..51f6c833d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java @@ -37,6 +37,13 @@ import java.util.Random; public class Int32ArrayColumnEncoderTest { + + @Test(expected = IllegalArgumentException.class) + public void testInvalidDataType() { + ColumnEncoderFactory.get(ColumnEncoding.INT32_ARRAY) + .readColumn(ByteBuffer.allocate(0), TSDataType.DOUBLE, 0); + } + @Test public void testIntColumn() { final int positionCount = 10; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 304996bba..4fed96dd3 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.encoder.PlainEncoder; import org.apache.tsfile.enums.TSDataType; @@ -34,6 +35,7 @@ import org.junit.Assert; import org.junit.Test; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.util.Arrays; @@ -303,6 +305,73 @@ public void testCalcTypeSizeFromObject() { } } + @Test + public void testDeserializeColumn() { + boolean[] nullIndicators = {true, false}; + + ByteBuffer booleanBuffer = ByteBuffer.wrap(new byte[] {(byte) 0b1000_0000, 0x01}); + Column booleanColumn = + (Column) + Type.fromTsDataType(TSDataType.BOOLEAN) + .deserializeColumn(booleanBuffer, 2, nullIndicators); + Assert.assertTrue(booleanColumn.isNull(0)); + Assert.assertTrue(booleanColumn.getBoolean(1)); + Assert.assertEquals(1, booleanBuffer.position()); + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT32, TSDataType.DATE}) { + Column column = + (Column) + Type.fromTsDataType(dataType) + .deserializeColumn(intBuffer(20260714), 2, nullIndicators); + Assert.assertEquals(dataType, column.getDataType()); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(20260714, column.getInt(1)); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT64, TSDataType.TIMESTAMP}) { + Column column = + (Column) + Type.fromTsDataType(dataType) + .deserializeColumn(longBuffer(123456789L), 2, nullIndicators); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(123456789L, column.getLong(1)); + } + + Column floatColumn = + (Column) + Type.fromTsDataType(TSDataType.FLOAT) + .deserializeColumn(intBuffer(Float.floatToIntBits(1.25F)), 2, nullIndicators); + Assert.assertTrue(floatColumn.isNull(0)); + Assert.assertEquals(1.25F, floatColumn.getFloat(1), 0); + + Column doubleColumn = + (Column) + Type.fromTsDataType(TSDataType.DOUBLE) + .deserializeColumn(longBuffer(Double.doubleToLongBits(2.5D)), 2, nullIndicators); + Assert.assertTrue(doubleColumn.isNull(0)); + Assert.assertEquals(2.5D, doubleColumn.getDouble(1), 0); + + Binary binary = new Binary("test", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Column column = + (Column) + Type.fromTsDataType(dataType) + .deserializeColumn(binaryBuffer(binary), 2, nullIndicators); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(binary, column.getBinary(1)); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).deserializeColumn(ByteBuffer.allocate(0), 0, null); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + @Test public void testEstimateArraySize() { int size = 10; @@ -449,4 +518,23 @@ public void testGetCurrentValueForVectorAndUnknown() { Assert.assertSame(vector, batchData.currentValue()); Assert.assertNull(Type.fromTsDataType(TSDataType.UNKNOWN).getCurrentValue(null)); } + + private static ByteBuffer intBuffer(int value) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.putInt(value); + return buffer.flip(); + } + + private static ByteBuffer longBuffer(long value) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(value); + return buffer.flip(); + } + + private static ByteBuffer binaryBuffer(Binary value) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES + value.getLength()); + buffer.putInt(value.getLength()); + buffer.put(value.getValues()); + return buffer.flip(); + } } From 86184865bd6ad375ba1b36eba75024e201928f7a Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 18:54:54 +0800 Subject: [PATCH 040/131] add Column.serializeWithoutNulls --- .../apache/tsfile/block/column/Column.java | 7 ++ .../common/block/column/BinaryColumn.java | 13 ++ .../common/block/column/BooleanColumn.java | 25 ++++ .../common/block/column/DoubleColumn.java | 11 ++ .../read/common/block/column/FloatColumn.java | 11 ++ .../block/column/Int32ArrayColumnEncoder.java | 38 +----- .../block/column/Int64ArrayColumnEncoder.java | 70 +++-------- .../read/common/block/column/IntColumn.java | 11 ++ .../read/common/block/column/LongColumn.java | 11 ++ .../read/common/block/column/NullColumn.java | 7 ++ .../read/common/block/column/TimeColumn.java | 9 ++ .../ColumnSerializeWithoutNullsTest.java | 114 ++++++++++++++++++ 12 files changed, 241 insertions(+), 86 deletions(-) create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/common/block/ColumnSerializeWithoutNullsTest.java diff --git a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java index 93e346dc5..9b7c88edd 100644 --- a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java +++ b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java @@ -23,6 +23,8 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Arrays; public interface Column { @@ -108,6 +110,11 @@ default TsPrimitiveType getTsPrimitiveType(int position) { throw new UnsupportedOperationException(getClass().getName()); } + /** Serializes all non-null values in this column. */ + default void serializeWithoutNulls(DataOutputStream output) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Returns whether the values at the two positions are equal. */ default boolean arePositionsEqual(int pos1, int pos2) { return arePositionsEqual(pos1, this, pos2); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index b2b3285e5..a11885f7a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -27,6 +27,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Arrays; import java.util.Optional; @@ -143,6 +145,17 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBinary(getBinary(position)); } + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + Binary value = values[i + arrayOffset]; + output.writeInt(value.getLength()); + output.write(value.getValues()); + } + } + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { boolean thisIsNull = isNull(thisPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java index e40ff4e3b..ac0c0dac1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java @@ -27,6 +27,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -125,6 +127,29 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBoolean(getBoolean(position)); } + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + int nonNullCount = 0; + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + nonNullCount++; + } + } + + byte[] packedValues = new byte[(nonNullCount + 7) / 8]; + int nonNullIndex = 0; + for (int i = 0; i < positionCount; i++) { + if (isNull(i)) { + continue; + } + if (values[i + arrayOffset]) { + packedValues[nonNullIndex / 8] |= (byte) (0b1000_0000 >>> (nonNullIndex % 8)); + } + nonNullIndex++; + } + output.write(packedValues); + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { boolean thisIsNull = isNull(thisPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java index c12c200a7..366a99d42 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java @@ -27,6 +27,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -126,6 +128,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsDouble(getDouble(position)); } + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeLong(Double.doubleToLongBits(values[i + arrayOffset])); + } + } + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { boolean thisIsNull = isNull(thisPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java index 3755aef1e..35b110cb8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java @@ -27,6 +27,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -141,6 +143,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsFloat(getFloat(position)); } + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeInt(Float.floatToIntBits(values[i + arrayOffset])); + } + } + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { boolean thisIsNull = isNull(thisPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java index 22335915a..4e49502cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java @@ -58,38 +58,12 @@ public void writeColumn(DataOutputStream output, Column column) throws IOExcepti ColumnEncoder.serializeNullIndicators(output, column); TSDataType dataType = column.getDataType(); - int positionCount = column.getPositionCount(); - switch (dataType) { - case INT32: - case DATE: - if (column.mayHaveNull()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeInt(column.getInt(i)); - } - } - } else { - for (int i = 0; i < positionCount; i++) { - output.writeInt(column.getInt(i)); - } - } - break; - case FLOAT: - if (column.mayHaveNull()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeInt(Float.floatToIntBits(column.getFloat(i))); - } - } - } else { - for (int i = 0; i < positionCount; i++) { - output.writeInt(Float.floatToIntBits(column.getFloat(i))); - } - } - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT32 + && dataType != TSDataType.DATE + && dataType != TSDataType.FLOAT) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + column.serializeWithoutNulls(output); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java index 0076c88f0..e7e3272c1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import java.io.DataOutputStream; import java.io.IOException; @@ -39,41 +40,16 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun // | byte | list[byte] | list[int64] | // +---------------+-----------------+-------------+ - boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); - switch (dataType) { - case INT64: - case TIMESTAMP: - long[] values = new long[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - values[i] = input.getLong(); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - values[i] = input.getLong(); - } - } - } - return new LongColumn(0, positionCount, nullIndicators, values); - case DOUBLE: - double[] doubleValues = new double[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - doubleValues[i] = Double.longBitsToDouble(input.getLong()); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - doubleValues[i] = Double.longBitsToDouble(input.getLong()); - } - } - } - return new DoubleColumn(0, positionCount, nullIndicators, doubleValues); - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT64 + && dataType != TSDataType.TIMESTAMP + && dataType != TSDataType.DOUBLE) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + + boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); + return (Column) + Type.fromTsDataType(dataType).deserializeColumn(input, positionCount, nullIndicators); } @Override @@ -82,26 +58,12 @@ public void writeColumn(DataOutputStream output, Column column) throws IOExcepti ColumnEncoder.serializeNullIndicators(output, column); TSDataType dataType = column.getDataType(); - int positionCount = column.getPositionCount(); - switch (dataType) { - case INT64: - case TIMESTAMP: - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeLong(column.getLong(i)); - } - } - break; - case DOUBLE: - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeLong(Double.doubleToLongBits(column.getDouble(i))); - } - } - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT64 + && dataType != TSDataType.TIMESTAMP + && dataType != TSDataType.DOUBLE) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + column.serializeWithoutNulls(output); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index de05f1f4d..10e6179d3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -27,6 +27,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -197,6 +199,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsInt(getInt(position), dataType); } + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeInt(values[i + arrayOffset]); + } + } + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { boolean thisIsNull = isNull(thisPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 8edd31d5a..8f089cdf2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -27,6 +27,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -141,6 +143,15 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsLong(getLong(position)); } + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeLong(values[i + arrayOffset]); + } + } + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { boolean thisIsNull = isNull(thisPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index 30092aeb3..4fdb5a718 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -25,6 +25,8 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.utils.RamUsageEstimator; +import java.io.DataOutputStream; + import static java.util.Objects.requireNonNull; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkArrayRange; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkReadablePosition; @@ -81,6 +83,11 @@ public boolean[] isNull() { throw new UnsupportedOperationException(getClass().getName()); } + @Override + public void serializeWithoutNulls(DataOutputStream output) { + // There are no non-null values to serialize. + } + @Override public int getPositionCount() { return positionCount; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java index 2c9ddc964..4156482af 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java @@ -25,6 +25,8 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.utils.RamUsageEstimator; +import java.io.DataOutputStream; +import java.io.IOException; import java.util.Arrays; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkArrayRange; @@ -90,6 +92,13 @@ public Object getObject(int position) { return getLong(position); } + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + output.writeLong(values[i + arrayOffset]); + } + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { return !that.isNull(thatPos) && getLong(thisPos) == that.getLong(thatPos); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/ColumnSerializeWithoutNullsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/ColumnSerializeWithoutNullsTest.java new file mode 100644 index 000000000..c412a8b9c --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/ColumnSerializeWithoutNullsTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.common.block; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.block.column.NullColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.utils.Binary; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +public class ColumnSerializeWithoutNullsTest { + + private static final boolean[] NULL_INDICATORS = {false, true, false, true}; + + @Test + public void testSerializeWithoutNulls() throws IOException { + ByteBuffer buffer = + serialize( + new BooleanColumn( + 4, Optional.of(NULL_INDICATORS), new boolean[] {true, true, false, true})); + Assert.assertEquals((byte) 0b1000_0000, buffer.get()); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = serialize(new IntColumn(4, Optional.of(NULL_INDICATORS), new int[] {1, 2, 3, 4})); + Assert.assertEquals(1, buffer.getInt()); + Assert.assertEquals(3, buffer.getInt()); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize(new LongColumn(4, Optional.of(NULL_INDICATORS), new long[] {1L, 2L, 3L, 4L})); + Assert.assertEquals(1L, buffer.getLong()); + Assert.assertEquals(3L, buffer.getLong()); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize(new FloatColumn(4, Optional.of(NULL_INDICATORS), new float[] {1F, 2F, 3F, 4F})); + Assert.assertEquals(1F, Float.intBitsToFloat(buffer.getInt()), 0); + Assert.assertEquals(3F, Float.intBitsToFloat(buffer.getInt()), 0); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize(new DoubleColumn(4, Optional.of(NULL_INDICATORS), new double[] {1D, 2D, 3D, 4D})); + Assert.assertEquals(1D, Double.longBitsToDouble(buffer.getLong()), 0); + Assert.assertEquals(3D, Double.longBitsToDouble(buffer.getLong()), 0); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize( + new BinaryColumn( + 4, + Optional.of(NULL_INDICATORS), + new Binary[] {binary("a"), binary("b"), binary("cc"), binary("d")})); + Assert.assertEquals(binary("a"), readBinary(buffer)); + Assert.assertEquals(binary("cc"), readBinary(buffer)); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = serialize(new TimeColumn(4, new long[] {1L, 2L, 3L, 4L})); + Assert.assertEquals(1L, buffer.getLong()); + Assert.assertEquals(2L, buffer.getLong()); + Assert.assertEquals(3L, buffer.getLong()); + Assert.assertEquals(4L, buffer.getLong()); + Assert.assertFalse(buffer.hasRemaining()); + + Assert.assertFalse(serialize(new NullColumn(4)).hasRemaining()); + } + + private static ByteBuffer serialize(Column column) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + column.serializeWithoutNulls(new DataOutputStream(output)); + return ByteBuffer.wrap(output.toByteArray()); + } + + private static Binary binary(String value) { + return new Binary(value, StandardCharsets.UTF_8); + } + + private static Binary readBinary(ByteBuffer buffer) { + byte[] value = new byte[buffer.getInt()]; + buffer.get(value); + return new Binary(value); + } +} From a410c130526b53c4adb5c1bf8943d9fdced887ea Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 14 Jul 2026 19:04:39 +0800 Subject: [PATCH 041/131] add Type.createNullColumn --- .../read/common/block/column/NullColumn.java | 24 +---------- .../read/common/type/AbstractIntType.java | 6 +++ .../read/common/type/AbstractLongType.java | 6 +++ .../read/common/type/AbstractVarcharType.java | 6 +++ .../tsfile/read/common/type/BlobType.java | 6 +++ .../tsfile/read/common/type/BooleanType.java | 6 +++ .../tsfile/read/common/type/DoubleType.java | 6 +++ .../tsfile/read/common/type/FloatType.java | 6 +++ .../tsfile/read/common/type/ObjectType.java | 6 +++ .../apache/tsfile/read/common/type/Type.java | 6 +++ .../tsfile/read/common/type/VectorType.java | 7 ++++ .../common/block/NullColumnUnitTest.java | 42 +++++++++++++++++++ 12 files changed, 105 insertions(+), 22 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index 4fdb5a718..06d720275 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -23,6 +23,7 @@ import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; import java.io.DataOutputStream; @@ -151,28 +152,7 @@ public void reverse() { public static Column create(TSDataType dataType, int positionCount) { requireNonNull(dataType, "dataType is null"); - switch (dataType) { - case BOOLEAN: - return new RunLengthEncodedColumn(BooleanColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case INT32: - case DATE: - return new RunLengthEncodedColumn(IntColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case INT64: - case TIMESTAMP: - return new RunLengthEncodedColumn(LongColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case FLOAT: - return new RunLengthEncodedColumn(FloatColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case DOUBLE: - return new RunLengthEncodedColumn(DoubleColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); - default: - throw new IllegalArgumentException( - Messages.format("error.read.null_col_unknown_type", dataType)); - } + return Type.fromTsDataType(dataType).createNullColumn(positionCount); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 46443a47f..cb6811f06 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -39,6 +39,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -291,6 +292,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new IntColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(IntColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 942871f7c..24cf4f719 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -38,6 +38,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -287,6 +288,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new LongColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(LongColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 22669d3f4..42a4ebe08 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -30,6 +30,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -232,6 +233,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new BinaryColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 094729539..b07358fe6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -31,6 +31,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -243,6 +244,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new BinaryColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index e293af2dc..91dab6a74 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -33,6 +33,7 @@ import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.read.common.block.column.ColumnEncoder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -264,6 +265,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new BooleanColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BooleanColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 55b9f1f23..2fc48fb5c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -38,6 +38,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -297,6 +298,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new DoubleColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(DoubleColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 5a69b5178..f70726b22 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -37,6 +37,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -295,6 +296,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new FloatColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(FloatColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 00276b810..39db0aedc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -31,6 +31,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -244,6 +245,11 @@ public Column createColumnWithZeroPosition(int positionCount) { return new BinaryColumn(positionCount); } + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index ccd7f983f..984dfa12f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -245,6 +245,12 @@ default Column createColumnWithZeroPosition(int positionCount) { throw new UnsupportedOperationException(getClass().getName()); } + /** Creates an all-null column with {@code positionCount} positions. */ + default Column createNullColumn(int positionCount) { + throw new IllegalArgumentException( + Messages.format("error.read.null_col_unknown_type", getTypeEnum())); + } + /** Returns the default encoding configured for this type. */ default TSEncoding getDefaultEncoding(TSFileConfig config) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index afd538693..5307000bc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; import org.apache.tsfile.encoding.decoder.LongChimpDecoder; @@ -118,6 +119,12 @@ public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIn throw new UnsupportedOperationException(getDisplayName()); } + @Override + public Column createNullColumn(int positionCount) { + throw new IllegalArgumentException( + Messages.format("error.read.null_col_unknown_type", getTypeEnum())); + } + @Override public String toString(Field field) { throw new UnSupportedDataTypeException(field.getDataType().toString()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java index 1c0fa5b2b..b1fcb3e8d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java @@ -22,12 +22,46 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.column.NullColumn; +import org.apache.tsfile.read.common.type.Type; import org.junit.Assert; import org.junit.Test; public class NullColumnUnitTest { + @Test + public void testCreatingNullColumnsByType() { + Object[][] testCases = { + {TSDataType.BOOLEAN, TSDataType.BOOLEAN}, + {TSDataType.INT32, TSDataType.INT32}, + {TSDataType.DATE, TSDataType.INT32}, + {TSDataType.INT64, TSDataType.INT64}, + {TSDataType.TIMESTAMP, TSDataType.INT64}, + {TSDataType.FLOAT, TSDataType.FLOAT}, + {TSDataType.DOUBLE, TSDataType.DOUBLE}, + {TSDataType.TEXT, TSDataType.TEXT}, + {TSDataType.STRING, TSDataType.TEXT}, + {TSDataType.BLOB, TSDataType.TEXT}, + {TSDataType.OBJECT, TSDataType.TEXT} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + TSDataType expectedDataType = (TSDataType) testCase[1]; + assertNullColumn(Type.fromTsDataType(dataType).createNullColumn(10), expectedDataType); + assertNullColumn(NullColumn.create(dataType, 10), expectedDataType); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + NullColumn.create(dataType, 10); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException ignored) { + // Expected. + } + } + } + @Test public void testCreatingBooleanNullColumn() { Column nullColumn = NullColumn.create(TSDataType.BOOLEAN, 10); @@ -87,4 +121,12 @@ public void testCreatingDoubleNullColumn() { Assert.assertTrue(nullColumn.isNull(0)); Assert.assertTrue(nullColumn.isNull(9)); } + + private static void assertNullColumn(Column column, TSDataType expectedDataType) { + Assert.assertEquals(expectedDataType, column.getDataType()); + Assert.assertEquals(10, column.getPositionCount()); + Assert.assertTrue(column.mayHaveNull()); + Assert.assertTrue(column.isNull(0)); + Assert.assertTrue(column.isNull(9)); + } } From f9ebbe9734ec51d8e499461207147cb10a2fbb69 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 09:41:19 +0800 Subject: [PATCH 042/131] add Type.addPoint --- .../read/common/type/AbstractIntType.java | 7 +++ .../read/common/type/AbstractLongType.java | 7 +++ .../read/common/type/AbstractVarcharType.java | 7 +++ .../tsfile/read/common/type/BlobType.java | 7 +++ .../tsfile/read/common/type/BooleanType.java | 7 +++ .../tsfile/read/common/type/DoubleType.java | 7 +++ .../tsfile/read/common/type/FloatType.java | 7 +++ .../tsfile/read/common/type/ObjectType.java | 7 +++ .../apache/tsfile/read/common/type/Type.java | 8 ++++ .../tsfile/read/common/type/VectorType.java | 8 ++++ .../read/query/dataset/TableResultSet.java | 32 +------------ .../tsfile/read/common/type/TypeTest.java | 48 +++++++++++++++++++ 12 files changed, 122 insertions(+), 30 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index cb6811f06..0b64977b0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -40,10 +40,12 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -156,6 +158,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setIntV(from.getInt()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getInt(columnName)); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 24cf4f719..8277b3895 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -39,10 +39,12 @@ import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -155,6 +157,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setLongV(from.getLong()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getLong(columnName)); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 42a4ebe08..e52968e38 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -31,9 +31,11 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -133,6 +135,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getString(columnName)); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index b07358fe6..32cf509d0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -32,10 +32,12 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -139,6 +141,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getBinary(columnName)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BLOB); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 91dab6a74..25a4ec29d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -34,10 +34,12 @@ import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.read.common.block.column.ColumnEncoder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -147,6 +149,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBoolV(from.getBoolean()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getBoolean(columnName)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BOOLEAN); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 2fc48fb5c..2d971637d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -39,10 +39,12 @@ import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -158,6 +160,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setDoubleV(from.getDouble()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getDouble(columnName)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.DOUBLE); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index f70726b22..434565e5f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -38,10 +38,12 @@ import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -156,6 +158,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setFloatV(from.getFloat()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getFloat(columnName)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.FLOAT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 39db0aedc..e67ae1964 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -32,9 +32,11 @@ import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -140,6 +142,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getBinary(columnName)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.OBJECT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 984dfa12f..f558b2732 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -30,9 +30,11 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -230,6 +232,12 @@ default void setTo(TsPrimitiveType from, Field to) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** Adds the value of {@code columnName} in {@code resultSet} to {@code record}. */ + default void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", this.getTypeEnum())); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 5307000bc..c0ab2df98 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -31,9 +31,11 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.record.TSRecord; import java.io.DataOutputStream; import java.io.IOException; @@ -114,6 +116,12 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getVector(); } + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", getTypeEnum())); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { throw new UnsupportedOperationException(getDisplayName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java index dabe7533b..74f501429 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java @@ -22,6 +22,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.block.TsBlockReader; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.record.TSRecord; @@ -184,36 +185,7 @@ private boolean cacheNextRecord() throws IOException { cachedRecord.dataPointList.add(null); continue; } - switch (dataType) { - case INT32: - case DATE: - cachedRecord.addPoint(columnName, getInt(columnName)); - break; - case INT64: - case TIMESTAMP: - cachedRecord.addPoint(columnName, getLong(columnName)); - break; - case FLOAT: - cachedRecord.addPoint(columnName, getFloat(columnName)); - break; - case DOUBLE: - cachedRecord.addPoint(columnName, getDouble(columnName)); - break; - case STRING: - case TEXT: - cachedRecord.addPoint(columnName, getString(columnName)); - break; - case BLOB: - cachedRecord.addPoint(columnName, getBinary(columnName)); - break; - case BOOLEAN: - cachedRecord.addPoint(columnName, getBoolean(columnName)); - break; - case VECTOR: - case UNKNOWN: - default: - break; - } + Type.fromTsDataType(dataType).addPoint(cachedRecord, columnName, TableResultSet.this); } return true; } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 4fed96dd3..76c6599c0 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -23,17 +23,21 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.encoder.PlainEncoder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.record.TSRecord; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -280,6 +284,50 @@ public void testSetToField() { } } + @Test + public void testAddPoint() { + ResultSet resultSet = Mockito.mock(ResultSet.class); + IDeviceID deviceID = Mockito.mock(IDeviceID.class); + Mockito.when(resultSet.getBoolean("value")).thenReturn(true); + Mockito.when(resultSet.getInt("value")).thenReturn(1); + Mockito.when(resultSet.getLong("value")).thenReturn(2L); + Mockito.when(resultSet.getFloat("value")).thenReturn(1.25F); + Mockito.when(resultSet.getDouble("value")).thenReturn(2.5D); + Mockito.when(resultSet.getString("value")).thenReturn("text"); + Mockito.when(resultSet.getBinary("value")).thenReturn(new byte[] {0x01, 0x23}); + + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 1}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 2L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, new Binary("text", StandardCharsets.UTF_8)}, + {TSDataType.STRING, new Binary("text", StandardCharsets.UTF_8)}, + {TSDataType.BLOB, new Binary(new byte[] {0x01, 0x23})}, + {TSDataType.OBJECT, new Binary(new byte[] {0x01, 0x23})} + }; + + for (Object[] testCase : testCases) { + TSRecord record = new TSRecord(deviceID, 1L); + Type.fromTsDataType((TSDataType) testCase[0]).addPoint(record, "value", resultSet); + Assert.assertEquals(1, record.dataPointList.size()); + Assert.assertEquals(testCase[1], record.dataPointList.get(0).getValue()); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + TSRecord record = new TSRecord(deviceID, 1L); + try { + Type.fromTsDataType(dataType).addPoint(record, "value", resultSet); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + @Test public void testCalcTypeSizeFromObject() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From e28b0c567228d78ff5137cdc11bae6a6b76b57e1 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:13:42 +0800 Subject: [PATCH 043/131] add READ_PAGE_VALUE_TO_BATCHDATA_SERVICE --- .../tsfile/read/reader/page/PageReader.java | 60 ++-------- .../org/apache/tsfile/utils/TypeServices.java | 104 ++++++++++++++++++ 2 files changed, 113 insertions(+), 51 deletions(-) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java index 2db020cf8..04f2c94a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java @@ -30,12 +30,15 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.PageDataValueReader; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; @@ -45,6 +48,7 @@ import java.util.List; import java.util.Optional; import java.util.function.LongConsumer; +import java.util.function.LongPredicate; import static org.apache.tsfile.read.reader.series.PaginationController.UNLIMITED_PAGINATION_CONTROLLER; import static org.apache.tsfile.utils.Preconditions.checkArgument; @@ -149,64 +153,18 @@ private void uncompressDataIfNecessary() throws IOException { /** * @return the returned BatchData may be empty, but never be null */ - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning @Override public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { uncompressDataIfNecessary(); BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false); boolean allSatisfy = recordFilter == null || recordFilter.allSatisfy(this); + PageDataValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_BATCHDATA_SERVICE.call(Type.fromTsDataType(dataType)); + LongPredicate isDeleted = this::isDeleted; while (timeDecoder.hasNext(timeBuffer)) { long timestamp = timeDecoder.readLong(timeBuffer); - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyBoolean(timestamp, aBoolean))) { - pageData.putBoolean(timestamp, aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyInteger(timestamp, anInt))) { - pageData.putInt(timestamp, anInt); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timestamp) && (allSatisfy || recordFilter.satisfyLong(timestamp, aLong))) { - pageData.putLong(timestamp, aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyFloat(timestamp, aFloat))) { - pageData.putFloat(timestamp, aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyDouble(timestamp, aDouble))) { - pageData.putDouble(timestamp, aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyBinary(timestamp, aBinary))) { - pageData.putBinary(timestamp, aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read( + valueDecoder, valueBuffer, recordFilter, pageData, timestamp, allSatisfy, isDeleted); } return pageData.flip(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java new file mode 100644 index 000000000..7b8d91780 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.nio.ByteBuffer; +import java.util.function.LongPredicate; + +public final class TypeServices { + + public static final TypeService READ_PAGE_VALUE_TO_BATCHDATA_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + boolean value = decoder.readBoolean(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyBoolean(timestamp, value))) { + pageData.putBoolean(timestamp, value); + } + }; + case INT32, DATE -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + int value = decoder.readInt(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyInteger(timestamp, value))) { + pageData.putInt(timestamp, value); + } + }; + case INT64, TIMESTAMP -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + long value = decoder.readLong(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyLong(timestamp, value))) { + pageData.putLong(timestamp, value); + } + }; + case FLOAT -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + float value = decoder.readFloat(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyFloat(timestamp, value))) { + pageData.putFloat(timestamp, value); + } + }; + case DOUBLE -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + double value = decoder.readDouble(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyDouble(timestamp, value))) { + pageData.putDouble(timestamp, value); + } + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + Binary value = decoder.readBinary(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyBinary(timestamp, value))) { + pageData.putBinary(timestamp, value); + } + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + }; + }; + + private TypeServices() {} + + @FunctionalInterface + public interface PageDataValueReader { + + void read( + Decoder decoder, + ByteBuffer buffer, + Filter filter, + BatchData pageData, + long timestamp, + boolean allSatisfy, + LongPredicate isDeleted); + } +} From f537048108c8e820a3c493b5fbd57e503b112bc2 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:27:22 +0800 Subject: [PATCH 044/131] add READ_PAGE_VALUE_TO_TSBLOCK_SERVICE --- .../tsfile/read/reader/page/PageReader.java | 186 ++------------- .../org/apache/tsfile/utils/TypeServices.java | 212 ++++++++++++++++++ 2 files changed, 234 insertions(+), 164 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java index 04f2c94a4..60a877a41 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.reader.page; -import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.PageHeader; @@ -29,17 +28,16 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; -import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.series.PaginationController; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.PageDataBlockValueReader; +import org.apache.tsfile.utils.TypeServices.PageDataReadStatus; import org.apache.tsfile.utils.TypeServices.PageDataValueReader; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; import java.io.Serializable; @@ -178,175 +176,35 @@ public TsBlock getAllSatisfiedData() throws IOException { public TsBlock getAllSatisfiedData(LongConsumer filterRowsRecorder) throws IOException { uncompressDataIfNecessary(); TsBlockBuilder builder; - int initialExpectedEntries = (int) pageHeader.getStatistics().getCount(); + int initialExpectedEntries = pageHeader.getStatistics().getCount(); if (paginationController.hasSetLimit()) { initialExpectedEntries = (int) Math.min(initialExpectedEntries, paginationController.getCurLimit()); } builder = new TsBlockBuilder(initialExpectedEntries, Collections.singletonList(dataType)); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); long allFilteredRows = 0; boolean allSatisfy = recordFilter == null || recordFilter.allSatisfy(this); - switch (dataType) { - case BOOLEAN: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyBoolean(timestamp, aBoolean)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeBoolean(aBoolean); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case INT32: - case DATE: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - int anInt = valueDecoder.readInt(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyInteger(timestamp, anInt)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeInt(anInt); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case INT64: - case TIMESTAMP: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - long aLong = valueDecoder.readLong(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyLong(timestamp, aLong)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeLong(aLong); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case FLOAT: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - float aFloat = valueDecoder.readFloat(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyFloat(timestamp, aFloat)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeFloat(aFloat); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case DOUBLE: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - double aDouble = valueDecoder.readDouble(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyDouble(timestamp, aDouble)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeDouble(aDouble); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyBinary(timestamp, aBinary)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeBinary(aBinary); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } + PageDataBlockValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_TSBLOCK_SERVICE.call(Type.fromTsDataType(dataType)); + LongPredicate isDeleted = this::isDeleted; + while (timeDecoder.hasNext(timeBuffer)) { + long timestamp = timeDecoder.readLong(timeBuffer); + PageDataReadStatus status = + valueReader.read( + valueDecoder, + valueBuffer, + recordFilter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController); + if (status == PageDataReadStatus.FILTERED) { + allFilteredRows++; + } else if (status == PageDataReadStatus.STOP) { break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); + } } if (filterRowsRecorder != null && allFilteredRows > 0) { filterRowsRecorder.accept(allFilteredRows); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 7b8d91780..72634a73e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -21,8 +21,10 @@ import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; @@ -87,6 +89,196 @@ public final class TypeServices { }; }; + public static final TypeService READ_PAGE_VALUE_TO_TSBLOCK_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + boolean value = decoder.readBoolean(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyBoolean(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeBoolean(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case INT32, DATE -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + int value = decoder.readInt(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyInteger(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeInt(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case INT64, TIMESTAMP -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + long value = decoder.readLong(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyLong(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeLong(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case FLOAT -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + float value = decoder.readFloat(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyFloat(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeFloat(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case DOUBLE -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + double value = decoder.readDouble(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyDouble(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeDouble(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + Binary value = decoder.readBinary(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyBinary(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeBinary(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + }; + }; + private TypeServices() {} @FunctionalInterface @@ -101,4 +293,24 @@ void read( boolean allSatisfy, LongPredicate isDeleted); } + + @FunctionalInterface + public interface PageDataBlockValueReader { + + PageDataReadStatus read( + Decoder decoder, + ByteBuffer buffer, + Filter filter, + TsBlockBuilder builder, + long timestamp, + boolean allSatisfy, + LongPredicate isDeleted, + PaginationController paginationController); + } + + public enum PageDataReadStatus { + CONTINUE, + FILTERED, + STOP + } } From 31094284b6c5d0b5c026eb30b6cbcb43b47d206e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:32:12 +0800 Subject: [PATCH 045/131] refactor nextBatch --- .../read/reader/page/ValuePageReader.java | 59 ++++--------------- 1 file changed, 10 insertions(+), 49 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index 50835f5f8..e0b711252 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -28,10 +28,13 @@ import org.apache.tsfile.read.common.BatchDataFactory; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.PageDataValueReader; import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; @@ -39,6 +42,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.function.LongPredicate; public class ValuePageReader { @@ -115,60 +119,17 @@ public BatchData nextBatch(long[] timeBatch, boolean ascending, Filter filter) throws IOException { uncompressDataIfNecessary(); BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false); + PageDataValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_BATCHDATA_SERVICE.call(Type.fromTsDataType(dataType)); + boolean allSatisfy = filter == null; + LongPredicate isDeleted = this::isDeleted; for (int i = 0; i < timeBatch.length; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { continue; } long timestamp = timeBatch[i]; - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyBoolean(timestamp, aBoolean))) { - pageData.putBoolean(timestamp, aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyInteger(timestamp, anInt))) { - pageData.putInt(timestamp, anInt); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timestamp) && (filter == null || filter.satisfyLong(timestamp, aLong))) { - pageData.putLong(timestamp, aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timestamp) && (filter == null || filter.satisfyFloat(timestamp, aFloat))) { - pageData.putFloat(timestamp, aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyDouble(timestamp, aDouble))) { - pageData.putDouble(timestamp, aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyBinary(timestamp, aBinary))) { - pageData.putBinary(timestamp, aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read( + valueDecoder, valueBuffer, filter, pageData, timestamp, allSatisfy, isDeleted); } return pageData.flip(); } From 45e92b0fc57cd4710203c72c6f0775bb108dfdd8 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:37:05 +0800 Subject: [PATCH 046/131] refactor nextValue --- .../read/reader/page/ValuePageReader.java | 55 ++---------------- .../org/apache/tsfile/utils/TypeServices.java | 57 +++++++++++++++++++ 2 files changed, 61 insertions(+), 51 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index e0b711252..f01302f93 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -34,6 +34,7 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.PageDataTsPrimitiveValueReader; import org.apache.tsfile.utils.TypeServices.PageDataValueReader; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -136,60 +137,12 @@ public BatchData nextBatch(long[] timeBatch, boolean ascending, Filter filter) public TsPrimitiveType nextValue(long timestamp, int timeIndex) throws IOException { uncompressDataIfNecessary(); - TsPrimitiveType resultValue = null; if (valueBuffer == null || ((bitmap[timeIndex / 8] & 0xFF) & (MASK >>> (timeIndex % 8))) == 0) { return null; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsBoolean(aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = - dataType.equals(TSDataType.INT32) - ? new TsPrimitiveType.TsInt(anInt) - : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsLong(aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsFloat(aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } - - return resultValue; + PageDataTsPrimitiveValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_TSPRIMITIVETYPE_SERVICE.call(Type.fromTsDataType(dataType)); + return valueReader.read(valueDecoder, valueBuffer, timestamp, this::isDeleted); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 72634a73e..aeb4df8f3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -20,6 +20,7 @@ package org.apache.tsfile.utils; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.type.service.TypeService; @@ -279,6 +280,55 @@ public final class TypeServices { }; }; + public static final TypeService + READ_PAGE_VALUE_TO_TSPRIMITIVETYPE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, buffer, timestamp, isDeleted) -> { + boolean value = decoder.readBoolean(buffer); + return isDeleted.test(timestamp) + ? null + : new TsPrimitiveType.TsBoolean(value); + }; + case INT32 -> + (decoder, buffer, timestamp, isDeleted) -> { + int value = decoder.readInt(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsInt(value); + }; + case DATE -> + (decoder, buffer, timestamp, isDeleted) -> { + int value = decoder.readInt(buffer); + return isDeleted.test(timestamp) + ? null + : new TsPrimitiveType.TsInt(value, TSDataType.DATE); + }; + case INT64, TIMESTAMP -> + (decoder, buffer, timestamp, isDeleted) -> { + long value = decoder.readLong(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsLong(value); + }; + case FLOAT -> + (decoder, buffer, timestamp, isDeleted) -> { + float value = decoder.readFloat(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsFloat(value); + }; + case DOUBLE -> + (decoder, buffer, timestamp, isDeleted) -> { + double value = decoder.readDouble(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsDouble(value); + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, buffer, timestamp, isDeleted) -> { + Binary value = decoder.readBinary(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsBinary(value); + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, buffer, timestamp, isDeleted) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + }; + }; + private TypeServices() {} @FunctionalInterface @@ -308,6 +358,13 @@ PageDataReadStatus read( PaginationController paginationController); } + @FunctionalInterface + public interface PageDataTsPrimitiveValueReader { + + TsPrimitiveType read( + Decoder decoder, ByteBuffer buffer, long timestamp, LongPredicate isDeleted); + } + public enum PageDataReadStatus { CONTINUE, FILTERED, From 74df1589f3c9866141b86936f7cbd7cfbb1e9af6 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:41:56 +0800 Subject: [PATCH 047/131] refactor nextValueBatch --- .../read/reader/page/ValuePageReader.java | 52 ++----------------- 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index f01302f93..0038027df 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -155,58 +155,14 @@ public TsPrimitiveType[] nextValueBatch(long[] timeBatch) throws IOException { if (valueBuffer == null) { return valueBatch; } + PageDataTsPrimitiveValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_TSPRIMITIVETYPE_SERVICE.call(Type.fromTsDataType(dataType)); + LongPredicate isDeleted = this::isDeleted; for (int i = 0; i < size; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { continue; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsBoolean(aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = - dataType.equals(TSDataType.INT32) - ? new TsPrimitiveType.TsInt(anInt) - : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsLong(aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsFloat(aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueBatch[i] = valueReader.read(valueDecoder, valueBuffer, timeBatch[i], isDeleted); } return valueBatch; } From 6263681c424f381a464f5acdb8aa1c494e5b620d Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:46:39 +0800 Subject: [PATCH 048/131] refactor writeColumnBuilderWithNextBatch --- .../read/reader/page/ValuePageReader.java | 86 +-------------- .../org/apache/tsfile/utils/TypeServices.java | 102 ++++++++++++++++++ 2 files changed, 106 insertions(+), 82 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index 0038027df..8fec002b4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -34,6 +34,7 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.PageDataColumnBuilderValueReader; import org.apache.tsfile.utils.TypeServices.PageDataTsPrimitiveValueReader; import org.apache.tsfile.utils.TypeServices.PageDataValueReader; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -179,6 +180,8 @@ public void writeColumnBuilderWithNextBatch( } return; } + PageDataColumnBuilderValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE.call(Type.fromTsDataType(dataType)); for (int i = 0; i < readEndIndex; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { if (keepCurrentRow[i]) { @@ -186,88 +189,7 @@ public void writeColumnBuilderWithNextBatch( } continue; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeBoolean(aBoolean); - } - } - break; - case INT32: - int anInt = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeInt(anInt); - } - } - break; - case DATE: - int anDate = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - if (columnBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) columnBuilder).writeDate(anDate); - } else { - columnBuilder.writeInt(anDate); - } - } - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeLong(aLong); - } - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeFloat(aFloat); - } - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeDouble(aDouble); - } - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeBinary(aBinary); - } - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read(valueDecoder, valueBuffer, columnBuilder, keepCurrentRow[i], isDeleted[i]); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index aeb4df8f3..9cc305439 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -19,10 +19,12 @@ package org.apache.tsfile.utils; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.series.PaginationController; @@ -329,6 +331,95 @@ public final class TypeServices { }; }; + public static final TypeService + READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + boolean value = decoder.readBoolean(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeBoolean(value); + } + } + }; + case INT32 -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + int value = decoder.readInt(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeInt(value); + } + } + }; + case DATE -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + int value = decoder.readInt(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else if (builder instanceof BinaryColumnBuilder) { + ((BinaryColumnBuilder) builder).writeDate(value); + } else { + builder.writeInt(value); + } + } + }; + case INT64, TIMESTAMP -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + long value = decoder.readLong(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeLong(value); + } + } + }; + case FLOAT -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + float value = decoder.readFloat(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeFloat(value); + } + } + }; + case DOUBLE -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + double value = decoder.readDouble(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeDouble(value); + } + } + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + Binary value = decoder.readBinary(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeBinary(value); + } + } + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + }; + }; + private TypeServices() {} @FunctionalInterface @@ -365,6 +456,17 @@ TsPrimitiveType read( Decoder decoder, ByteBuffer buffer, long timestamp, LongPredicate isDeleted); } + @FunctionalInterface + public interface PageDataColumnBuilderValueReader { + + void read( + Decoder decoder, + ByteBuffer buffer, + ColumnBuilder builder, + boolean keepCurrentRow, + boolean isDeleted); + } + public enum PageDataReadStatus { CONTINUE, FILTERED, From 2496389a036f6d41d0d0190e042805f6f8a66795 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:50:07 +0800 Subject: [PATCH 049/131] refactor writeColumnBuilderWithNextBatch --- .../read/reader/page/ValuePageReader.java | 57 +------------------ 1 file changed, 3 insertions(+), 54 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index 8fec002b4..ff92f3eaf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -204,6 +204,8 @@ public void writeColumnBuilderWithNextBatch( } return; } + PageDataColumnBuilderValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE.call(Type.fromTsDataType(dataType)); for (int i = 0; i < readEndIndex; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { if (keepCurrentRow[i]) { @@ -211,60 +213,7 @@ public void writeColumnBuilderWithNextBatch( } continue; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeBoolean(aBoolean); - } - break; - case INT32: - int anInt = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeInt(anInt); - } - break; - case DATE: - int anDate = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - if (columnBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) columnBuilder).writeDate(anDate); - } else { - columnBuilder.writeInt(anDate); - } - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeLong(aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeFloat(aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read(valueDecoder, valueBuffer, columnBuilder, keepCurrentRow[i], false); } } From c5d08390e03e9ac4f169d4dee50b1787e7ae9067 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 10:54:17 +0800 Subject: [PATCH 050/131] refactor writeColumnBuilderWithNextBatch --- .../read/reader/page/ValuePageReader.java | 142 ++---------------- 1 file changed, 14 insertions(+), 128 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index ff92f3eaf..cd0799f3f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -27,17 +27,14 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.BatchDataFactory; import org.apache.tsfile.read.common.TimeRange; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.utils.TypeServices; import org.apache.tsfile.utils.TypeServices.PageDataColumnBuilderValueReader; import org.apache.tsfile.utils.TypeServices.PageDataTsPrimitiveValueReader; import org.apache.tsfile.utils.TypeServices.PageDataValueReader; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; import java.io.Serializable; @@ -224,131 +221,20 @@ public void writeColumnBuilderWithNextBatch( columnBuilder.appendNull(readEndIndex - readStartIndex); return; } - - switch (dataType) { - case BOOLEAN: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readBoolean(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - columnBuilder.writeBoolean(aBoolean); - } - break; - case INT32: - case DATE: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readInt(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - int aInt = valueDecoder.readInt(valueBuffer); - if (dataType == TSDataType.INT32) { - columnBuilder.writeInt(aInt); - } else { - if (columnBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) columnBuilder).writeDate(aInt); - } else { - columnBuilder.writeInt(aInt); - } - } - } - break; - case INT64: - case TIMESTAMP: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readLong(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - long aLong = valueDecoder.readLong(valueBuffer); - columnBuilder.writeLong(aLong); - } - break; - case FLOAT: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readFloat(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - float aFloat = valueDecoder.readFloat(valueBuffer); - columnBuilder.writeFloat(aFloat); - } - break; - case DOUBLE: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readDouble(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - double aDouble = valueDecoder.readDouble(valueBuffer); - columnBuilder.writeDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readBinary(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - Binary aBinary = valueDecoder.readBinary(valueBuffer); - columnBuilder.writeBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); + PageDataColumnBuilderValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE.call(Type.fromTsDataType(dataType)); + // skip useless data + for (int i = 0; i < readStartIndex; i++) { + if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) != 0) { + valueReader.read(valueDecoder, valueBuffer, columnBuilder, false, false); + } + } + for (int i = readStartIndex; i < readEndIndex; i++) { + if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { + columnBuilder.appendNull(); + continue; + } + valueReader.read(valueDecoder, valueBuffer, columnBuilder, true, false); } } From 20de1d2ed99efb1a1f25a49b9563759467b6e274 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 11:04:56 +0800 Subject: [PATCH 051/131] refactor writeEmptyDataInOneRow --- .../org/apache/tsfile/utils/TypeServices.java | 24 +++++++++++++ .../chunk/AlignedChunkGroupWriterImpl.java | 36 ++++--------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 9cc305439..cae6b0c63 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; @@ -29,6 +30,7 @@ import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import java.nio.ByteBuffer; import java.util.function.LongPredicate; @@ -420,6 +422,22 @@ public final class TypeServices { }; }; + public static final TypeService WRITE_EMPTY_VALUE_TO_CHUNK_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> writer -> writer.write(-1, false, true); + case INT32, DATE -> writer -> writer.write(-1, 0, true); + case INT64, TIMESTAMP -> writer -> writer.write(-1, 0L, true); + case FLOAT -> writer -> writer.write(-1, 0.0F, true); + case DOUBLE -> writer -> writer.write(-1, 0.0D, true); + case TEXT, BLOB, STRING, OBJECT -> writer -> writer.write(-1, null, true); + case ROW, UNKNOWN, VECTOR -> + writer -> { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", type.getTypeEnum())); + }; + }; + private TypeServices() {} @FunctionalInterface @@ -467,6 +485,12 @@ void read( boolean isDeleted); } + @FunctionalInterface + public interface EmptyValueChunkWriter { + + void write(ValueChunkWriter writer); + } + public enum PageDataReadStatus { CONTINUE, FILTERED, diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 07e82767d..7c783acf0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -31,8 +31,11 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.EmptyValueChunkWriter; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -357,35 +360,10 @@ public void tryToAddEmptyPageAndData(ValueChunkWriter valueChunkWriter) throws I private void writeEmptyDataInOneRow(List valueChunkWriterList) { for (ValueChunkWriter valueChunkWriter : valueChunkWriterList) { - TSDataType dataType = valueChunkWriter.getDataType(); - switch (dataType) { - case BOOLEAN: - valueChunkWriter.write(-1, false, true); - break; - case INT32: - case DATE: - valueChunkWriter.write(-1, 0, true); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(-1, 0L, true); - break; - case FLOAT: - valueChunkWriter.write(-1, 0.0f, true); - break; - case DOUBLE: - valueChunkWriter.write(-1, 0.0d, true); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueChunkWriter.write(-1, null, true); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + EmptyValueChunkWriter emptyValueWriter = + TypeServices.WRITE_EMPTY_VALUE_TO_CHUNK_SERVICE.call( + Type.fromTsDataType(valueChunkWriter.getDataType())); + emptyValueWriter.write(valueChunkWriter); } } From b66784553960dddeed438003a1881c3d2f68d0ef Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 11:14:15 +0800 Subject: [PATCH 052/131] refactor getDataPoint --- .../read/common/type/AbstractIntType.java | 7 +++ .../read/common/type/AbstractLongType.java | 7 +++ .../read/common/type/AbstractVarcharType.java | 7 +++ .../tsfile/read/common/type/BlobType.java | 7 +++ .../tsfile/read/common/type/BooleanType.java | 7 +++ .../tsfile/read/common/type/DateType.java | 7 +++ .../tsfile/read/common/type/DoubleType.java | 7 +++ .../tsfile/read/common/type/FloatType.java | 7 +++ .../tsfile/read/common/type/ObjectType.java | 7 +++ .../apache/tsfile/read/common/type/Type.java | 7 +++ .../write/record/datapoint/DataPoint.java | 38 +-------------- .../tsfile/read/common/type/TypeTest.java | 46 +++++++++++++++++++ 12 files changed, 118 insertions(+), 36 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 0b64977b0..6f33048a9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -46,6 +46,8 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -163,6 +165,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getInt(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new IntDataPoint(measurementId, Integer.parseInt(value)); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 8277b3895..1465190a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -45,6 +45,8 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -162,6 +164,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getLong(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new LongDataPoint(measurementId, Long.parseLong(value)); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index e52968e38..62a116fdf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -36,6 +36,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -140,6 +142,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getString(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 32cf509d0..a45717925 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -38,6 +38,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -146,6 +148,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBinary(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BLOB); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 25a4ec29d..9a687d199 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -40,6 +40,8 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; +import org.apache.tsfile.write.record.datapoint.DataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -154,6 +156,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBoolean(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new BooleanDataPoint(measurementId, Boolean.parseBoolean(value)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BOOLEAN); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index a0829798e..c99aa6de3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -28,6 +28,8 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.utils.TsPrimitiveType.TsInt; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -53,6 +55,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsInt((int) value, TSDataType.DATE); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); + } + @Override public void addValue(int rowIndex, Object value, Object column) { checkValueType(value, LocalDate.class, "LocalDate"); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 2d971637d..cc397af90 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -45,6 +45,8 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -165,6 +167,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getDouble(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new DoubleDataPoint(measurementId, Double.parseDouble(value)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.DOUBLE); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 434565e5f..b743bfe55 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -44,6 +44,8 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.FloatDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -163,6 +165,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getFloat(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new FloatDataPoint(measurementId, Float.parseFloat(value)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.FLOAT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index e67ae1964..c1244ca8a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -37,6 +37,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -147,6 +149,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBinary(columnName)); } + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.OBJECT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index f558b2732..a00c8d2be 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -35,6 +35,7 @@ import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -238,6 +239,12 @@ default void addPoint(TSRecord record, String columnName, ResultSet resultSet) { Messages.format("error.common.unsupported_data_type", this.getTypeEnum())); } + /** Creates a data point by parsing a string value. */ + default DataPoint getDataPoint(String measurementId, String value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java index 103e7503d..388f6aa3b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java @@ -19,11 +19,10 @@ package org.apache.tsfile.write.record.datapoint; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.StringContainer; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -67,46 +66,13 @@ protected DataPoint(TSDataType type, String measurementId) { * @return data point class according to data type */ public static DataPoint getDataPoint(TSDataType dataType, String measurementId, String value) { - DataPoint dataPoint = null; try { - switch (dataType) { - case INT32: - dataPoint = new IntDataPoint(measurementId, Integer.parseInt(value)); - break; - case DATE: - dataPoint = new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); - break; - case INT64: - case TIMESTAMP: - dataPoint = new LongDataPoint(measurementId, Long.parseLong(value)); - break; - case FLOAT: - dataPoint = new FloatDataPoint(measurementId, Float.parseFloat(value)); - break; - case DOUBLE: - dataPoint = new DoubleDataPoint(measurementId, Double.parseDouble(value)); - break; - case BOOLEAN: - dataPoint = new BooleanDataPoint(measurementId, Boolean.parseBoolean(value)); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - dataPoint = - new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + return Type.fromTsDataType(dataType).getDataPoint(measurementId, value); } catch (Exception e) { throw new UnSupportedDataTypeException( Messages.format( "error.write.datapoint_type_value_mismatch", measurementId, dataType, value)); } - - return dataPoint; } /** diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 76c6599c0..0b1500fbf 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -34,6 +34,7 @@ import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; import org.junit.Assert; import org.junit.Test; @@ -121,6 +122,51 @@ public void testCalcTypeSize() { } } + @Test + public void testGetDataPoint() { + Object[][] testCases = { + {TSDataType.BOOLEAN, "true", true}, + {TSDataType.INT32, "1", 1}, + {TSDataType.DATE, "2026-07-15", 20260715}, + {TSDataType.INT64, "2", 2L}, + {TSDataType.TIMESTAMP, "3", 3L}, + {TSDataType.FLOAT, "1.25", 1.25F}, + {TSDataType.DOUBLE, "2.5", 2.5D}, + {TSDataType.TEXT, "text", new Binary("text", TSFileConfig.STRING_CHARSET)}, + {TSDataType.STRING, "string", new Binary("string", TSFileConfig.STRING_CHARSET)}, + {TSDataType.BLOB, "blob", new Binary("blob", TSFileConfig.STRING_CHARSET)}, + {TSDataType.OBJECT, "object", new Binary("object", TSFileConfig.STRING_CHARSET)} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + String value = (String) testCase[1]; + Object expectedValue = testCase[2]; + + DataPoint typeDataPoint = Type.fromTsDataType(dataType).getDataPoint("s", value); + Assert.assertEquals("s", typeDataPoint.getMeasurementId()); + Assert.assertEquals(expectedValue, typeDataPoint.getValue()); + + DataPoint factoryDataPoint = DataPoint.getDataPoint(dataType, "s", value); + Assert.assertEquals(typeDataPoint.getClass(), factoryDataPoint.getClass()); + Assert.assertEquals(expectedValue, factoryDataPoint.getValue()); + } + + try { + DataPoint.getDataPoint(TSDataType.INT32, "s", "invalid"); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + + try { + DataPoint.getDataPoint(TSDataType.VECTOR, "s", "value"); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + @Test public void testToString() { Object[][] testCases = { From d0b15934adc6cde5685749c287d306c2ca8eed17 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 11:41:20 +0800 Subject: [PATCH 053/131] refactor write --- .../read/common/type/AbstractIntType.java | 6 +++ .../read/common/type/AbstractLongType.java | 6 +++ .../read/common/type/AbstractVarcharType.java | 6 +++ .../tsfile/read/common/type/BlobType.java | 6 +++ .../tsfile/read/common/type/BooleanType.java | 6 +++ .../tsfile/read/common/type/DoubleType.java | 6 +++ .../tsfile/read/common/type/FloatType.java | 6 +++ .../tsfile/read/common/type/ObjectType.java | 6 +++ .../apache/tsfile/read/common/type/Type.java | 7 +++ .../tsfile/read/common/type/VectorType.java | 14 ++++++ .../chunk/AlignedChunkGroupWriterImpl.java | 29 +----------- .../tsfile/read/common/type/TypeTest.java | 47 ++++++++++++++++++- 12 files changed, 116 insertions(+), 29 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 6f33048a9..fe8426b50 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -45,6 +45,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.IntDataPoint; @@ -170,6 +171,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new IntDataPoint(measurementId, Integer.parseInt(value)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? 0 : (int) value, isNull); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 1465190a3..8841e39d1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -44,6 +44,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.LongDataPoint; @@ -169,6 +170,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new LongDataPoint(measurementId, Long.parseLong(value)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, (long) value, isNull); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 62a116fdf..f07e71e76 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; @@ -147,6 +148,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, (Binary) value, isNull); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index a45717925..6a01d56f3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -37,6 +37,7 @@ import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; @@ -153,6 +154,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, (Binary) value, isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BLOB); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 9a687d199..460c4b16c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -39,6 +39,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -161,6 +162,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new BooleanDataPoint(measurementId, Boolean.parseBoolean(value)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, (boolean) value, isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BOOLEAN); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index cc397af90..d75a540cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -44,6 +44,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; @@ -172,6 +173,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new DoubleDataPoint(measurementId, Double.parseDouble(value)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, (double) value, isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.DOUBLE); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index b743bfe55..14e316f1f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -43,6 +43,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.FloatDataPoint; @@ -170,6 +171,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new FloatDataPoint(measurementId, Float.parseFloat(value)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, (float) value, isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.FLOAT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index c1244ca8a..f91f1f887 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -36,6 +36,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; @@ -154,6 +155,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); } + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, (Binary) value, isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.OBJECT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index a00c8d2be..a4b02afb6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -34,6 +34,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -245,6 +246,12 @@ default DataPoint getDataPoint(String measurementId, String value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes a value to an aligned value chunk. */ + default void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index c0ab2df98..1a92b35ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -35,7 +35,9 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -47,6 +49,18 @@ public class VectorType extends AbstractLongType { private VectorType() {} + @Override + public DataPoint getDataPoint(String measurementId, String value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + @Override public void init(BatchData batchData) { batchData.initVectorValues(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 7c783acf0..9ecc4f4ad 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -174,34 +174,7 @@ public int write(long time, List data) throws WriteProcessException, if (valueChunkWriter == null) { valueChunkWriter = tryToAddSeriesWriterInternal(point.getMeasurementSchema()); } - switch (point.getType()) { - case BOOLEAN: - valueChunkWriter.write(time, (boolean) point.getValue(), isNull); - break; - case INT32: - case DATE: - valueChunkWriter.write(time, isNull ? 0 : (int) point.getValue(), isNull); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(time, (long) point.getValue(), isNull); - break; - case FLOAT: - valueChunkWriter.write(time, (float) point.getValue(), isNull); - break; - case DOUBLE: - valueChunkWriter.write(time, (double) point.getValue(), isNull); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueChunkWriter.write(time, (Binary) point.getValue(), isNull); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", point.getType())); - } + Type.fromTsDataType(point.getType()).write(valueChunkWriter, time, point.getValue(), isNull); } if (!emptyValueChunkWriters.isEmpty()) { writeEmptyDataInOneRow(emptyValueChunkWriters); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 0b1500fbf..5eeaa9121 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -33,6 +33,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -160,7 +161,51 @@ public void testGetDataPoint() { } try { - DataPoint.getDataPoint(TSDataType.VECTOR, "s", "value"); + DataPoint.getDataPoint(TSDataType.VECTOR, "s", "1"); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + @Test + public void testWriteValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, true, false); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, 1, false); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DATE).write(writer, 3L, null, true); + Mockito.verify(writer).write(3L, 0, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 4L, 2L, false); + Mockito.verify(writer).write(4L, 2L, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 5L, 1.25F, false); + Mockito.verify(writer).write(5L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 6L, 2.5D, false); + Mockito.verify(writer).write(6L, 2.5D, false); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 7L, binary, false); + Mockito.verify(writer).write(7L, binary, false); + Mockito.reset(writer); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 8L, null, true); Assert.fail("Expected UnSupportedDataTypeException"); } catch (UnSupportedDataTypeException ignored) { // Expected. From 09a435d0c6ec71bdf77c2a3afb41c26e9056c34a Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 11:48:17 +0800 Subject: [PATCH 054/131] refactor write --- .../read/common/type/AbstractIntType.java | 6 +++ .../read/common/type/AbstractLongType.java | 6 +++ .../read/common/type/AbstractVarcharType.java | 6 +++ .../tsfile/read/common/type/BlobType.java | 6 +++ .../tsfile/read/common/type/BooleanType.java | 6 +++ .../tsfile/read/common/type/DateType.java | 10 ++++ .../tsfile/read/common/type/DoubleType.java | 6 +++ .../tsfile/read/common/type/FloatType.java | 6 +++ .../tsfile/read/common/type/ObjectType.java | 6 +++ .../apache/tsfile/read/common/type/Type.java | 7 +++ .../tsfile/read/common/type/VectorType.java | 7 +++ .../chunk/AlignedChunkGroupWriterImpl.java | 45 +--------------- .../tsfile/read/common/type/TypeTest.java | 52 +++++++++++++++++++ 13 files changed, 126 insertions(+), 43 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index fe8426b50..e9437c880 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -176,6 +176,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, isNull ? 0 : (int) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((int[]) column)[rowIndex], isNull); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 8841e39d1..e9151e5d6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -175,6 +175,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (long) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((long[]) column)[rowIndex], isNull); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index f07e71e76..e538bd773 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -153,6 +153,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (Binary) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) column)[rowIndex], isNull); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 6a01d56f3..1ff6e1505 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -159,6 +159,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (Binary) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) column)[rowIndex], isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BLOB); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 460c4b16c..2a4b91c29 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -167,6 +167,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (boolean) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((boolean[]) column)[rowIndex], isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BOOLEAN); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index c99aa6de3..80f3cd67c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -28,6 +28,7 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.utils.TsPrimitiveType.TsInt; +import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.IntDataPoint; @@ -60,6 +61,15 @@ public DataPoint getDataPoint(String measurementId, String value) { return new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write( + time, + isNull ? 0 : DateUtils.parseDateExpressionToInt(((LocalDate[]) column)[rowIndex]), + isNull); + } + @Override public void addValue(int rowIndex, Object value, Object column) { checkValueType(value, LocalDate.class, "LocalDate"); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index d75a540cd..20153d48c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -178,6 +178,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (double) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((double[]) column)[rowIndex], isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.DOUBLE); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 14e316f1f..795a43a9e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -176,6 +176,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (float) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((float[]) column)[rowIndex], isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.FLOAT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index f91f1f887..665a58a1f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -160,6 +160,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (Binary) value, isNull); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) column)[rowIndex], isNull); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.OBJECT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index a4b02afb6..d2b7a09b2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -252,6 +252,13 @@ default void write(ValueChunkWriter writer, long time, Object value, boolean isN Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes a value at {@code rowIndex} in a Tablet column to an aligned value chunk. */ + default void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 1a92b35ae..07cab02cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -61,6 +61,13 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write( + ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + @Override public void init(BatchData batchData) { batchData.initVectorValues(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 9ecc4f4ad..8896b58a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -32,11 +32,8 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.TypeServices; import org.apache.tsfile.utils.TypeServices.EmptyValueChunkWriter; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -46,7 +43,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.time.LocalDate; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -231,45 +227,8 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) // check isNull by bitMap in tablet ValueChunkWriter valueChunkWriter = tryToAddSeriesWriterInternal(measurementSchemas.get(columnIndex)); - switch (measurementSchemas.get(columnIndex).getType()) { - case BOOLEAN: - valueChunkWriter.write( - time, ((boolean[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case INT32: - valueChunkWriter.write(time, ((int[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case DATE: - valueChunkWriter.write( - time, - isNull - ? 0 - : DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.getValues()[columnIndex])[row]), - isNull); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(time, ((long[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case FLOAT: - valueChunkWriter.write(time, ((float[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case DOUBLE: - valueChunkWriter.write(time, ((double[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueChunkWriter.write(time, ((Binary[]) tablet.getValues()[columnIndex])[row], isNull); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format( - "error.write.type_not_supported", - measurementSchemas.get(columnIndex).getType())); - } + Type.fromTsDataType(measurementSchemas.get(columnIndex).getType()) + .write(valueChunkWriter, time, tablet.getValues()[columnIndex], row, isNull); } // TODO: we can write the null columns after whole insertion, according to the point number // in the time chunk before and after, no need to do it in a row-by-row manner diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 5eeaa9121..7dedd8444 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -212,6 +212,58 @@ public void testWriteValueChunk() { } } + @Test + public void testWriteTabletValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + + Type.fromTsDataType(TSDataType.BOOLEAN) + .write(writer, 1L, new boolean[] {false, true}, 1, false); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, new int[] {0, 1}, 1, false); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DATE) + .write( + writer, + 3L, + new LocalDate[] {LocalDate.of(2026, 7, 14), LocalDate.of(2026, 7, 15)}, + 1, + false); + Mockito.verify(writer).write(3L, 20260715, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 4L, new long[] {0L, 2L}, 1, false); + Mockito.verify(writer).write(4L, 2L, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 5L, new float[] {0F, 1.25F}, 1, false); + Mockito.verify(writer).write(5L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 6L, new double[] {0D, 2.5D}, 1, false); + Mockito.verify(writer).write(6L, 2.5D, false); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Binary[] binaries = {Binary.EMPTY_VALUE, binary}; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 7L, binaries, 1, false); + Mockito.verify(writer).write(7L, binary, false); + Mockito.reset(writer); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 8L, new long[] {1L}, 0, false); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + @Test public void testToString() { Object[][] testCases = { From 1fe0a48007f4b4b1ef643ee9ff895b0bdcbab9df Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 11:55:17 +0800 Subject: [PATCH 055/131] refactor write --- .../read/common/type/AbstractIntType.java | 6 +++ .../read/common/type/AbstractLongType.java | 6 +++ .../read/common/type/AbstractVarcharType.java | 6 +++ .../tsfile/read/common/type/BlobType.java | 6 +++ .../tsfile/read/common/type/BooleanType.java | 6 +++ .../tsfile/read/common/type/DateType.java | 6 +++ .../tsfile/read/common/type/DoubleType.java | 6 +++ .../tsfile/read/common/type/FloatType.java | 6 +++ .../tsfile/read/common/type/ObjectType.java | 6 +++ .../apache/tsfile/read/common/type/Type.java | 7 +++ .../tsfile/read/common/type/VectorType.java | 7 +++ .../chunk/NonAlignedChunkGroupWriterImpl.java | 53 ++----------------- .../tsfile/read/common/type/TypeTest.java | 48 +++++++++++++++++ 13 files changed, 120 insertions(+), 49 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index e9437c880..9dc327dd5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -45,6 +45,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -182,6 +183,11 @@ public void write( writer.write(time, ((int[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((int[]) column)[rowIndex]); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index e9151e5d6..598f81211 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -44,6 +44,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -181,6 +182,11 @@ public void write( writer.write(time, ((long[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((long[]) column)[rowIndex]); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index e538bd773..1c992b4d8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -159,6 +160,11 @@ public void write( writer.write(time, ((Binary[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((Binary[]) column)[rowIndex]); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 1ff6e1505..c44de7b1f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -37,6 +37,7 @@ import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -165,6 +166,11 @@ public void write( writer.write(time, ((Binary[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((Binary[]) column)[rowIndex]); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BLOB); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 2a4b91c29..c6f1d9c6c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -39,6 +39,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; @@ -173,6 +174,11 @@ public void write( writer.write(time, ((boolean[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((boolean[]) column)[rowIndex]); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BOOLEAN); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 80f3cd67c..e2f2f1c0c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -28,6 +28,7 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.utils.TsPrimitiveType.TsInt; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.IntDataPoint; @@ -70,6 +71,11 @@ public void write( isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, DateUtils.parseDateExpressionToInt(((LocalDate[]) column)[rowIndex])); + } + @Override public void addValue(int rowIndex, Object value, Object column) { checkValueType(value, LocalDate.class, "LocalDate"); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 20153d48c..306e78cd4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -44,6 +44,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -184,6 +185,11 @@ public void write( writer.write(time, ((double[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((double[]) column)[rowIndex]); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.DOUBLE); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 795a43a9e..ac670fe78 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -43,6 +43,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -182,6 +183,11 @@ public void write( writer.write(time, ((float[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((float[]) column)[rowIndex]); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.FLOAT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 665a58a1f..afd2ef8e6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -36,6 +36,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -166,6 +167,11 @@ public void write( writer.write(time, ((Binary[]) column)[rowIndex], isNull); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((Binary[]) column)[rowIndex]); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.OBJECT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index d2b7a09b2..2cecaec5e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -34,6 +34,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -259,6 +260,12 @@ default void write( Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes a value at {@code rowIndex} in a Tablet column to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Returns the retained memory size of an array column in Tablet. */ default long arrayRamBytesUsed(Object array) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 07cab02cd..9ea6eec3c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -68,6 +69,12 @@ public void write( Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + @Override public void init(BatchData batchData) { batchData.initVectorValues(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index b525064fa..ea73884d2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -22,13 +22,10 @@ import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.i18n.Messages; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -38,7 +35,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.time.LocalDate; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -118,7 +114,8 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) continue; } String measurementId = timeseries.get(column).getMeasurementName(); - TSDataType tsDataType = timeseries.get(column).getType(); + Type type = Type.fromTsDataType(timeseries.get(column).getType()); + ChunkWriterImpl chunkWriter = chunkWriters.get(measurementId); pointCount = 0; for (int row = startRowIndex; row < endRowIndex; row++) { // check isNull in tablet @@ -130,49 +127,7 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) long time = tablet.getTimestamps()[row]; checkIsHistoryData(measurementId, time); pointCount++; - switch (tsDataType) { - case INT32: - chunkWriters.get(measurementId).write(time, ((int[]) tablet.getValues()[column])[row]); - break; - case DATE: - chunkWriters - .get(measurementId) - .write( - time, - DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.getValues()[column])[row])); - break; - case INT64: - case TIMESTAMP: - chunkWriters.get(measurementId).write(time, ((long[]) tablet.getValues()[column])[row]); - break; - case FLOAT: - chunkWriters - .get(measurementId) - .write(time, ((float[]) tablet.getValues()[column])[row]); - break; - case DOUBLE: - chunkWriters - .get(measurementId) - .write(time, ((double[]) tablet.getValues()[column])[row]); - break; - case BOOLEAN: - chunkWriters - .get(measurementId) - .write(time, ((boolean[]) tablet.getValues()[column])[row]); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkWriters - .get(measurementId) - .write(time, ((Binary[]) tablet.getValues()[column])[row]); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", tsDataType)); - } + type.write(chunkWriter, time, tablet.getValues()[column], row); lastTimeMap.put(measurementId, time); } maxPointCount = Math.max(pointCount, maxPointCount); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 7dedd8444..ae2b43ef4 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -33,6 +33,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -264,6 +265,53 @@ public void testWriteTabletValueChunk() { } } + @Test + public void testWriteNonAlignedTabletValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, new boolean[] {false, true}, 1); + Mockito.verify(writer).write(1L, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, new int[] {0, 1}, 1); + Mockito.verify(writer).write(2L, 1); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DATE) + .write( + writer, 3L, new LocalDate[] {LocalDate.of(2026, 7, 14), LocalDate.of(2026, 7, 15)}, 1); + Mockito.verify(writer).write(3L, 20260715); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 4L, new long[] {0L, 2L}, 1); + Mockito.verify(writer).write(4L, 2L); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 5L, new float[] {0F, 1.25F}, 1); + Mockito.verify(writer).write(5L, 1.25F); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 6L, new double[] {0D, 2.5D}, 1); + Mockito.verify(writer).write(6L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Binary[] binaries = {Binary.EMPTY_VALUE, binary}; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 7L, binaries, 1); + Mockito.verify(writer).write(7L, binary); + Mockito.reset(writer); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 8L, new long[] {1L}, 0); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + @Test public void testToString() { Object[][] testCases = { From 94392a3a10c18af60a70a056cd6d08269c0f2e59 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 12:04:14 +0800 Subject: [PATCH 056/131] refactor write --- .../read/common/type/AbstractIntType.java | 10 +++- .../read/common/type/AbstractLongType.java | 10 +++- .../read/common/type/AbstractVarcharType.java | 10 +++- .../tsfile/read/common/type/BlobType.java | 10 +++- .../tsfile/read/common/type/BooleanType.java | 10 +++- .../tsfile/read/common/type/DateType.java | 4 +- .../tsfile/read/common/type/DoubleType.java | 10 +++- .../tsfile/read/common/type/FloatType.java | 10 +++- .../tsfile/read/common/type/ObjectType.java | 4 +- .../apache/tsfile/read/common/type/Type.java | 9 ++- .../tsfile/read/common/type/VectorType.java | 9 ++- .../write/v4/TableTsBlock2TsFileWriter.java | 35 ++---------- .../tsfile/read/common/type/TypeTest.java | 57 +++++++++++++++++++ 13 files changed, 138 insertions(+), 50 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 9dc327dd5..24f3efacd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -179,8 +179,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((int[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((int[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getInt(rowIndex), isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 598f81211..de33a258f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -178,8 +178,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((long[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((long[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getLong(rowIndex), isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 1c992b4d8..82e3e9c97 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -156,8 +156,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((Binary[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? null : column.getBinary(rowIndex), isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index c44de7b1f..87d36ea74 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -162,8 +162,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((Binary[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? null : column.getBinary(rowIndex), isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index c6f1d9c6c..93248ee10 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -170,8 +170,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((boolean[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((boolean[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? false : column.getBoolean(rowIndex), isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index e2f2f1c0c..ac7f59e9f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -64,10 +64,10 @@ public DataPoint getDataPoint(String measurementId, String value) { @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { writer.write( time, - isNull ? 0 : DateUtils.parseDateExpressionToInt(((LocalDate[]) column)[rowIndex]), + isNull ? 0 : DateUtils.parseDateExpressionToInt(((LocalDate[]) array)[rowIndex]), isNull); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 306e78cd4..a7df9b955 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -181,8 +181,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((double[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((double[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getDouble(rowIndex), isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index ac670fe78..c8c6a4a67 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -179,8 +179,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((float[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((float[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getFloat(rowIndex), isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index afd2ef8e6..eab4869ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -163,8 +163,8 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { - writer.write(time, ((Binary[]) column)[rowIndex], isNull); + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) array)[rowIndex], isNull); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 2cecaec5e..c1785f6b1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -255,7 +255,14 @@ default void write(ValueChunkWriter writer, long time, Object value, boolean isN /** Writes a value at {@code rowIndex} in a Tablet column to an aligned value chunk. */ default void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes a value at {@code rowIndex} in a TsBlock column to an aligned value chunk. */ + default void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { throw new UnSupportedDataTypeException( Messages.format("error.write.type_not_supported", getTypeEnum())); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 9ea6eec3c..1dad96ad4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -64,7 +64,14 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu @Override public void write( - ValueChunkWriter writer, long time, Object column, int rowIndex, boolean isNull) { + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { throw new UnSupportedDataTypeException( Messages.format("error.write.type_not_supported", getTypeEnum())); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java index 8e1561864..6c0503832 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java @@ -28,8 +28,8 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl; import org.apache.tsfile.write.chunk.IChunkGroupWriter; import org.apache.tsfile.write.chunk.TableChunkGroupWriterImpl; @@ -198,13 +198,16 @@ public int getRowCount() { */ private class TableTsBlockChunkGroupWriterImpl extends TableChunkGroupWriterImpl { private final ValueChunkWriter[] valueChunkWriters; + private final Type[] valueTypes; public TableTsBlockChunkGroupWriterImpl(IDeviceID deviceId) throws IOException { super(deviceId); // Initialize ValueChunkWriter for each measurement this.valueChunkWriters = new ValueChunkWriter[fieldColumnSchemas.length]; + this.valueTypes = new Type[fieldColumnSchemas.length]; for (int i = 0; i < fieldColumnSchemas.length; i++) { valueChunkWriters[i] = tryToAddSeriesWriterInternal(fieldColumnSchemas[i]); + valueTypes[i] = Type.fromTsDataType(valueChunkWriters[i].getDataType()); } } @@ -227,35 +230,7 @@ public int write(Column timeColumn, Column[] valueColumns, int startRowIndex, in Column valueColumn = valueColumns[valueColumnIndex]; ValueChunkWriter valueChunkWriter = valueChunkWriters[valueColumnIndex]; boolean isNull = valueColumn.isNull(rowIndex); - switch (valueChunkWriter.getDataType()) { - case BOOLEAN: - valueChunkWriter.write( - time, isNull ? false : valueColumn.getBoolean(rowIndex), isNull); - break; - case INT32: - case DATE: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getInt(rowIndex), isNull); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getLong(rowIndex), isNull); - break; - case FLOAT: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getFloat(rowIndex), isNull); - break; - case DOUBLE: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getDouble(rowIndex), isNull); - break; - case TEXT: - case BLOB: - case STRING: - valueChunkWriter.write(time, isNull ? null : valueColumn.getBinary(rowIndex), isNull); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format( - "error.write.type_not_supported", valueChunkWriter.getDataType().getType())); - } + valueTypes[valueColumnIndex].write(valueChunkWriter, time, valueColumn, rowIndex, isNull); } timeChunkWriter.write(time); lastTime = time; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index ae2b43ef4..1c14649db 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -265,6 +265,63 @@ public void testWriteTabletValueChunk() { } } + @Test + public void testWriteTsBlockValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + Column column = Mockito.mock(Column.class); + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Mockito.when(column.getBoolean(1)).thenReturn(true); + Mockito.when(column.getInt(1)).thenReturn(1); + Mockito.when(column.getLong(1)).thenReturn(2L); + Mockito.when(column.getFloat(1)).thenReturn(1.25F); + Mockito.when(column.getDouble(1)).thenReturn(2.5D); + Mockito.when(column.getBinary(1)).thenReturn(binary); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, column, 1, false); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT32, TSDataType.DATE}) { + Type.fromTsDataType(dataType).write(writer, 2L, column, 1, false); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT64, TSDataType.TIMESTAMP}) { + Type.fromTsDataType(dataType).write(writer, 3L, column, 1, false); + Mockito.verify(writer).write(3L, 2L, false); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 4L, column, 1, false); + Mockito.verify(writer).write(4L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 5L, column, 1, false); + Mockito.verify(writer).write(5L, 2.5D, false); + Mockito.reset(writer); + + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB}) { + Type.fromTsDataType(dataType).write(writer, 6L, column, 1, false); + Mockito.verify(writer).write(6L, binary, false); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.INT32).write(writer, 7L, column, 1, true); + Mockito.verify(writer).write(7L, 0, true); + Mockito.reset(writer); + + for (TSDataType dataType : new TSDataType[] {TSDataType.OBJECT, TSDataType.VECTOR}) { + try { + Type.fromTsDataType(dataType).write(writer, 8L, column, 1, false); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + @Test public void testWriteNonAlignedTabletValueChunk() { ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); From 6a49281fb97b13d9801cce0983f74fdaa6f5ed63 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 12:10:16 +0800 Subject: [PATCH 057/131] refactor write --- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 6 ++ .../tsfile/read/common/type/VectorType.java | 6 ++ .../write/chunk/AlignedChunkWriterImpl.java | 42 ++++-------- .../tsfile/read/common/type/TypeTest.java | 64 +++++++++++++++++++ 12 files changed, 129 insertions(+), 29 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 24f3efacd..514a300b8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -177,6 +177,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, isNull ? 0 : (int) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getInt() : Integer.MAX_VALUE, value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index de33a258f..b5e761534 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -176,6 +176,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (long) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getLong() : Long.MAX_VALUE, value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 82e3e9c97..dcf26d8e5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -154,6 +154,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (Binary) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getBinary() : Binary.EMPTY_VALUE, value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 87d36ea74..b9ac45e79 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -160,6 +160,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (Binary) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getBinary() : Binary.EMPTY_VALUE, value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 93248ee10..6355a944e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -168,6 +168,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (boolean) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null && value.getBoolean(), value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index a7df9b955..ed5da69a1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -179,6 +179,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (double) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getDouble() : Double.MAX_VALUE, value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index c8c6a4a67..e5214a672 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -177,6 +177,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (float) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getFloat() : Float.MAX_VALUE, value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index eab4869ae..7d61c7ec7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -161,6 +161,11 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu writer.write(time, (Binary) value, isNull); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getBinary() : Binary.EMPTY_VALUE, value == null); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index c1785f6b1..66e71d282 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -253,6 +253,12 @@ default void write(ValueChunkWriter writer, long time, Object value, boolean isN Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes a TsPrimitiveType value to an aligned value chunk. */ + default void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Writes a value at {@code rowIndex} in a Tablet column to an aligned value chunk. */ default void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 1dad96ad4..713513462 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -62,6 +62,12 @@ public void write(ValueChunkWriter writer, long time, Object value, boolean isNu Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java index 17c74cf62..12e19a5e6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -31,6 +31,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -39,7 +40,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -48,6 +48,7 @@ public class AlignedChunkWriterImpl implements IChunkWriter { protected TimeChunkWriter timeChunkWriter; protected List valueChunkWriterList; + private Type[] valueTypes; protected int valueIndex; protected EncryptParameter encryptParam; @@ -303,41 +304,24 @@ public void write(long time, Binary value, boolean isNull, int valueIndex) { } public void write(long time, TsPrimitiveType[] points) { + if (valueTypes == null) { + initValueTypes(); + } valueIndex = 0; for (TsPrimitiveType point : points) { ValueChunkWriter writer = valueChunkWriterList.get(valueIndex++); - switch (writer.getDataType()) { - case INT64: - case TIMESTAMP: - writer.write(time, point != null ? point.getLong() : Long.MAX_VALUE, point == null); - break; - case INT32: - case DATE: - writer.write(time, point != null ? point.getInt() : Integer.MAX_VALUE, point == null); - break; - case FLOAT: - writer.write(time, point != null ? point.getFloat() : Float.MAX_VALUE, point == null); - break; - case DOUBLE: - writer.write(time, point != null ? point.getDouble() : Double.MAX_VALUE, point == null); - break; - case BOOLEAN: - writer.write(time, point != null ? point.getBoolean() : false, point == null); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - writer.write( - time, - point != null ? point.getBinary() : new Binary("".getBytes(StandardCharsets.UTF_8)), - point == null); - break; - } + valueTypes[valueIndex - 1].write(writer, time, point); } write(time); } + private void initValueTypes() { + valueTypes = new Type[valueChunkWriterList.size()]; + for (int i = 0; i < valueChunkWriterList.size(); i++) { + valueTypes[i] = Type.fromTsDataType(valueChunkWriterList.get(i).getDataType()); + } + } + public void write(long time) { valueIndex = 0; timeChunkWriter.write(time); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 1c14649db..c0a23b163 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -213,6 +213,70 @@ public void testWriteValueChunk() { } } + @Test + public void testWriteTsPrimitiveValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, new TsPrimitiveType.TsBoolean(true)); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, new TsPrimitiveType.TsInt(1)); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 3L, new TsPrimitiveType.TsLong(2L)); + Mockito.verify(writer).write(3L, 2L, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 4L, new TsPrimitiveType.TsFloat(1.25F)); + Mockito.verify(writer).write(4L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 5L, new TsPrimitiveType.TsDouble(2.5D)); + Mockito.verify(writer).write(5L, 2.5D, false); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 6L, new TsPrimitiveType.TsBinary(binary)); + Mockito.verify(writer).write(6L, binary, false); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.INT32).write(writer, 7L, (TsPrimitiveType) null); + Mockito.verify(writer).write(7L, Integer.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 8L, (TsPrimitiveType) null); + Mockito.verify(writer).write(8L, Long.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 9L, (TsPrimitiveType) null); + Mockito.verify(writer).write(9L, Float.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 10L, (TsPrimitiveType) null); + Mockito.verify(writer).write(10L, Double.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 11L, (TsPrimitiveType) null); + Mockito.verify(writer).write(11L, false, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TEXT).write(writer, 12L, (TsPrimitiveType) null); + Mockito.verify(writer).write(12L, Binary.EMPTY_VALUE, true); + Mockito.reset(writer); + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 13L, (TsPrimitiveType) null); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + @Test public void testWriteTabletValueChunk() { ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); From 908778e41745d517b1998dda522d403b5b91797a Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 12:16:56 +0800 Subject: [PATCH 058/131] refactor batchWrite --- .../read/common/type/AbstractIntType.java | 6 ++ .../read/common/type/AbstractLongType.java | 6 ++ .../read/common/type/AbstractVarcharType.java | 6 ++ .../tsfile/read/common/type/BlobType.java | 6 ++ .../tsfile/read/common/type/BooleanType.java | 6 ++ .../tsfile/read/common/type/DoubleType.java | 6 ++ .../tsfile/read/common/type/FloatType.java | 6 ++ .../tsfile/read/common/type/ObjectType.java | 6 ++ .../apache/tsfile/read/common/type/Type.java | 7 +++ .../tsfile/read/common/type/VectorType.java | 7 +++ .../write/chunk/AlignedChunkWriterImpl.java | 34 ++--------- .../tsfile/read/common/type/TypeTest.java | 59 +++++++++++++++++++ .../writer/AlignedChunkWriterImplTest.java | 28 +++++++++ 13 files changed, 153 insertions(+), 30 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 514a300b8..ae9ff49b9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -194,6 +194,12 @@ public void write( writer.write(time, isNull ? 0 : column.getInt(rowIndex), isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getInts(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((int[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index b5e761534..082b000d7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -193,6 +193,12 @@ public void write( writer.write(time, isNull ? 0 : column.getLong(rowIndex), isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getLongs(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((long[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index dcf26d8e5..bbfe88555 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -171,6 +171,12 @@ public void write( writer.write(time, isNull ? null : column.getBinary(rowIndex), isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((Binary[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index b9ac45e79..f3c568367 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -177,6 +177,12 @@ public void write( writer.write(time, isNull ? null : column.getBinary(rowIndex), isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((Binary[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 6355a944e..839a0d56f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -185,6 +185,12 @@ public void write( writer.write(time, isNull ? false : column.getBoolean(rowIndex), isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBooleans(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((boolean[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index ed5da69a1..b4bef243e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -196,6 +196,12 @@ public void write( writer.write(time, isNull ? 0 : column.getDouble(rowIndex), isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getDoubles(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((double[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index e5214a672..ff6678079 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -194,6 +194,12 @@ public void write( writer.write(time, isNull ? 0 : column.getFloat(rowIndex), isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getFloats(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((float[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 7d61c7ec7..d4efe794f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -172,6 +172,12 @@ public void write( writer.write(time, ((Binary[]) array)[rowIndex], isNull); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { writer.write(time, ((Binary[]) column)[rowIndex]); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 66e71d282..70873f6e4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -273,6 +273,13 @@ default void write( Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes a batch of values in a TsBlock column to an aligned value chunk. */ + default void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + throw new UnsupportedOperationException( + Messages.format("error.write.chunk_unknown_type", getTypeEnum())); + } + /** Writes a value at {@code rowIndex} in a Tablet column to a non-aligned chunk. */ default void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 713513462..1e360bfd4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -82,6 +82,13 @@ public void write( Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + throw new UnsupportedOperationException( + Messages.format("error.write.chunk_unknown_type", getTypeEnum())); + } + @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java index 12e19a5e6..ecdf14151 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -29,7 +29,6 @@ import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; @@ -335,6 +334,9 @@ public void writeTime(long time) { } public void write(TimeColumn timeColumn, Column[] valueColumns, int batchSize) { + if (valueTypes == null) { + initValueTypes(); + } if (remainingPointsNumber < batchSize) { int pointsHasWritten = (int) remainingPointsNumber; batchWrite(timeColumn, valueColumns, pointsHasWritten, 0); @@ -351,35 +353,7 @@ private void batchWrite( for (Column column : valueColumns) { ValueChunkWriter chunkWriter = valueChunkWriterList.get(valueIndex++); - TSDataType tsDataType = chunkWriter.getDataType(); - switch (tsDataType) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkWriter.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); - break; - case DOUBLE: - chunkWriter.write(times, column.getDoubles(), column.isNull(), batchSize, arrayOffset); - break; - case BOOLEAN: - chunkWriter.write(times, column.getBooleans(), column.isNull(), batchSize, arrayOffset); - break; - case INT64: - case TIMESTAMP: - chunkWriter.write(times, column.getLongs(), column.isNull(), batchSize, arrayOffset); - break; - case INT32: - case DATE: - chunkWriter.write(times, column.getInts(), column.isNull(), batchSize, arrayOffset); - break; - case FLOAT: - chunkWriter.write(times, column.getFloats(), column.isNull(), batchSize, arrayOffset); - break; - default: - throw new UnsupportedOperationException( - Messages.format("error.write.chunk_unknown_type", tsDataType)); - } + valueTypes[valueIndex - 1].write(chunkWriter, times, column, batchSize, arrayOffset); } write(times, batchSize, arrayOffset); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index c0a23b163..ccc3c6df5 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -386,6 +386,65 @@ public void testWriteTsBlockValueChunk() { } } + @Test + public void testWriteTsBlockValueChunkBatch() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + Column column = Mockito.mock(Column.class); + long[] times = {1L, 2L}; + boolean[] isNull = {false, true}; + boolean[] booleans = {true, false}; + int[] ints = {1, 0}; + long[] longs = {2L, 0L}; + float[] floats = {1.25F, 0F}; + double[] doubles = {2.5D, 0D}; + Binary[] binaries = {new Binary("value", TSFileConfig.STRING_CHARSET), Binary.EMPTY_VALUE}; + Mockito.when(column.isNull()).thenReturn(isNull); + Mockito.when(column.getBooleans()).thenReturn(booleans); + Mockito.when(column.getInts()).thenReturn(ints); + Mockito.when(column.getLongs()).thenReturn(longs); + Mockito.when(column.getFloats()).thenReturn(floats); + Mockito.when(column.getDoubles()).thenReturn(doubles); + Mockito.when(column.getBinaries()).thenReturn(binaries); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, booleans, isNull, 1, 1); + Mockito.reset(writer); + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT32, TSDataType.DATE}) { + Type.fromTsDataType(dataType).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, ints, isNull, 1, 1); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT64, TSDataType.TIMESTAMP}) { + Type.fromTsDataType(dataType).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, longs, isNull, 1, 1); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, floats, isNull, 1, 1); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, doubles, isNull, 1, 1); + Mockito.reset(writer); + + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, binaries, isNull, 1, 1); + Mockito.reset(writer); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, times, column, 1, 1); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + @Test public void testWriteNonAlignedTabletValueChunk() { ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java index 749141235..5a8dd5a99 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java @@ -22,6 +22,9 @@ import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,12 +34,37 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; public class AlignedChunkWriterImplTest { + @Test + public void testBatchWrite() { + AlignedChunkWriterImpl chunkWriter = + new AlignedChunkWriterImpl(new VectorMeasurementSchemaStub()); + TsBlockBuilder builder = + new TsBlockBuilder(Arrays.asList(TSDataType.FLOAT, TSDataType.INT32, TSDataType.DOUBLE)); + for (int time = 1; time <= 20; time++) { + builder.getTimeColumnBuilder().writeLong(time); + builder.getValueColumnBuilders()[0].writeFloat(time); + builder.getValueColumnBuilders()[1].writeInt(time); + builder.getValueColumnBuilders()[2].writeDouble(time); + builder.declarePosition(); + } + TsBlock tsBlock = builder.build(); + + chunkWriter.write( + (TimeColumn) tsBlock.getTimeColumn(), + tsBlock.getValueColumns(), + tsBlock.getPositionCount()); + chunkWriter.sealCurrentPage(); + + assertEquals(485, chunkWriter.getSerializedChunkSize()); + } + @Test public void testWrite1() { VectorMeasurementSchemaStub measurementSchema = new VectorMeasurementSchemaStub(); From 276d21c25ce3221db02e983060f69cf4fd5681d7 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 12:25:01 +0800 Subject: [PATCH 059/131] refactor writeWithTsRecord --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 7 ++++ .../tsfile/read/common/type/VectorType.java | 9 +++++ .../tsfile/utils/TsFileGeneratorUtils.java | 38 ++----------------- .../tsfile/read/common/type/TypeTest.java | 25 ++++++++++++ 9 files changed, 70 insertions(+), 34 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index ae9ff49b9..ee74d2bcc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -172,6 +172,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new IntDataPoint(measurementId, Integer.parseInt(value)); } + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new IntDataPoint(measurementId, (int) value); + } + @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { writer.write(time, isNull ? 0 : (int) value, isNull); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 082b000d7..f748ec7c7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -171,6 +171,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new LongDataPoint(measurementId, Long.parseLong(value)); } + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new LongDataPoint(measurementId, value); + } + @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { writer.write(time, (long) value, isNull); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 839a0d56f..870ad9bed 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -163,6 +163,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new BooleanDataPoint(measurementId, Boolean.parseBoolean(value)); } + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new BooleanDataPoint(measurementId, true); + } + @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { writer.write(time, (boolean) value, isNull); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index b4bef243e..3df7586f0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -174,6 +174,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new DoubleDataPoint(measurementId, Double.parseDouble(value)); } + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new DoubleDataPoint(measurementId, (double) value); + } + @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { writer.write(time, (double) value, isNull); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index ff6678079..3df52108b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -172,6 +172,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new FloatDataPoint(measurementId, Float.parseFloat(value)); } + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new FloatDataPoint(measurementId, (float) value); + } + @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { writer.write(time, (float) value, isNull); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 70873f6e4..6e894b396 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -38,6 +38,7 @@ import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -247,6 +248,12 @@ default DataPoint getDataPoint(String measurementId, String value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Creates a data point from a long value for test file generation. */ + default DataPoint getDataPoint(String measurementId, long value) { + return new StringDataPoint( + measurementId, new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + } + /** Writes a value to an aligned value chunk. */ default void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 1e360bfd4..9075b467a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; import org.apache.tsfile.encoding.decoder.LongChimpDecoder; @@ -32,6 +33,7 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -39,6 +41,7 @@ import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; import java.io.DataOutputStream; import java.io.IOException; @@ -56,6 +59,12 @@ public DataPoint getDataPoint(String measurementId, String value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new StringDataPoint( + measurementId, new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + } + @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java index 6b941e73f..6d1dd9e15 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java @@ -27,15 +27,11 @@ import org.apache.tsfile.fileSystem.fsFactory.FSFactory; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; import org.apache.tsfile.write.record.datapoint.DataPoint; -import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; -import org.apache.tsfile.write.record.datapoint.FloatDataPoint; -import org.apache.tsfile.write.record.datapoint.IntDataPoint; -import org.apache.tsfile.write.record.datapoint.LongDataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -68,35 +64,9 @@ public static void writeWithTsRecord( // construct TsRecord TSRecord tsRecord = new TSRecord(deviceId, time); for (IMeasurementSchema schema : schemas) { - DataPoint dPoint; - switch (schema.getType()) { - case INT64: - case TIMESTAMP: - dPoint = new LongDataPoint(schema.getMeasurementName(), startValue); - break; - case INT32: - case DATE: - dPoint = new IntDataPoint(schema.getMeasurementName(), (int) startValue); - break; - case DOUBLE: - dPoint = new DoubleDataPoint(schema.getMeasurementName(), (double) startValue); - break; - case FLOAT: - dPoint = new FloatDataPoint(schema.getMeasurementName(), (float) startValue); - break; - case BOOLEAN: - dPoint = new BooleanDataPoint(schema.getMeasurementName(), true); - break; - case TEXT: - case BLOB: - case STRING: - default: - dPoint = - new StringDataPoint( - schema.getMeasurementName(), - new Binary(String.valueOf(startValue), TSFileConfig.STRING_CHARSET)); - break; - } + DataPoint dPoint = + Type.fromTsDataType(schema.getType()) + .getDataPoint(schema.getMeasurementName(), startValue); tsRecord.addTuple(dPoint); } // write diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index ccc3c6df5..c5e1a460f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -169,6 +169,31 @@ public void testGetDataPoint() { } } + @Test + public void testGetDataPointFromLong() { + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 1}, + {TSDataType.INT64, 1L}, + {TSDataType.TIMESTAMP, 1L}, + {TSDataType.FLOAT, 1.0F}, + {TSDataType.DOUBLE, 1.0D}, + {TSDataType.TEXT, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.STRING, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.BLOB, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.OBJECT, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.VECTOR, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.UNKNOWN, new Binary("1", TSFileConfig.STRING_CHARSET)} + }; + + for (Object[] testCase : testCases) { + DataPoint dataPoint = Type.fromTsDataType((TSDataType) testCase[0]).getDataPoint("s", 1L); + Assert.assertEquals("s", dataPoint.getMeasurementId()); + Assert.assertEquals(testCase[1], dataPoint.getValue()); + } + } + @Test public void testWriteValueChunk() { ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); From b455d28339601f2dcea2bbe07ecccc2d0162c61e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 12:29:36 +0800 Subject: [PATCH 060/131] refactor getEncoder --- .../encoding/encoder/TSEncodingBuilder.java | 18 ++--------- .../org/apache/tsfile/utils/TypeServices.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index 77d6568ee..9804dd0a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -25,6 +25,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.TypeServices; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; @@ -143,21 +145,7 @@ public static class Rle extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - case BOOLEAN: - return new IntRleEncoder(); - case INT64: - case TIMESTAMP: - return new LongRleEncoder(); - case FLOAT: - case DOUBLE: - return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.RLE, type)); - } + return TypeServices.RLE_ENCODER_SERVICE.call(Type.fromTsDataType(type)).build(maxPointNumber); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index cae6b0c63..08c720d51 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -21,7 +21,12 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.encoder.Encoder; +import org.apache.tsfile.encoding.encoder.FloatEncoder; +import org.apache.tsfile.encoding.encoder.IntRleEncoder; +import org.apache.tsfile.encoding.encoder.LongRleEncoder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.TsBlockBuilder; @@ -438,6 +443,27 @@ public final class TypeServices { }; }; + public static final TypeService RLE_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE, BOOLEAN -> maxPointNumber -> new IntRleEncoder(); + case INT64, TIMESTAMP -> maxPointNumber -> new LongRleEncoder(); + case FLOAT -> + maxPointNumber -> + new FloatEncoder(TSEncoding.RLE, TSDataType.FLOAT, maxPointNumber); + case DOUBLE -> + maxPointNumber -> + new FloatEncoder(TSEncoding.RLE, TSDataType.DOUBLE, maxPointNumber); + case TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + maxPointNumber -> { + throw new UnSupportedDataTypeException( + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.RLE, + type.getTypeEnum())); + }; + }; + private TypeServices() {} @FunctionalInterface @@ -491,6 +517,12 @@ public interface EmptyValueChunkWriter { void write(ValueChunkWriter writer); } + @FunctionalInterface + public interface EncoderBuilder { + + Encoder build(int maxPointNumber); + } + public enum PageDataReadStatus { CONTINUE, FILTERED, From 35d33499d2d49587958a538941788f6c991078e1 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 17:25:27 +0800 Subject: [PATCH 061/131] refactor getEncoder --- .../encoding/encoder/TSEncodingBuilder.java | 17 ++-------- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DateType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../org/apache/tsfile/utils/TypeServices.java | 22 +++++++++++++ .../tsfile/read/common/type/TypeTest.java | 32 +++++++++++++++++++ 13 files changed, 107 insertions(+), 14 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index 9804dd0a3..25fb657e5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -186,20 +186,9 @@ public static class Ts2Diff extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new DeltaBinaryEncoder.IntDeltaEncoder(); - case INT64: - case TIMESTAMP: - return new DeltaBinaryEncoder.LongDeltaEncoder(); - case FLOAT: - case DOUBLE: - return new FloatEncoder(TSEncoding.TS_2DIFF, type, maxPointNumber); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.TS_2DIFF, type)); - } + return TypeServices.TS_2DIFF_ENCODER_SERVICE + .call(Type.fromTsDataType(type)) + .build(maxPointNumber); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index ee74d2bcc..8604ef81d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -266,6 +266,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((int[]) column)[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((int[]) target)[targetIndex] = ((int[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new int[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index f748ec7c7..53344447d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -265,6 +265,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((long[]) column)[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((long[]) target)[targetIndex] = ((long[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new long[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index bbfe88555..e1030206c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -207,6 +207,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((Binary[]) column)[rowIndex] = toBinaryValue(value); } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; + } + protected Binary toBinaryValue(Object value) { if (value != null && !(value instanceof Binary) && !(value instanceof String)) { throw new IllegalArgumentException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index f3c568367..da6ccc249 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -235,6 +235,11 @@ public void addValue(int rowIndex, Object value, Object column) { } } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new Binary[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 870ad9bed..8c4033b3f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -234,6 +234,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((boolean[]) column)[rowIndex] = value != null && (boolean) value; } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((boolean[]) target)[targetIndex] = ((boolean[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new boolean[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index ac7f59e9f..ed21a3c3d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -82,6 +82,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((LocalDate[]) column)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((LocalDate[]) target)[targetIndex] = ((LocalDate[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new LocalDate[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 3df7586f0..339732b3b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -275,6 +275,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((double[]) column)[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((double[]) target)[targetIndex] = ((double[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new double[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 3df52108b..b80fbb0f4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -273,6 +273,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((float[]) column)[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((float[]) target)[targetIndex] = ((float[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new float[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index d4efe794f..1345101ee 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -230,6 +230,11 @@ public void addValue(int rowIndex, Object value, Object column) { } } + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; + } + @Override public Object createArray(int capacity) { return new Binary[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 6e894b396..7f34536f4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -174,6 +174,11 @@ default void addValue(int rowIndex, Object value, Object column) { throw new UnsupportedOperationException(getClass().getName()); } + /** Copies one element from {@code source} to {@code target}. */ + default void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Creates an array column with {@code capacity} entries for this type. */ default Object createArray(int capacity) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 08c720d51..6a5b58617 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.encoder.DeltaBinaryEncoder; import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.encoding.encoder.FloatEncoder; import org.apache.tsfile.encoding.encoder.IntRleEncoder; @@ -464,6 +465,27 @@ public final class TypeServices { }; }; + public static final TypeService TS_2DIFF_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> maxPointNumber -> new DeltaBinaryEncoder.IntDeltaEncoder(); + case INT64, TIMESTAMP -> maxPointNumber -> new DeltaBinaryEncoder.LongDeltaEncoder(); + case FLOAT -> + maxPointNumber -> + new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT, maxPointNumber); + case DOUBLE -> + maxPointNumber -> + new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.DOUBLE, maxPointNumber); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + maxPointNumber -> { + throw new UnSupportedDataTypeException( + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.TS_2DIFF, + type.getTypeEnum())); + }; + }; + private TypeServices() {} @FunctionalInterface diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index c5e1a460f..bd0928537 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -42,6 +42,7 @@ import org.junit.Test; import org.mockito.Mockito; +import java.lang.reflect.Array; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.LocalDate; @@ -847,6 +848,37 @@ public void testEstimateArraySize() { } } + @Test + public void testCopyArrayElement() { + LocalDate date = LocalDate.of(2026, 7, 15); + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new boolean[] {false, true}}, + {TSDataType.INT32, new int[] {0, 1}}, + {TSDataType.DATE, new LocalDate[] {null, date}}, + {TSDataType.INT64, new long[] {0L, 1L}}, + {TSDataType.TIMESTAMP, new long[] {0L, 1L}}, + {TSDataType.FLOAT, new float[] {0.0F, 1.0F}}, + {TSDataType.DOUBLE, new double[] {0.0D, 1.0D}}, + {TSDataType.TEXT, new Binary[] {null, binary}}, + {TSDataType.STRING, new Binary[] {null, binary}}, + {TSDataType.BLOB, new Binary[] {null, binary}}, + {TSDataType.OBJECT, new Binary[] {null, binary}} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object source = testCase[1]; + Object target = type.createArray(2); + + type.copyArrayElement(source, 1, target, 0); + type.copyArrayElement(source, 0, target, 1); + + Assert.assertEquals(Array.get(source, 1), Array.get(target, 0)); + Assert.assertEquals(Array.get(source, 0), Array.get(target, 1)); + } + } + @Test public void testEstimateValueSize() { Object[][] testCases = { From 0f17adbf2672d94884e3a3d33d86a79bc8760908 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 17:47:04 +0800 Subject: [PATCH 062/131] add Type.copyArrayElement --- .../read/common/type/AbstractIntType.java | 10 ++++++++++ .../read/common/type/AbstractLongType.java | 10 ++++++++++ .../read/common/type/AbstractVarcharType.java | 10 ++++++++++ .../tsfile/read/common/type/BlobType.java | 10 ++++++++++ .../tsfile/read/common/type/BooleanType.java | 10 ++++++++++ .../tsfile/read/common/type/DateType.java | 10 ++++++++++ .../tsfile/read/common/type/DoubleType.java | 10 ++++++++++ .../tsfile/read/common/type/FloatType.java | 10 ++++++++++ .../tsfile/read/common/type/ObjectType.java | 10 ++++++++++ .../apache/tsfile/read/common/type/Type.java | 6 ++++++ .../tsfile/read/common/type/VectorType.java | 10 ++++++++++ .../tsfile/read/common/type/TypeTest.java | 17 ++++++++++++++++- 12 files changed, 122 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 8604ef81d..7c18136dd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -56,6 +56,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -271,6 +272,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((int[]) target)[targetIndex] = ((int[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + int[] sourceArray = (int[]) source; + int[] targetArray = (int[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new int[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 53344447d..e15c09253 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -55,6 +55,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -270,6 +271,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((long[]) target)[targetIndex] = ((long[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + long[] sourceArray = (long[]) source; + long[] targetArray = (long[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new long[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index e1030206c..2c2270970 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -45,6 +45,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -212,6 +213,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + Binary[] sourceArray = (Binary[]) source; + Binary[] targetArray = (Binary[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + protected Binary toBinaryValue(Object value) { if (value != null && !(value instanceof Binary) && !(value instanceof String)) { throw new IllegalArgumentException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index da6ccc249..6ad652cea 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -240,6 +241,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + Binary[] sourceArray = (Binary[]) source; + Binary[] targetArray = (Binary[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new Binary[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 8c4033b3f..6d38019aa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -50,6 +50,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -239,6 +240,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((boolean[]) target)[targetIndex] = ((boolean[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + boolean[] sourceArray = (boolean[]) source; + boolean[] targetArray = (boolean[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new boolean[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index ed21a3c3d..13efb0cd0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -38,6 +38,7 @@ import java.nio.ByteBuffer; import java.time.LocalDate; import java.util.Arrays; +import java.util.Iterator; import java.util.Optional; public class DateType extends AbstractIntType { @@ -87,6 +88,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((LocalDate[]) target)[targetIndex] = ((LocalDate[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + LocalDate[] sourceArray = (LocalDate[]) source; + LocalDate[] targetArray = (LocalDate[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new LocalDate[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 339732b3b..2db65d3bc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -55,6 +55,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -280,6 +281,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((double[]) target)[targetIndex] = ((double[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + double[] sourceArray = (double[]) source; + double[] targetArray = (double[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new double[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index b80fbb0f4..3193180f6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -54,6 +54,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -278,6 +279,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((float[]) target)[targetIndex] = ((float[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + float[] sourceArray = (float[]) source; + float[] targetArray = (float[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new float[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 1345101ee..b3507e140 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -235,6 +236,15 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + Binary[] sourceArray = (Binary[]) source; + Binary[] targetArray = (Binary[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public Object createArray(int capacity) { return new Binary[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 7f34536f4..aaeae5635 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -43,6 +43,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Iterator; import java.util.List; public interface Type { @@ -179,6 +180,11 @@ default void copyArrayElement(Object source, int sourceIndex, Object target, int throw new UnsupportedOperationException(getClass().getName()); } + /** Copies the selected elements from {@code source} to {@code target} in iterator order. */ + default void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Creates an array column with {@code capacity} entries for this type. */ default Object createArray(int capacity) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 9075b467a..e97ddd517 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -46,6 +46,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Iterator; public class VectorType extends AbstractLongType { @@ -77,6 +78,15 @@ public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { + long[] sourceArray = (long[]) source; + long[] targetArray = (long[]) target; + for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { + targetArray[targetIndex] = sourceArray[sourceIndex.next()]; + } + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index bd0928537..8206703a6 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -863,7 +863,8 @@ public void testCopyArrayElement() { {TSDataType.TEXT, new Binary[] {null, binary}}, {TSDataType.STRING, new Binary[] {null, binary}}, {TSDataType.BLOB, new Binary[] {null, binary}}, - {TSDataType.OBJECT, new Binary[] {null, binary}} + {TSDataType.OBJECT, new Binary[] {null, binary}}, + {TSDataType.VECTOR, new long[] {0L, 1L}} }; for (Object[] testCase : testCases) { @@ -876,6 +877,20 @@ public void testCopyArrayElement() { Assert.assertEquals(Array.get(source, 1), Array.get(target, 0)); Assert.assertEquals(Array.get(source, 0), Array.get(target, 1)); + + Object iteratorTarget = type.createArray(2); + type.copyArrayElement(source, Arrays.asList(1, 0).iterator(), iteratorTarget); + + Assert.assertEquals(Array.get(source, 1), Array.get(iteratorTarget, 0)); + Assert.assertEquals(Array.get(source, 0), Array.get(iteratorTarget, 1)); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN) + .copyArrayElement(new Object[0], Arrays.asList().iterator(), new Object[0]); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. } } From 834c6932fde3e2c6ca8b74e5a61ed2138b3a8e63 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 18:09:06 +0800 Subject: [PATCH 063/131] add Type.arrayCopyOf --- .../tsfile/read/common/type/AbstractIntType.java | 5 +++++ .../read/common/type/AbstractLongType.java | 5 +++++ .../read/common/type/AbstractVarcharType.java | 6 ++++++ .../apache/tsfile/read/common/type/BlobType.java | 6 ++++++ .../tsfile/read/common/type/BooleanType.java | 5 +++++ .../apache/tsfile/read/common/type/DateType.java | 5 +++++ .../tsfile/read/common/type/DoubleType.java | 5 +++++ .../tsfile/read/common/type/FloatType.java | 5 +++++ .../tsfile/read/common/type/ObjectType.java | 6 ++++++ .../org/apache/tsfile/read/common/type/Type.java | 5 +++++ .../tsfile/read/common/type/VectorType.java | 6 ++++++ .../apache/tsfile/read/common/type/TypeTest.java | 16 ++++++++++++++++ 12 files changed, 75 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 7c18136dd..4da21a7fe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -281,6 +281,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((int[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new int[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index e15c09253..24870769a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -280,6 +280,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((long[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new long[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 2c2270970..ea3810b42 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -44,6 +44,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -222,6 +223,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((Binary[]) array, newLength); + } + protected Binary toBinaryValue(Object value) { if (value != null && !(value instanceof Binary) && !(value instanceof String)) { throw new IllegalArgumentException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 6ad652cea..1d7adf81f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -46,6 +46,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -250,6 +251,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((Binary[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new Binary[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 6d38019aa..cdc768164 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -249,6 +249,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((boolean[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new boolean[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 13efb0cd0..cc8922fa8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -97,6 +97,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((LocalDate[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new LocalDate[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 2db65d3bc..e055e2f25 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -290,6 +290,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((double[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new double[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 3193180f6..6ca2046d0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -288,6 +288,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((float[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new float[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index b3507e140..0ece4044c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -45,6 +45,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -245,6 +246,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((Binary[]) array, newLength); + } + @Override public Object createArray(int capacity) { return new Binary[capacity]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index aaeae5635..e43253d2c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -185,6 +185,11 @@ default void copyArrayElement(Object source, Iterator sourceIndex, Obje throw new UnsupportedOperationException(getClass().getName()); } + /** Copies {@code array}, truncating or padding it to {@code newLength}. */ + default Object arrayCopyOf(Object array, int newLength) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Creates an array column with {@code capacity} entries for this type. */ default Object createArray(int capacity) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index e97ddd517..76747ce7a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -46,6 +46,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Iterator; public class VectorType extends AbstractLongType { @@ -87,6 +88,11 @@ public void copyArrayElement(Object source, Iterator sourceIndex, Objec } } + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((long[]) array, newLength); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 8206703a6..8688ab55d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -883,6 +883,15 @@ public void testCopyArrayElement() { Assert.assertEquals(Array.get(source, 1), Array.get(iteratorTarget, 0)); Assert.assertEquals(Array.get(source, 0), Array.get(iteratorTarget, 1)); + + Object expanded = type.arrayCopyOf(source, 3); + Assert.assertEquals(3, Array.getLength(expanded)); + Assert.assertEquals(Array.get(source, 0), Array.get(expanded, 0)); + Assert.assertEquals(Array.get(source, 1), Array.get(expanded, 1)); + + Object truncated = type.arrayCopyOf(source, 1); + Assert.assertEquals(1, Array.getLength(truncated)); + Assert.assertEquals(Array.get(source, 0), Array.get(truncated, 0)); } try { @@ -892,6 +901,13 @@ public void testCopyArrayElement() { } catch (UnsupportedOperationException ignored) { // Expected. } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).arrayCopyOf(new Object[0], 1); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } } @Test From 37c8f36824456bbb33d2efbd19410b7f3efbdc4b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 18:13:42 +0800 Subject: [PATCH 064/131] remove Type.copyArrayElement --- .../tsfile/read/common/type/AbstractIntType.java | 10 ---------- .../tsfile/read/common/type/AbstractLongType.java | 10 ---------- .../read/common/type/AbstractVarcharType.java | 10 ---------- .../apache/tsfile/read/common/type/BlobType.java | 10 ---------- .../tsfile/read/common/type/BooleanType.java | 10 ---------- .../apache/tsfile/read/common/type/DateType.java | 10 ---------- .../apache/tsfile/read/common/type/DoubleType.java | 10 ---------- .../apache/tsfile/read/common/type/FloatType.java | 10 ---------- .../apache/tsfile/read/common/type/ObjectType.java | 10 ---------- .../org/apache/tsfile/read/common/type/Type.java | 6 ------ .../apache/tsfile/read/common/type/VectorType.java | 10 ---------- .../apache/tsfile/read/common/type/TypeTest.java | 14 -------------- 12 files changed, 120 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 4da21a7fe..3d2e3fdd6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -56,7 +56,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -272,15 +271,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((int[]) target)[targetIndex] = ((int[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - int[] sourceArray = (int[]) source; - int[] targetArray = (int[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((int[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 24870769a..6588528a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -55,7 +55,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -271,15 +270,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((long[]) target)[targetIndex] = ((long[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - long[] sourceArray = (long[]) source; - long[] targetArray = (long[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((long[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index ea3810b42..ec053f58c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -46,7 +46,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -214,15 +213,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - Binary[] sourceArray = (Binary[]) source; - Binary[] targetArray = (Binary[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((Binary[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 1d7adf81f..e72109f83 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -48,7 +48,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -242,15 +241,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - Binary[] sourceArray = (Binary[]) source; - Binary[] targetArray = (Binary[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((Binary[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index cdc768164..3eb798093 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -50,7 +50,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -240,15 +239,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((boolean[]) target)[targetIndex] = ((boolean[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - boolean[] sourceArray = (boolean[]) source; - boolean[] targetArray = (boolean[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((boolean[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index cc8922fa8..bd08351bd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -38,7 +38,6 @@ import java.nio.ByteBuffer; import java.time.LocalDate; import java.util.Arrays; -import java.util.Iterator; import java.util.Optional; public class DateType extends AbstractIntType { @@ -88,15 +87,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((LocalDate[]) target)[targetIndex] = ((LocalDate[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - LocalDate[] sourceArray = (LocalDate[]) source; - LocalDate[] targetArray = (LocalDate[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((LocalDate[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index e055e2f25..5180e0a99 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -55,7 +55,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -281,15 +280,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((double[]) target)[targetIndex] = ((double[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - double[] sourceArray = (double[]) source; - double[] targetArray = (double[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((double[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 6ca2046d0..97e49bb76 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -54,7 +54,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -279,15 +278,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((float[]) target)[targetIndex] = ((float[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - float[] sourceArray = (float[]) source; - float[] targetArray = (float[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((float[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 0ece4044c..4b888bf3c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -47,7 +47,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -237,15 +236,6 @@ public void copyArrayElement(Object source, int sourceIndex, Object target, int ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - Binary[] sourceArray = (Binary[]) source; - Binary[] targetArray = (Binary[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((Binary[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index e43253d2c..885a79455 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -43,7 +43,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Iterator; import java.util.List; public interface Type { @@ -180,11 +179,6 @@ default void copyArrayElement(Object source, int sourceIndex, Object target, int throw new UnsupportedOperationException(getClass().getName()); } - /** Copies the selected elements from {@code source} to {@code target} in iterator order. */ - default void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - throw new UnsupportedOperationException(getClass().getName()); - } - /** Copies {@code array}, truncating or padding it to {@code newLength}. */ default Object arrayCopyOf(Object array, int newLength) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 76747ce7a..e57e3fcf2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -47,7 +47,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Iterator; public class VectorType extends AbstractLongType { @@ -79,15 +78,6 @@ public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } - @Override - public void copyArrayElement(Object source, Iterator sourceIndex, Object target) { - long[] sourceArray = (long[]) source; - long[] targetArray = (long[]) target; - for (int targetIndex = 0; sourceIndex.hasNext(); targetIndex++) { - targetArray[targetIndex] = sourceArray[sourceIndex.next()]; - } - } - @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((long[]) array, newLength); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 8688ab55d..d8212e4df 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -878,12 +878,6 @@ public void testCopyArrayElement() { Assert.assertEquals(Array.get(source, 1), Array.get(target, 0)); Assert.assertEquals(Array.get(source, 0), Array.get(target, 1)); - Object iteratorTarget = type.createArray(2); - type.copyArrayElement(source, Arrays.asList(1, 0).iterator(), iteratorTarget); - - Assert.assertEquals(Array.get(source, 1), Array.get(iteratorTarget, 0)); - Assert.assertEquals(Array.get(source, 0), Array.get(iteratorTarget, 1)); - Object expanded = type.arrayCopyOf(source, 3); Assert.assertEquals(3, Array.getLength(expanded)); Assert.assertEquals(Array.get(source, 0), Array.get(expanded, 0)); @@ -894,14 +888,6 @@ public void testCopyArrayElement() { Assert.assertEquals(Array.get(source, 0), Array.get(truncated, 0)); } - try { - Type.fromTsDataType(TSDataType.UNKNOWN) - .copyArrayElement(new Object[0], Arrays.asList().iterator(), new Object[0]); - Assert.fail("Expected UnsupportedOperationException"); - } catch (UnsupportedOperationException ignored) { - // Expected. - } - try { Type.fromTsDataType(TSDataType.UNKNOWN).arrayCopyOf(new Object[0], 1); Assert.fail("Expected UnsupportedOperationException"); From bb44fbcf066458f7f61263dabbef70fa28b8e565 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 18:19:02 +0800 Subject: [PATCH 065/131] refactor getEncoder --- .../encoding/encoder/TSEncodingBuilder.java | 10 +------ .../org/apache/tsfile/utils/TypeServices.java | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index 25fb657e5..56f8f7350 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -227,15 +227,7 @@ public static class GorillaV1 extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case FLOAT: - return new SinglePrecisionEncoderV1(); - case DOUBLE: - return new DoublePrecisionEncoderV1(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.GORILLA_V1, type)); - } + return TypeServices.GORILLA_V1_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 6a5b58617..ad9045277 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -22,10 +22,12 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.encoding.encoder.DeltaBinaryEncoder; +import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV1; import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.encoding.encoder.FloatEncoder; import org.apache.tsfile.encoding.encoder.IntRleEncoder; import org.apache.tsfile.encoding.encoder.LongRleEncoder; +import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV1; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; @@ -486,6 +488,30 @@ public final class TypeServices { }; }; + public static final TypeService GORILLA_V1_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionEncoderV1(); + case DOUBLE -> new DoublePrecisionEncoderV1(); + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + TEXT, + BLOB, + STRING, + OBJECT, + ROW, + UNKNOWN, + VECTOR -> + throw new UnSupportedDataTypeException( + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.GORILLA_V1, + type.getTypeEnum())); + }; + private TypeServices() {} @FunctionalInterface From f33b5050cb04b6e7498b56ccdc84c5fd9db6a5af Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 18:40:44 +0800 Subject: [PATCH 066/131] add Type.setTo --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DateType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 7 ++++ .../tsfile/read/common/type/VectorType.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 42 +++++++++++++++++++ 12 files changed, 99 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 3d2e3fdd6..bd5f8f95c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -162,6 +162,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setIntV(from.getInt()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((int[]) toArray)[toIndex] = from.getInt(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getInt(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 6588528a3..669ff050f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -161,6 +161,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setLongV(from.getLong()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((long[]) toArray)[toIndex] = from.getLong(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getLong(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index ec053f58c..a9f3d5399 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -140,6 +140,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getString(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index e72109f83..6bd09bde4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -146,6 +146,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBinary(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 3eb798093..54e3496c4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -153,6 +153,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBoolV(from.getBoolean()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((boolean[]) toArray)[toIndex] = from.getBoolean(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBoolean(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index bd08351bd..3a678b874 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -82,6 +82,11 @@ public void addValue(int rowIndex, Object value, Object column) { ((LocalDate[]) column)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((LocalDate[]) toArray)[toIndex] = DateUtils.parseIntToLocalDate(from.getInt(fromIndex)); + } + @Override public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { ((LocalDate[]) target)[targetIndex] = ((LocalDate[]) source)[sourceIndex]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 5180e0a99..915fe3e62 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -164,6 +164,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setDoubleV(from.getDouble()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((double[]) toArray)[toIndex] = from.getDouble(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getDouble(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 97e49bb76..a51cebf94 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -162,6 +162,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setFloatV(from.getFloat()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((float[]) toArray)[toIndex] = from.getFloat(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getFloat(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 4b888bf3c..6522d365f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -147,6 +147,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBinary(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 885a79455..a5e4548c9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -246,6 +246,13 @@ default void setTo(TsPrimitiveType from, Field to) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** + * Copies a value from {@code from} at {@code fromIndex} to {@code toArray} at {@code toIndex}. + */ + default void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Adds the value of {@code columnName} in {@code resultSet} to {@code record}. */ default void addPoint(TSRecord record, String columnName, ResultSet resultSet) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index e57e3fcf2..ec32da0cf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -83,6 +83,11 @@ public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((long[]) array, newLength); } + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((long[]) toArray)[toIndex] = from.getLong(fromIndex); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index d8212e4df..e5a927a90 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -681,6 +681,48 @@ public void testSetToField() { } } + @Test + public void testSetToArray() { + Column column = Mockito.mock(Column.class); + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Mockito.when(column.getBoolean(1)).thenReturn(true); + Mockito.when(column.getInt(1)).thenReturn(1); + Mockito.when(column.getInt(2)).thenReturn(20260715); + Mockito.when(column.getLong(1)).thenReturn(2L); + Mockito.when(column.getFloat(1)).thenReturn(1.25F); + Mockito.when(column.getDouble(1)).thenReturn(2.5D); + Mockito.when(column.getBinary(1)).thenReturn(binary); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1, true}, + {TSDataType.INT32, 1, 1}, + {TSDataType.DATE, 2, LocalDate.of(2026, 7, 15)}, + {TSDataType.INT64, 1, 2L}, + {TSDataType.TIMESTAMP, 1, 2L}, + {TSDataType.FLOAT, 1, 1.25F}, + {TSDataType.DOUBLE, 1, 2.5D}, + {TSDataType.TEXT, 1, binary}, + {TSDataType.STRING, 1, binary}, + {TSDataType.BLOB, 1, binary}, + {TSDataType.OBJECT, 1, binary}, + {TSDataType.VECTOR, 1, 2L} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object target = type.createArray(1); + type.setTo(column, (int) testCase[1], target, 0); + Assert.assertEquals(testCase[2], Array.get(target, 0)); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).setTo(column, 1, new Object[1], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + @Test public void testAddPoint() { ResultSet resultSet = Mockito.mock(ResultSet.class); From 8788ea9e2ffbaff43f7f568ea19769d5826b21a0 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 18:48:36 +0800 Subject: [PATCH 067/131] add REGULAR_ENCODER_SERVICE --- .../tsfile/encoding/encoder/TSEncodingBuilder.java | 12 +----------- .../java/org/apache/tsfile/utils/TypeServices.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index 56f8f7350..334f77643 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -259,17 +259,7 @@ public static class Regular extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new RegularDataEncoder.IntRegularEncoder(); - case INT64: - case TIMESTAMP: - return new RegularDataEncoder.LongRegularEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.REGULAR, type)); - } + return TypeServices.REGULAR_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index ad9045277..564ab54f5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -27,6 +27,7 @@ import org.apache.tsfile.encoding.encoder.FloatEncoder; import org.apache.tsfile.encoding.encoder.IntRleEncoder; import org.apache.tsfile.encoding.encoder.LongRleEncoder; +import org.apache.tsfile.encoding.encoder.RegularDataEncoder; import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV1; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -512,6 +513,19 @@ public final class TypeServices { type.getTypeEnum())); }; + public static final TypeService REGULAR_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new RegularDataEncoder.IntRegularEncoder(); + case INT64, TIMESTAMP -> new RegularDataEncoder.LongRegularEncoder(); + case BOOLEAN, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.REGULAR, + type.getTypeEnum())); + }; + private TypeServices() {} @FunctionalInterface From 79205ab95d879906b8a9d584cc1392f3a6ef15d5 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 18:58:06 +0800 Subject: [PATCH 068/131] add GORILLA_V2_ENCODER_SERVICE --- .../encoding/encoder/TSEncodingBuilder.java | 16 +--------------- .../org/apache/tsfile/utils/TypeServices.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index 334f77643..f15ceedf5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -273,21 +273,7 @@ public static class GorillaV2 extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case FLOAT: - return new SinglePrecisionEncoderV2(); - case DOUBLE: - return new DoublePrecisionEncoderV2(); - case INT32: - case DATE: - return new IntGorillaEncoder(); - case INT64: - case TIMESTAMP: - return new LongGorillaEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.GORILLA, type)); - } + return TypeServices.GORILLA_V2_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 564ab54f5..82e48dbe3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -23,12 +23,16 @@ import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.encoding.encoder.DeltaBinaryEncoder; import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV1; +import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV2; import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.encoding.encoder.FloatEncoder; +import org.apache.tsfile.encoding.encoder.IntGorillaEncoder; import org.apache.tsfile.encoding.encoder.IntRleEncoder; +import org.apache.tsfile.encoding.encoder.LongGorillaEncoder; import org.apache.tsfile.encoding.encoder.LongRleEncoder; import org.apache.tsfile.encoding.encoder.RegularDataEncoder; import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV1; +import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV2; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; @@ -526,6 +530,21 @@ public final class TypeServices { type.getTypeEnum())); }; + public static final TypeService GORILLA_V2_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionEncoderV2(); + case DOUBLE -> new DoublePrecisionEncoderV2(); + case INT32, DATE -> new IntGorillaEncoder(); + case INT64, TIMESTAMP -> new LongGorillaEncoder(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw new UnSupportedDataTypeException( + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.GORILLA, + type.getTypeEnum())); + }; + private TypeServices() {} @FunctionalInterface From 581ef77d7642399dbf56abd0af6d675aef7c2c15 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 15 Jul 2026 19:13:08 +0800 Subject: [PATCH 069/131] add estimateArraySize --- .../read/common/type/AbstractIntType.java | 2 +- .../read/common/type/AbstractLongType.java | 2 +- .../read/common/type/AbstractVarcharType.java | 2 +- .../tsfile/read/common/type/BlobType.java | 2 +- .../tsfile/read/common/type/BooleanType.java | 2 +- .../tsfile/read/common/type/DateType.java | 2 +- .../tsfile/read/common/type/DoubleType.java | 2 +- .../tsfile/read/common/type/FloatType.java | 2 +- .../tsfile/read/common/type/ObjectType.java | 2 +- .../apache/tsfile/read/common/type/Type.java | 10 +++--- .../apache/tsfile/write/record/Tablet.java | 2 +- .../tsfile/read/common/type/TypeTest.java | 35 +++++++++++++++++++ 12 files changed, 50 insertions(+), 15 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index bd5f8f95c..94ff50afe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -343,7 +343,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((int[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 669ff050f..13283f635 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -342,7 +342,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((long[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index a9f3d5399..73d1874c9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -283,7 +283,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((Binary[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 6bd09bde4..d9a50da00 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -294,7 +294,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((Binary[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 54e3496c4..0e76db44f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -319,7 +319,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((boolean[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 3a678b874..3a730f238 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -154,7 +154,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((LocalDate[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 915fe3e62..357147121 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -352,7 +352,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((double[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index a51cebf94..47767434b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -350,7 +350,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((float[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 6522d365f..7d25baf9d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -289,7 +289,7 @@ public boolean arrayEquals(Object left, Object right, int rowSize) { } @Override - public long arrayRamBytesUsed(Object array) { + public long estimateArraySize(Object array) { return RamUsageEstimator.sizeOf((Binary[]) array); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index a5e4548c9..03658af3a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -194,6 +194,11 @@ default long estimateArraySize(int size) { throw new UnsupportedOperationException(getClass().getName()); } + /** Estimates the retained memory size of {@code array}. */ + default long estimateArraySize(Object array) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Estimates the retained memory size of a value of this type. */ default long estimateValueSize() { throw new UnsupportedOperationException(getClass().getName()); @@ -310,11 +315,6 @@ default void write(ChunkWriterImpl writer, long time, Object column, int rowInde Messages.format("error.write.type_not_supported", getTypeEnum())); } - /** Returns the retained memory size of an array column in Tablet. */ - default long arrayRamBytesUsed(Object array) { - throw new UnsupportedOperationException(getClass().getName()); - } - /** Creates a column with {@code positionCount} capacity for this type. */ default Column createColumnWithMaxPosition(int positionCount) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index f30c298fd..843588c59 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -1287,7 +1287,7 @@ public long ramBytesUsed() { if (values == null || values.length <= column) { continue; } - totalSizeInBytes += Type.fromTsDataType(tsDataType).arrayRamBytesUsed(values[column]); + totalSizeInBytes += Type.fromTsDataType(tsDataType).estimateArraySize(values[column]); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index e5a927a90..c63c6095e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -890,6 +890,41 @@ public void testEstimateArraySize() { } } + @Test + public void testEstimateArraySizeWithArray() { + boolean[] booleans = {false, true}; + int[] ints = {0, 1}; + LocalDate[] dates = {null, LocalDate.of(2026, 7, 15)}; + long[] longs = {0L, 1L}; + float[] floats = {0.0F, 1.0F}; + double[] doubles = {0.0D, 1.0D}; + Binary[] binaries = {null, new Binary("test", StandardCharsets.UTF_8)}; + + assertEstimateArraySize(TSDataType.BOOLEAN, booleans, RamUsageEstimator.sizeOf(booleans)); + assertEstimateArraySize(TSDataType.INT32, ints, RamUsageEstimator.sizeOf(ints)); + assertEstimateArraySize(TSDataType.DATE, dates, RamUsageEstimator.sizeOf(dates)); + assertEstimateArraySize(TSDataType.INT64, longs, RamUsageEstimator.sizeOf(longs)); + assertEstimateArraySize(TSDataType.TIMESTAMP, longs, RamUsageEstimator.sizeOf(longs)); + assertEstimateArraySize(TSDataType.FLOAT, floats, RamUsageEstimator.sizeOf(floats)); + assertEstimateArraySize(TSDataType.DOUBLE, doubles, RamUsageEstimator.sizeOf(doubles)); + assertEstimateArraySize(TSDataType.TEXT, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.STRING, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.BLOB, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.OBJECT, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.VECTOR, longs, RamUsageEstimator.sizeOf(longs)); + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).estimateArraySize(new Object[0]); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + private void assertEstimateArraySize(TSDataType dataType, Object array, long expected) { + Assert.assertEquals(expected, Type.fromTsDataType(dataType).estimateArraySize(array)); + } + @Test public void testCopyArrayElement() { LocalDate date = LocalDate.of(2026, 7, 15); From 718d3e81350b2c4dbc25f82e1248163013b0ff4f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 09:35:35 +0800 Subject: [PATCH 070/131] refactor addValue --- .../org/apache/tsfile/read/common/type/AbstractIntType.java | 4 ++-- .../apache/tsfile/read/common/type/AbstractLongType.java | 4 ++-- .../apache/tsfile/read/common/type/AbstractVarcharType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/BlobType.java | 6 +++--- .../org/apache/tsfile/read/common/type/BooleanType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/DateType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/DoubleType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/FloatType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/ObjectType.java | 6 +++--- .../main/java/org/apache/tsfile/read/common/type/Type.java | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 94ff50afe..74ea234b9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -266,9 +266,9 @@ public void writeDouble(ColumnBuilder builder, double value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { checkValueType(value, Integer.class, "Integer"); - ((int[]) column)[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; + ((int[]) array)[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 13283f635..86f5a619f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -265,9 +265,9 @@ public void writeDouble(ColumnBuilder builder, double value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { checkValueType(value, Long.class, "Long"); - ((long[]) column)[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; + ((long[]) array)[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 73d1874c9..b75a406b6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -209,8 +209,8 @@ public void writeBinary(ColumnBuilder builder, Binary value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { - ((Binary[]) column)[rowIndex] = toBinaryValue(value); + public void addValue(int rowIndex, Object value, Object array) { + ((Binary[]) array)[rowIndex] = toBinaryValue(value); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index d9a50da00..4b96d27f1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -222,7 +222,7 @@ public void writeBinary(ColumnBuilder builder, Binary value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { if (value != null && !(value instanceof Binary) && !(value instanceof String)) { throw new IllegalArgumentException( Messages.format( @@ -232,9 +232,9 @@ public void addValue(int rowIndex, Object value, Object column) { value.getClass().getName())); } if (value instanceof Binary) { - ((Binary[]) column)[rowIndex] = (Binary) value; + ((Binary[]) array)[rowIndex] = (Binary) value; } else { - ((Binary[]) column)[rowIndex] = + ((Binary[]) array)[rowIndex] = value != null ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) : Binary.EMPTY_VALUE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 0e76db44f..176cd89d2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -234,9 +234,9 @@ public void writeBoolean(ColumnBuilder builder, boolean value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { checkValueType(value, Boolean.class, "Boolean"); - ((boolean[]) column)[rowIndex] = value != null && (boolean) value; + ((boolean[]) array)[rowIndex] = value != null && (boolean) value; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 3a730f238..67e9f5413 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -77,9 +77,9 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { checkValueType(value, LocalDate.class, "LocalDate"); - ((LocalDate[]) column)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; + ((LocalDate[]) array)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 357147121..e9f241385 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -275,9 +275,9 @@ public void writeDouble(ColumnBuilder builder, double value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { checkValueType(value, Double.class, "Double"); - ((double[]) column)[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; + ((double[]) array)[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 47767434b..cb15d620c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -273,9 +273,9 @@ public void writeDouble(ColumnBuilder builder, double value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { checkValueType(value, Float.class, "Float"); - ((float[]) column)[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; + ((float[]) array)[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 7d25baf9d..6aab40d4f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -217,7 +217,7 @@ public void writeBinary(ColumnBuilder builder, Binary value) { } @Override - public void addValue(int rowIndex, Object value, Object column) { + public void addValue(int rowIndex, Object value, Object array) { if (value != null && !(value instanceof Binary) && !(value instanceof String)) { throw new IllegalArgumentException( Messages.format( @@ -227,9 +227,9 @@ public void addValue(int rowIndex, Object value, Object column) { value.getClass().getName())); } if (value instanceof Binary) { - ((Binary[]) column)[rowIndex] = (Binary) value; + ((Binary[]) array)[rowIndex] = (Binary) value; } else { - ((Binary[]) column)[rowIndex] = + ((Binary[]) array)[rowIndex] = value != null ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) : Binary.EMPTY_VALUE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 03658af3a..ce93ed3cf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -169,8 +169,8 @@ default void writeObject(ColumnBuilder builder, Object value) { builder.writeObject(value); } - /** Adds a value to the array column at {@code rowIndex}. */ - default void addValue(int rowIndex, Object value, Object column) { + /** Adds a value to the array at {@code rowIndex}. */ + default void addValue(int rowIndex, Object value, Object array) { throw new UnsupportedOperationException(getClass().getName()); } From 469647fb4a341b8820c8add579ac672517f4b015 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 10:17:51 +0800 Subject: [PATCH 071/131] add Date compatibility --- .../tsfile/read/common/type/DateType.java | 72 +++++++++++++++++-- .../tsfile/read/common/type/TypeTest.java | 43 +++++++++++ 2 files changed, 110 insertions(+), 5 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 67e9f5413..308f2b845 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -65,6 +65,10 @@ public DataPoint getDataPoint(String measurementId, String value) { @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + if (array instanceof int[]) { + super.write(writer, time, array, rowIndex, isNull); + return; + } writer.write( time, isNull ? 0 : DateUtils.parseDateExpressionToInt(((LocalDate[]) array)[rowIndex]), @@ -73,28 +77,57 @@ public void write( @Override public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + if (column instanceof int[]) { + super.write(writer, time, column, rowIndex); + return; + } writer.write(time, DateUtils.parseDateExpressionToInt(((LocalDate[]) column)[rowIndex])); } @Override public void addValue(int rowIndex, Object value, Object array) { + if (array instanceof int[]) { + if (value instanceof Integer) { + ((int[]) array)[rowIndex] = (int) value; + } else { + checkValueType(value, LocalDate.class, "LocalDate"); + ((int[]) array)[rowIndex] = + value != null + ? DateUtils.parseDateExpressionToInt((LocalDate) value) + : DateUtils.EMPTY_DATE_INT; + } + return; + } checkValueType(value, LocalDate.class, "LocalDate"); ((LocalDate[]) array)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; } @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + if (toArray instanceof int[]) { + super.setTo(from, fromIndex, toArray, toIndex); + return; + } ((LocalDate[]) toArray)[toIndex] = DateUtils.parseIntToLocalDate(from.getInt(fromIndex)); } @Override public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { - ((LocalDate[]) target)[targetIndex] = ((LocalDate[]) source)[sourceIndex]; + if (target instanceof int[]) { + ((int[]) target)[targetIndex] = getDateInt(source, sourceIndex); + return; + } + ((LocalDate[]) target)[targetIndex] = + source instanceof int[] + ? DateUtils.parseIntToLocalDate(((int[]) source)[sourceIndex]) + : ((LocalDate[]) source)[sourceIndex]; } @Override public Object arrayCopyOf(Object array, int newLength) { - return Arrays.copyOf((LocalDate[]) array, newLength); + return array instanceof int[] + ? Arrays.copyOf((int[]) array, newLength) + : Arrays.copyOf((LocalDate[]) array, newLength); } @Override @@ -122,6 +155,12 @@ public Field getField(Object value) { @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { + if (array instanceof int[]) { + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(((int[]) array)[i], stream); + } + return; + } LocalDate[] values = (LocalDate[]) array; for (int i = 0; i < rowSize; i++) { ReadWriteIOUtils.write( @@ -149,13 +188,28 @@ public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIn @Override public boolean arrayEquals(Object left, Object right, int rowSize) { - return hasEnoughLength(left, right, rowSize) - && Arrays.equals((LocalDate[]) left, 0, rowSize, (LocalDate[]) right, 0, rowSize); + if (!hasEnoughLength(left, right, rowSize)) { + return false; + } + if (left instanceof int[] && right instanceof int[]) { + return Arrays.equals((int[]) left, 0, rowSize, (int[]) right, 0, rowSize); + } + if (left instanceof LocalDate[] && right instanceof LocalDate[]) { + return Arrays.equals((LocalDate[]) left, 0, rowSize, (LocalDate[]) right, 0, rowSize); + } + for (int i = 0; i < rowSize; i++) { + if (getDateInt(left, i) != getDateInt(right, i)) { + return false; + } + } + return true; } @Override public long estimateArraySize(Object array) { - return RamUsageEstimator.sizeOf((LocalDate[]) array); + return array instanceof int[] + ? RamUsageEstimator.sizeOf((int[]) array) + : RamUsageEstimator.sizeOf((LocalDate[]) array); } @Override @@ -182,6 +236,14 @@ public String getDisplayName() { return "DATE"; } + private int getDateInt(Object array, int index) { + if (array instanceof int[]) { + return ((int[]) array)[index]; + } + LocalDate value = ((LocalDate[]) array)[index]; + return value != null ? DateUtils.parseDateExpressionToInt(value) : DateUtils.EMPTY_DATE_INT; + } + public static DateType getInstance() { return DATE; } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index c63c6095e..08534e721 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -30,6 +30,7 @@ import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -42,6 +43,9 @@ import org.junit.Test; import org.mockito.Mockito; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; import java.lang.reflect.Array; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -723,6 +727,45 @@ public void testSetToArray() { } } + @Test + public void testDateAddValueWithIntArray() throws IOException { + Type dateType = Type.fromTsDataType(TSDataType.DATE); + int[] values = new int[2]; + + dateType.addValue(0, LocalDate.of(2026, 7, 14), values); + dateType.addValue(1, null, values); + + Assert.assertArrayEquals(new int[] {20260714, DateUtils.EMPTY_DATE_INT}, values); + + dateType.addValue(0, 20260715, values); + Assert.assertEquals(20260715, values[0]); + + Column column = Mockito.mock(Column.class); + Mockito.when(column.getInt(0)).thenReturn(20260715); + dateType.setTo(column, 0, values, 0); + Assert.assertEquals(20260715, values[0]); + + LocalDate[] localDates = new LocalDate[2]; + dateType.copyArrayElement(values, 0, localDates, 0); + dateType.copyArrayElement(localDates, 0, values, 0); + Assert.assertEquals(LocalDate.of(2026, 7, 15), localDates[0]); + Assert.assertEquals(20260715, values[0]); + Assert.assertTrue( + dateType.arrayEquals( + values, new LocalDate[] {LocalDate.of(2026, 7, 15), LocalDate.of(1000, 1, 1)}, 2)); + + Object copied = dateType.arrayCopyOf(values, 3); + Assert.assertTrue(copied instanceof int[]); + Assert.assertArrayEquals(new int[] {20260715, DateUtils.EMPTY_DATE_INT, 0}, (int[]) copied); + Assert.assertEquals(RamUsageEstimator.sizeOf(values), dateType.estimateArraySize(values)); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + dateType.serializeArray(values, values.length, new DataOutputStream(output)); + ByteBuffer buffer = ByteBuffer.wrap(output.toByteArray()); + Assert.assertEquals(20260715, buffer.getInt()); + Assert.assertEquals(DateUtils.EMPTY_DATE_INT, buffer.getInt()); + } + @Test public void testAddPoint() { ResultSet resultSet = Mockito.mock(ResultSet.class); From c648de0956600ab54da6ac92ec7ff072ae62928e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 12:09:50 +0800 Subject: [PATCH 072/131] add Date compatibility --- .../main/java/org/apache/tsfile/utils/TsPrimitiveType.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java index 75d1b2c77..0bd41191e 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java @@ -237,12 +237,12 @@ public int getInt() { @Override public double getDouble() { - return (double) value; + return value; } @Override public long getLong() { - return (long) value; + return value; } @Override @@ -252,6 +252,9 @@ public float getFloat() { @Override public Binary getBinary() { + if (dataType == TSDataType.DATE) { + return new Binary(TSDataType.getDateStringValue(value), StandardCharsets.UTF_8); + } return new Binary(String.valueOf(this.value), StandardCharsets.UTF_8); } From e7959bcd205325f2894885a14c383d42fe03817c Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 15:12:42 +0800 Subject: [PATCH 073/131] refactor putValueToField --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 6 +++ .../tsfile/read/common/type/VectorType.java | 6 +++ .../dataset/DataSetWithoutTimeGenerator.java | 45 +++---------------- .../tsfile/read/common/type/TypeTest.java | 12 +++++ 12 files changed, 69 insertions(+), 40 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 74ea234b9..23105d1c1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -162,6 +162,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setIntV(from.getInt()); } + @Override + public void setTo(BatchData from, Field to) { + to.setIntV(from.getInt()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((int[]) toArray)[toIndex] = from.getInt(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 86f5a619f..88b647d07 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -161,6 +161,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setLongV(from.getLong()); } + @Override + public void setTo(BatchData from, Field to) { + to.setLongV(from.getLong()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((long[]) toArray)[toIndex] = from.getLong(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index b75a406b6..b2050e44f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -140,6 +140,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(BatchData from, Field to) { + to.setBinaryV(from.getBinary()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 4b96d27f1..4a89a9318 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -146,6 +146,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(BatchData from, Field to) { + to.setBinaryV(from.getBinary()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 176cd89d2..485a1c2df 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -153,6 +153,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBoolV(from.getBoolean()); } + @Override + public void setTo(BatchData from, Field to) { + to.setBoolV(from.getBoolean()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((boolean[]) toArray)[toIndex] = from.getBoolean(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index e9f241385..cdccf388c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -164,6 +164,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setDoubleV(from.getDouble()); } + @Override + public void setTo(BatchData from, Field to) { + to.setDoubleV(from.getDouble()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((double[]) toArray)[toIndex] = from.getDouble(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index cb15d620c..8bfd80ddd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -162,6 +162,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setFloatV(from.getFloat()); } + @Override + public void setTo(BatchData from, Field to) { + to.setFloatV(from.getFloat()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((float[]) toArray)[toIndex] = from.getFloat(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 6aab40d4f..e4d15e056 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -147,6 +147,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(BatchData from, Field to) { + to.setBinaryV(from.getBinary()); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index ce93ed3cf..3de8249cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -251,6 +251,12 @@ default void setTo(TsPrimitiveType from, Field to) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** Copies the current value from {@code from} to {@code to}. */ + default void setTo(BatchData from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + /** * Copies a value from {@code from} at {@code fromIndex} to {@code toArray} at {@code toIndex}. */ diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index ec32da0cf..c0ab9cf1d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -222,6 +222,12 @@ public void setTo(TsPrimitiveType from, Field to) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + @Override + public void setTo(BatchData from, Field to) { + TsPrimitiveType value = from.getVector()[0]; + Type.fromTsDataType(value.getDataType()).setTo(value, to); + } + @Override public long estimateValueSize() { return RamUsageEstimator.NUM_BYTES_OBJECT_REF; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java index 4e3b870ca..966131732 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java @@ -20,13 +20,12 @@ package org.apache.tsfile.read.query.dataset; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.series.AbstractFileSeriesReader; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; import java.util.ArrayList; @@ -152,44 +151,10 @@ private Long timeHeapGet() { } private Field putValueToField(BatchData col) { - TSDataType type = col.getDataType(); - Field field; - if (type == TSDataType.VECTOR) { - field = new Field((col.getVector())[0].getDataType()); - } else { - field = new Field(col.getDataType()); - } - switch (col.getDataType()) { - case BOOLEAN: - field.setBoolV(col.getBoolean()); - break; - case INT32: - case DATE: - field.setIntV(col.getInt()); - break; - case INT64: - case TIMESTAMP: - field.setLongV(col.getLong()); - break; - case FLOAT: - field.setFloatV(col.getFloat()); - break; - case DOUBLE: - field.setDoubleV(col.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - field.setBinaryV(col.getBinary()); - break; - case VECTOR: - Field.setTsPrimitiveValue((col.getVector())[0], field); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.read.dataset_unsupported_type", col.getDataType())); - } + TSDataType dataType = col.getDataType(); + Field field = + new Field(dataType == TSDataType.VECTOR ? col.getVector()[0].getDataType() : dataType); + Type.fromTsDataType(dataType).setTo(col, field); return field; } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 08534e721..f250f53c5 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -660,6 +660,12 @@ public void testSetToField() { type.setTo(from, directTarget); Assert.assertEquals(testCase[2], type.getValue(directTarget)); + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(1L, from.getValue()); + Field batchTarget = new Field(dataType); + type.setTo(batchData, batchTarget); + Assert.assertEquals(testCase[2], type.getValue(batchTarget)); + if (from.getDataType() == dataType) { Field factoryTarget = new Field(dataType); Field.setTsPrimitiveValue(from, factoryTarget); @@ -683,6 +689,12 @@ public void testSetToField() { } catch (UnSupportedDataTypeException ignored) { // Expected. } + + BatchData vectorBatchData = new BatchData(TSDataType.VECTOR); + vectorBatchData.putVector(1L, new TsPrimitiveType[] {new TsPrimitiveType.TsInt(1)}); + Field vectorTarget = new Field(TSDataType.INT32); + Type.fromTsDataType(TSDataType.VECTOR).setTo(vectorBatchData, vectorTarget); + Assert.assertEquals(1, vectorTarget.getIntV()); } @Test From 0ec71f5ecbc815451dd4bd3ce51a16145958f3f8 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 15:27:09 +0800 Subject: [PATCH 074/131] add Type.serializeValue --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 6 +++ .../tsfile/read/common/type/BlobType.java | 6 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 6 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/VectorType.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 39 +++++++++++++++++++ 11 files changed, 92 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 23105d1c1..917c6018b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -101,6 +101,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.writeInt(value.getInt()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + buffer.putInt((int) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 88b647d07..6c413eea7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -100,6 +100,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.writeLong(value.getLong()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + buffer.putLong((long) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index b2050e44f..d6c2b2829 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -34,6 +34,7 @@ import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; @@ -95,6 +96,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.write(binary.getValues()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + ReadWriteIOUtils.write((Binary) value, buffer); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 4a89a9318..7129aeb7c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -36,6 +36,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; @@ -101,6 +102,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.write(binary.getValues()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + ReadWriteIOUtils.write((Binary) value, buffer); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 485a1c2df..64a3039c8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -99,6 +99,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.writeBoolean(value.getBoolean()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + buffer.put((boolean) value ? (byte) 1 : (byte) 0); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index cdccf388c..5435ed310 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -104,6 +104,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.writeDouble(value.getDouble()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + buffer.putDouble((double) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 8bfd80ddd..29aa31527 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -103,6 +103,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.writeFloat(value.getFloat()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + buffer.putFloat((float) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index e4d15e056..904928643 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; @@ -102,6 +103,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOE stream.write(binary.getValues()); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + ReadWriteIOUtils.write((Binary) value, buffer); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 3de8249cd..7523dd46a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -134,6 +134,11 @@ default void serialize(TsPrimitiveType value, DataOutputStream stream) throws IO Messages.format("error.read.batch_data_unknown_type", value.getDataType())); } + /** Serializes {@code value} to {@code buffer}. */ + default void serializeValue(Object value, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index c0ab9cf1d..e96c87c85 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -151,6 +151,11 @@ public void serialize(TsPrimitiveType value, DataOutputStream stream) { Messages.format("error.read.batch_data_unknown_type", value.getDataType())); } + @Override + public void serializeValue(Object value, ByteBuffer buffer) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public Decoder getDecoder(TSEncoding encoding) { return switch (encoding) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index f250f53c5..2c1903421 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -129,6 +129,45 @@ public void testCalcTypeSize() { } } + @Test + public void testSerializeValue() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260716}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object value = testCase[1]; + ByteBuffer buffer = ByteBuffer.allocate(64); + type.serializeValue(value, buffer); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + type.serialize(type.getTsPrimitiveType(value), new DataOutputStream(output)); + Assert.assertArrayEquals( + output.toByteArray(), Arrays.copyOf(buffer.array(), buffer.position())); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).serializeValue(null, ByteBuffer.allocate(0)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + @Test public void testGetDataPoint() { Object[][] testCases = { From 23ca7f52148a9efa27f867711c441d188b5d3968 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 15:51:48 +0800 Subject: [PATCH 075/131] refactor fetchRecords --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 6 ++++ .../tsfile/read/common/type/VectorType.java | 6 ++++ .../read/query/dataset/TreeResultSet.java | 32 ++----------------- .../tsfile/read/common/type/TypeTest.java | 15 ++++++++- 12 files changed, 68 insertions(+), 31 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 917c6018b..e34fe0ab6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -177,6 +177,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((int[]) toArray)[toIndex] = from.getInt(fromIndex); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getIntV()); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getInt(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 6c413eea7..b10f9e481 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -176,6 +176,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((long[]) toArray)[toIndex] = from.getLong(fromIndex); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getLongV()); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getLong(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index d6c2b2829..aa4f7adef 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -156,6 +156,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getStringValue()); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getString(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 7129aeb7c..90f9c198a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -162,6 +162,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getBinaryV().getValues()); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBinary(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 64a3039c8..596609e61 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -168,6 +168,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((boolean[]) toArray)[toIndex] = from.getBoolean(fromIndex); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getBoolV()); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBoolean(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 5435ed310..f42bae9d6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -179,6 +179,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((double[]) toArray)[toIndex] = from.getDouble(fromIndex); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getDoubleV()); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getDouble(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 29aa31527..7827ceee7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -177,6 +177,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((float[]) toArray)[toIndex] = from.getFloat(fromIndex); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getFloatV()); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getFloat(columnName)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 904928643..e2f1454e1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -168,6 +168,11 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { record.addPoint(columnName, resultSet.getBinary(columnName)); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getBinaryV().getValues()); + } + @Override public DataPoint getDataPoint(String measurementId, String value) { return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 7523dd46a..68694f943 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -275,6 +275,12 @@ default void addPoint(TSRecord record, String columnName, ResultSet resultSet) { Messages.format("error.common.unsupported_data_type", this.getTypeEnum())); } + /** Adds the value in {@code field} to {@code record}. */ + default void addPoint(TSRecord record, String columnName, Field field) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", this.getTypeEnum())); + } + /** Creates a data point by parsing a string value. */ default DataPoint getDataPoint(String measurementId, String value) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index e96c87c85..7e13982a1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -195,6 +195,12 @@ public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { Messages.format("error.common.unsupported_data_type", getTypeEnum())); } + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", getTypeEnum())); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { throw new UnsupportedOperationException(getDisplayName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java index 9483e1e7e..6321e0622 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java @@ -23,6 +23,7 @@ import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.write.record.TSRecord; import java.io.IOException; @@ -119,36 +120,7 @@ private boolean fetchRecords() throws IOException { Integer pathIdx = pathIndexMap.get(path); if (pathIdx != null) { Field field = currentRow.getField(pathIdx); - switch (field.getDataType()) { - case INT32: - case DATE: - record.addPoint(measurement, field.getIntV()); - break; - case INT64: - case TIMESTAMP: - record.addPoint(measurement, field.getLongV()); - break; - case FLOAT: - record.addPoint(measurement, field.getFloatV()); - break; - case DOUBLE: - record.addPoint(measurement, field.getDoubleV()); - break; - case STRING: - case TEXT: - record.addPoint(measurement, field.getStringValue()); - break; - case BLOB: - record.addPoint(measurement, field.getBinaryV().getValues()); - break; - case BOOLEAN: - record.addPoint(measurement, field.getBoolV()); - break; - case VECTOR: - case UNKNOWN: - default: - break; - } + Type.fromTsDataType(field.getDataType()).addPoint(record, measurement, field); } else { record.dataPointList.add(null); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 2c1903421..d49e4ee73 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -845,9 +845,15 @@ public void testAddPoint() { for (Object[] testCase : testCases) { TSRecord record = new TSRecord(deviceID, 1L); - Type.fromTsDataType((TSDataType) testCase[0]).addPoint(record, "value", resultSet); + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + type.addPoint(record, "value", resultSet); Assert.assertEquals(1, record.dataPointList.size()); Assert.assertEquals(testCase[1], record.dataPointList.get(0).getValue()); + + TSRecord fieldRecord = new TSRecord(deviceID, 1L); + type.addPoint(fieldRecord, "value", type.getField(testCase[1])); + Assert.assertEquals(1, fieldRecord.dataPointList.size()); + Assert.assertEquals(testCase[1], fieldRecord.dataPointList.get(0).getValue()); } for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { @@ -858,6 +864,13 @@ public void testAddPoint() { } catch (UnSupportedDataTypeException ignored) { // Expected. } + + try { + Type.fromTsDataType(dataType).addPoint(record, "value", new Field(dataType)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } } } From a1b78eb20dbb82827f4b0b01ad013f49d027858f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 17:12:05 +0800 Subject: [PATCH 076/131] add Type.serializeValue --- .../tsfile/read/common/type/AbstractIntType.java | 5 +++++ .../read/common/type/AbstractLongType.java | 5 +++++ .../read/common/type/AbstractVarcharType.java | 5 +++++ .../apache/tsfile/read/common/type/BlobType.java | 5 +++++ .../tsfile/read/common/type/BooleanType.java | 5 +++++ .../tsfile/read/common/type/DoubleType.java | 5 +++++ .../tsfile/read/common/type/FloatType.java | 5 +++++ .../tsfile/read/common/type/ObjectType.java | 5 +++++ .../org/apache/tsfile/read/common/type/Type.java | 5 +++++ .../tsfile/read/common/type/VectorType.java | 5 +++++ .../apache/tsfile/read/common/type/TypeTest.java | 16 ++++++++++++++-- 11 files changed, 64 insertions(+), 2 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index e34fe0ab6..7f2898643 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -106,6 +106,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { buffer.putInt((int) value); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeInt((int) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt32Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index b10f9e481..b62bbcffa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -105,6 +105,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { buffer.putLong((long) value); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeLong((long) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getInt64Encoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index aa4f7adef..32f52cb82 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -101,6 +101,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { ReadWriteIOUtils.write((Binary) value, buffer); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write((Binary) value, stream); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 90f9c198a..698063c1b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -107,6 +107,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { ReadWriteIOUtils.write((Binary) value, buffer); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write((Binary) value, stream); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 596609e61..b7c0793c3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -104,6 +104,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { buffer.put((boolean) value ? (byte) 1 : (byte) 0); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeBoolean((boolean) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getBooleanEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index f42bae9d6..6a2df4dae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -109,6 +109,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { buffer.putDouble((double) value); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeDouble((double) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getDoubleEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 7827ceee7..86140eecb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -108,6 +108,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { buffer.putFloat((float) value); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeFloat((float) value); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getFloatEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index e2f1454e1..91b0412ac 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -108,6 +108,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { ReadWriteIOUtils.write((Binary) value, buffer); } + @Override + public void serializeValue(Object value, DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write((Binary) value, stream); + } + @Override public TSEncoding getDefaultEncoding(TSFileConfig config) { return TSEncoding.valueOf(config.getTextEncoding()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 68694f943..6c35e423f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -139,6 +139,11 @@ default void serializeValue(Object value, ByteBuffer buffer) { throw new UnsupportedOperationException(getClass().getName()); } + /** Serializes {@code value} to {@code stream}. */ + default void serializeValue(Object value, DataOutputStream stream) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 7e13982a1..df3f6a69a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -156,6 +156,11 @@ public void serializeValue(Object value, ByteBuffer buffer) { throw new UnsupportedOperationException(getDisplayName()); } + @Override + public void serializeValue(Object value, DataOutputStream stream) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public Decoder getDecoder(TSEncoding encoding) { return switch (encoding) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index d49e4ee73..abba7836e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -154,8 +154,12 @@ public void testSerializeValue() throws IOException { ByteArrayOutputStream output = new ByteArrayOutputStream(); type.serialize(type.getTsPrimitiveType(value), new DataOutputStream(output)); - Assert.assertArrayEquals( - output.toByteArray(), Arrays.copyOf(buffer.array(), buffer.position())); + byte[] expected = output.toByteArray(); + Assert.assertArrayEquals(expected, Arrays.copyOf(buffer.array(), buffer.position())); + + output.reset(); + type.serializeValue(value, new DataOutputStream(output)); + Assert.assertArrayEquals(expected, output.toByteArray()); } for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { @@ -165,6 +169,14 @@ public void testSerializeValue() throws IOException { } catch (UnsupportedOperationException ignored) { // Expected. } + + try { + Type.fromTsDataType(dataType) + .serializeValue(null, new DataOutputStream(new ByteArrayOutputStream())); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } } } From 1a9fc7ebe1fef7b491bd3404d67aa50e55ea7911 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 17:43:53 +0800 Subject: [PATCH 077/131] refactor copy --- .../org/apache/tsfile/read/common/Field.java | 29 +---------------- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 6 ++++ .../apache/tsfile/read/common/FieldTest.java | 32 +++++++++++++++++++ 11 files changed, 79 insertions(+), 28 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java index 33980c6f3..21e192a07 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java @@ -25,7 +25,6 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.time.LocalDate; @@ -50,33 +49,7 @@ public Field(TSDataType dataType) { public static Field copy(Field field) { Field out = new Field(field.dataType); if (out.dataType != null) { - switch (out.dataType) { - case DOUBLE: - out.setDoubleV(field.getDoubleV()); - break; - case FLOAT: - out.setFloatV(field.getFloatV()); - break; - case INT64: - case TIMESTAMP: - out.setLongV(field.getLongV()); - break; - case INT32: - case DATE: - out.setIntV(field.getIntV()); - break; - case BOOLEAN: - out.setBoolV(field.getBoolV()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - out.setBinaryV(field.getBinaryV()); - break; - default: - throw new UnSupportedDataTypeException(out.dataType.toString()); - } + Type.fromTsDataType(out.dataType).setTo(field, out); } return out; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 7f2898643..89a8c9c74 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -172,6 +172,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setIntV(from.getInt()); } + @Override + public void setTo(Field from, Field to) { + to.setIntV(from.getIntV()); + } + @Override public void setTo(BatchData from, Field to) { to.setIntV(from.getInt()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index b62bbcffa..87b21852c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -171,6 +171,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setLongV(from.getLong()); } + @Override + public void setTo(Field from, Field to) { + to.setLongV(from.getLongV()); + } + @Override public void setTo(BatchData from, Field to) { to.setLongV(from.getLong()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 32f52cb82..ba3637031 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -151,6 +151,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(Field from, Field to) { + to.setBinaryV(from.getBinaryV()); + } + @Override public void setTo(BatchData from, Field to) { to.setBinaryV(from.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 698063c1b..d82e7dae9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -157,6 +157,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(Field from, Field to) { + to.setBinaryV(from.getBinaryV()); + } + @Override public void setTo(BatchData from, Field to) { to.setBinaryV(from.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index b7c0793c3..d6cffb867 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -163,6 +163,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBoolV(from.getBoolean()); } + @Override + public void setTo(Field from, Field to) { + to.setBoolV(from.getBoolV()); + } + @Override public void setTo(BatchData from, Field to) { to.setBoolV(from.getBoolean()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 6a2df4dae..53b305a12 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -174,6 +174,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setDoubleV(from.getDouble()); } + @Override + public void setTo(Field from, Field to) { + to.setDoubleV(from.getDoubleV()); + } + @Override public void setTo(BatchData from, Field to) { to.setDoubleV(from.getDouble()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 86140eecb..83deccdc3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -172,6 +172,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setFloatV(from.getFloat()); } + @Override + public void setTo(Field from, Field to) { + to.setFloatV(from.getFloatV()); + } + @Override public void setTo(BatchData from, Field to) { to.setFloatV(from.getFloat()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 91b0412ac..544701b4c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -158,6 +158,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(Field from, Field to) { + to.setBinaryV(from.getBinaryV()); + } + @Override public void setTo(BatchData from, Field to) { to.setBinaryV(from.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 6c35e423f..2aa3b6773 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -261,6 +261,12 @@ default void setTo(TsPrimitiveType from, Field to) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** Copies the value from {@code from} to {@code to}. */ + default void setTo(Field from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + /** Copies the current value from {@code from} to {@code to}. */ default void setTo(BatchData from, Field to) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java index 1be38da0e..0ee21c4fd 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java @@ -18,10 +18,17 @@ */ package org.apache.tsfile.read.common; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NullFieldException; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.junit.Assert; import org.junit.Test; +import java.nio.charset.StandardCharsets; + public class FieldTest { @Test(expected = NullFieldException.class) @@ -29,4 +36,29 @@ public void construct() { Field field = new Field(null); field.getIntV(); } + + @Test + public void copy() { + Object[][] values = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260716}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, new Binary("text", StandardCharsets.UTF_8)}, + {TSDataType.STRING, new Binary("string", StandardCharsets.UTF_8)}, + {TSDataType.BLOB, new Binary(new byte[] {1, 2})}, + {TSDataType.OBJECT, new Binary(BytesUtils.longToBytes(2L))} + }; + + for (Object[] value : values) { + TSDataType dataType = (TSDataType) value[0]; + Field source = Type.fromTsDataType(dataType).getField(value[1]); + Field copy = Field.copy(source); + Assert.assertEquals(source.getDataType(), copy.getDataType()); + Assert.assertEquals(source.getObjectValue(dataType), copy.getObjectValue(dataType)); + } + } } From e8a97849ce3d58b7a4fc94287d623fec0e07a69e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 18:04:48 +0800 Subject: [PATCH 078/131] add Type.deserialize --- .../read/common/type/AbstractIntType.java | 11 +++++ .../read/common/type/AbstractLongType.java | 11 +++++ .../read/common/type/AbstractVarcharType.java | 11 +++++ .../tsfile/read/common/type/BlobType.java | 11 +++++ .../tsfile/read/common/type/BooleanType.java | 11 +++++ .../tsfile/read/common/type/DoubleType.java | 11 +++++ .../tsfile/read/common/type/FloatType.java | 11 +++++ .../tsfile/read/common/type/ObjectType.java | 11 +++++ .../apache/tsfile/read/common/type/Type.java | 11 +++++ .../tsfile/read/common/type/VectorType.java | 11 +++++ .../tsfile/read/common/type/TypeTest.java | 49 +++++++++++++++++++ 11 files changed, 159 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 89a8c9c74..aa7268986 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -53,6 +53,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -344,6 +345,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readInt(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readInt(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { int[] intValues = new int[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 87b21852c..fa344b5f1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -52,6 +52,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -343,6 +344,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readLong(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readLong(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { long[] intValues = new long[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index ba3637031..afa1a7ecc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -44,6 +44,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -297,6 +298,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readBinary(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBinary(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { Binary[] values = deserializeBinaryValues(buffer, rowSize, nullIndicators); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index d82e7dae9..48ca59b29 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -46,6 +46,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -308,6 +309,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readBinary(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBinary(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { Binary[] binaries = deserializeBinaryValues(buffer, rowSize, nullIndicators); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index d6cffb867..8fd2a0028 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -47,6 +47,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -312,6 +313,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBoolean(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { if (nullIndicators == null) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 53b305a12..53a3cbd9a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -52,6 +52,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -353,6 +354,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readDouble(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readDouble(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { double[] values = new double[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 83deccdc3..633654d39 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -51,6 +51,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -351,6 +352,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readFloat(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readFloat(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { float[] values = new float[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 544701b4c..6055c7a64 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -45,6 +45,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -303,6 +304,16 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readBinary(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBinary(stream); + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { Binary[] values = deserializeBinaryValues(buffer, rowSize, nullIndicators); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 2aa3b6773..6441d0204 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -42,6 +42,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; @@ -230,6 +231,16 @@ default Object deserializeArray(ByteBuffer buffer, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } + /** Deserializes one value from {@code buffer} into {@code array} at {@code index}. */ + default void deserialize(Object[] array, int index, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Deserializes one value from {@code stream} into {@code array} at {@code index}. */ + default void deserialize(Object[] array, int index, InputStream stream) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Deserializes an array with {@code rowSize} entries from {@code buffer}. */ default Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index df3f6a69a..d5a32887b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -45,6 +45,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -211,6 +212,16 @@ public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIn throw new UnsupportedOperationException(getDisplayName()); } + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public Column createNullColumn(int positionCount) { throw new IllegalArgumentException( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index abba7836e..86b4920fc 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -43,6 +43,7 @@ import org.junit.Test; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -180,6 +181,54 @@ public void testSerializeValue() throws IOException { } } + @Test + public void testDeserializeValue() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260716}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + type.serializeValue(testCase[1], new DataOutputStream(output)); + Object[] values = new Object[2]; + type.deserialize(values, 1, ByteBuffer.wrap(output.toByteArray())); + Assert.assertEquals(testCase[1], values[1]); + + values[1] = null; + type.deserialize(values, 1, new ByteArrayInputStream(output.toByteArray())); + Assert.assertEquals(testCase[1], values[1]); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).deserialize(new Object[1], 0, ByteBuffer.allocate(0)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + + try { + Type.fromTsDataType(dataType) + .deserialize(new Object[1], 0, new ByteArrayInputStream(new byte[0])); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + @Test public void testGetDataPoint() { Object[][] testCases = { From da79d14156f811eebdbee3df5060c0f4788b8460 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 18:20:03 +0800 Subject: [PATCH 079/131] refactor fillSingleMeasurementColumn --- .../read/common/type/AbstractIntType.java | 5 ++++ .../read/common/type/AbstractLongType.java | 5 ++++ .../read/common/type/AbstractVarcharType.java | 5 ++++ .../tsfile/read/common/type/BlobType.java | 5 ++++ .../tsfile/read/common/type/BooleanType.java | 5 ++++ .../tsfile/read/common/type/DoubleType.java | 5 ++++ .../tsfile/read/common/type/FloatType.java | 5 ++++ .../tsfile/read/common/type/ObjectType.java | 5 ++++ .../apache/tsfile/read/common/type/Type.java | 6 ++++ .../block/SingleDeviceTsBlockReader.java | 30 ++----------------- 10 files changed, 48 insertions(+), 28 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index aa7268986..a4bb11fa9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -183,6 +183,11 @@ public void setTo(BatchData from, Field to) { to.setIntV(from.getInt()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getInts()[index] = from.getInt(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((int[]) toArray)[toIndex] = from.getInt(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index fa344b5f1..0ea510425 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -182,6 +182,11 @@ public void setTo(BatchData from, Field to) { to.setLongV(from.getLong()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getLongs()[index] = from.getLong(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((long[]) toArray)[toIndex] = from.getLong(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index afa1a7ecc..b63dcb871 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -162,6 +162,11 @@ public void setTo(BatchData from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 48ca59b29..7dc017228 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -168,6 +168,11 @@ public void setTo(BatchData from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 8fd2a0028..779c487f4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -174,6 +174,11 @@ public void setTo(BatchData from, Field to) { to.setBoolV(from.getBoolean()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBooleans()[index] = from.getBoolean(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((boolean[]) toArray)[toIndex] = from.getBoolean(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 53a3cbd9a..fcdba9d08 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -185,6 +185,11 @@ public void setTo(BatchData from, Field to) { to.setDoubleV(from.getDouble()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getDoubles()[index] = from.getDouble(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((double[]) toArray)[toIndex] = from.getDouble(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 633654d39..c30222253 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -183,6 +183,11 @@ public void setTo(BatchData from, Field to) { to.setFloatV(from.getFloat()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getFloats()[index] = from.getFloat(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((float[]) toArray)[toIndex] = from.getFloat(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 6055c7a64..1aadc3048 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -169,6 +169,11 @@ public void setTo(BatchData from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 6441d0204..02d4297ea 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -284,6 +284,12 @@ default void setTo(BatchData from, Field to) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** Copies the current value from {@code from} to {@code to} at {@code index}. */ + default void setTo(BatchData from, Column to, int index) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + /** * Copies a value from {@code from} at {@code fromIndex} to {@code toArray} at {@code toIndex}. */ diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java index 8eaab414f..979ca55d8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java @@ -25,6 +25,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.controller.IMetadataQuerier; import org.apache.tsfile.read.expression.ExpressionTree; @@ -267,34 +268,7 @@ private void fillIdColumn(Column column, Object val, int startPos, int endPos) { } private static void fillSingleMeasurementColumn(Column column, BatchData batchData, int pos) { - switch (batchData.getDataType()) { - case BOOLEAN: - column.getBooleans()[pos] = batchData.getBoolean(); - break; - case DOUBLE: - column.getDoubles()[pos] = batchData.getDouble(); - break; - case FLOAT: - column.getFloats()[pos] = batchData.getFloat(); - break; - case INT32: - case DATE: - column.getInts()[pos] = batchData.getInt(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - column.getBinaries()[pos] = batchData.getBinary(); - break; - case INT64: - case TIMESTAMP: - column.getLongs()[pos] = batchData.getLong(); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.block_reader_unsupported_type", batchData.getDataType())); - } + Type.fromTsDataType(batchData.getDataType()).setTo(batchData, column, pos); column.setPositionCount(pos + 1); } From 16cf24469a6d794f28ef0015fc1674b3b4e1b016 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 18:29:21 +0800 Subject: [PATCH 080/131] refactor setTo --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 6 ++++ .../block/SingleDeviceTsBlockReader.java | 31 ++----------------- 10 files changed, 48 insertions(+), 29 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index a4bb11fa9..ebea121ef 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -173,6 +173,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setIntV(from.getInt()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getInts()[index] = from.getInt(); + } + @Override public void setTo(Field from, Field to) { to.setIntV(from.getIntV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 0ea510425..60c9e1fc5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -172,6 +172,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setLongV(from.getLong()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getLongs()[index] = from.getLong(); + } + @Override public void setTo(Field from, Field to) { to.setLongV(from.getLongV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index b63dcb871..cfb171f17 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -152,6 +152,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + @Override public void setTo(Field from, Field to) { to.setBinaryV(from.getBinaryV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 7dc017228..6eead44e6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -158,6 +158,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + @Override public void setTo(Field from, Field to) { to.setBinaryV(from.getBinaryV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 779c487f4..1cb4adb34 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -164,6 +164,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBoolV(from.getBoolean()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBooleans()[index] = from.getBoolean(); + } + @Override public void setTo(Field from, Field to) { to.setBoolV(from.getBoolV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index fcdba9d08..6fe7ce43b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -175,6 +175,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setDoubleV(from.getDouble()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getDoubles()[index] = from.getDouble(); + } + @Override public void setTo(Field from, Field to) { to.setDoubleV(from.getDoubleV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index c30222253..44cd984ea 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -173,6 +173,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setFloatV(from.getFloat()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getFloats()[index] = from.getFloat(); + } + @Override public void setTo(Field from, Field to) { to.setFloatV(from.getFloatV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 1aadc3048..03ae5b1e0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -159,6 +159,11 @@ public void setTo(TsPrimitiveType from, Field to) { to.setBinaryV(from.getBinary()); } + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + @Override public void setTo(Field from, Field to) { to.setBinaryV(from.getBinaryV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 02d4297ea..1be37f026 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -272,6 +272,12 @@ default void setTo(TsPrimitiveType from, Field to) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** Copies the value from {@code from} to {@code to} at {@code index}. */ + default void setTo(TsPrimitiveType from, Column to, int index) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + /** Copies the value from {@code from} to {@code to}. */ default void setTo(Field from, Field to) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java index 979ca55d8..e4b68298f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java @@ -357,35 +357,8 @@ void fillInto(TsBlock block, int blockRowNum) { final List columnPositions = posInResult.get(i); for (Integer pos : columnPositions) { if (value != null) { - switch (value.getDataType()) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - block.getColumn(pos).getBinaries()[blockRowNum] = value.getBinary(); - break; - case INT32: - case DATE: - block.getColumn(pos).getInts()[blockRowNum] = value.getInt(); - break; - case INT64: - case TIMESTAMP: - block.getColumn(pos).getLongs()[blockRowNum] = value.getLong(); - break; - case BOOLEAN: - block.getColumn(pos).getBooleans()[blockRowNum] = value.getBoolean(); - break; - case FLOAT: - block.getColumn(pos).getFloats()[blockRowNum] = value.getFloat(); - break; - case DOUBLE: - block.getColumn(pos).getDoubles()[blockRowNum] = value.getDouble(); - break; - default: - throw new IllegalArgumentException( - Messages.format( - "error.read.block_reader_unsupported_type", value.getDataType())); - } + Type.fromTsDataType(value.getDataType()) + .setTo(value, block.getColumn(pos), blockRowNum); } else { block.getColumn(pos).setNull(blockRowNum, blockRowNum + 1); } From da643ec163e85173d32a550dea8653cf3a0e1ab0 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 16 Jul 2026 19:04:12 +0800 Subject: [PATCH 081/131] refactor fillIdColumn --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 7 ++++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 10 ++++-- .../apache/tsfile/read/common/type/Type.java | 5 +++ .../block/SingleDeviceTsBlockReader.java | 35 +------------------ 10 files changed, 51 insertions(+), 36 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index ebea121ef..fbd087161 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -178,6 +178,11 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getInts()[index] = from.getInt(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getInts(), startIndex, endIndex, (int) value); + } + @Override public void setTo(Field from, Field to) { to.setIntV(from.getIntV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 60c9e1fc5..5e8c7ec55 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -177,6 +177,11 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getLongs()[index] = from.getLong(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getLongs(), startIndex, endIndex, (long) value); + } + @Override public void setTo(Field from, Field to) { to.setLongV(from.getLongV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index cfb171f17..bc16b3343 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -157,6 +157,11 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getBinaries()[index] = from.getBinary(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getBinaries(), startIndex, endIndex, toBinaryValue(value)); + } + @Override public void setTo(Field from, Field to) { to.setBinaryV(from.getBinaryV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 6eead44e6..5c36ef0fa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -163,6 +163,13 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getBinaries()[index] = from.getBinary(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Binary[] values = new Binary[1]; + addValue(0, value, values); + Arrays.fill(to.getBinaries(), startIndex, endIndex, values[0]); + } + @Override public void setTo(Field from, Field to) { to.setBinaryV(from.getBinaryV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 1cb4adb34..b5990e5f5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -169,6 +169,11 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getBooleans()[index] = from.getBoolean(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getBooleans(), startIndex, endIndex, (boolean) value); + } + @Override public void setTo(Field from, Field to) { to.setBoolV(from.getBoolV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 6fe7ce43b..ce8bb46d2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -180,6 +180,11 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getDoubles()[index] = from.getDouble(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getDoubles(), startIndex, endIndex, (double) value); + } + @Override public void setTo(Field from, Field to) { to.setDoubleV(from.getDoubleV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 44cd984ea..4e68efcfa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -178,6 +178,11 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getFloats()[index] = from.getFloat(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getFloats(), startIndex, endIndex, (float) value); + } + @Override public void setTo(Field from, Field to) { to.setFloatV(from.getFloatV()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 03ae5b1e0..fb112fe92 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -37,6 +37,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.utils.TsPrimitiveType.TsBinary; import org.apache.tsfile.write.chunk.ChunkWriterImpl; import org.apache.tsfile.write.chunk.ValueChunkWriter; import org.apache.tsfile.write.record.TSRecord; @@ -164,6 +165,11 @@ public void setTo(TsPrimitiveType from, Column to, int index) { to.getBinaries()[index] = from.getBinary(); } + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getBinaries(), startIndex, endIndex, value); + } + @Override public void setTo(Field from, Field to) { to.setBinaryV(from.getBinaryV()); @@ -235,12 +241,12 @@ public Field getField(Object value) { @Override public TsPrimitiveType getTsPrimitiveType() { - return new TsPrimitiveType.TsBinary(); + return new TsBinary(); } @Override public TsPrimitiveType getTsPrimitiveType(Object value) { - return new TsPrimitiveType.TsBinary((Binary) value); + return new TsBinary((Binary) value); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 1be37f026..d8548328c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -278,6 +278,11 @@ default void setTo(TsPrimitiveType from, Column to, int index) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** Copies {@code value} to {@code to} from {@code startIndex} to {@code endIndex}. */ + default void setTo(Object value, Column to, int startIndex, int endIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Copies the value from {@code from} to {@code to}. */ default void setTo(Field from, Field to) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java index e4b68298f..e149a648a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java @@ -33,16 +33,13 @@ import org.apache.tsfile.read.query.executor.task.DeviceQueryTask; import org.apache.tsfile.read.reader.series.AbstractFileSeriesReader; import org.apache.tsfile.read.reader.series.FileSeriesReader; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -233,37 +230,7 @@ private void advanceColumn(BatchData batchData, MeasurementColumnContext columnC } private void fillIdColumn(Column column, Object val, int startPos, int endPos) { - switch (column.getDataType()) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - if (val instanceof String) { - val = new Binary(((String) val), StandardCharsets.UTF_8); - } - Arrays.fill(column.getBinaries(), startPos, endPos, val); - break; - case BOOLEAN: - Arrays.fill(column.getBooleans(), startPos, endPos, ((boolean) val)); - break; - case INT32: - case DATE: - Arrays.fill(column.getInts(), startPos, endPos, ((int) val)); - break; - case INT64: - case TIMESTAMP: - Arrays.fill(column.getLongs(), startPos, endPos, ((long) val)); - break; - case FLOAT: - Arrays.fill(column.getFloats(), startPos, endPos, ((float) val)); - break; - case DOUBLE: - Arrays.fill(column.getDoubles(), startPos, endPos, ((double) val)); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.block_reader_unsupported_type", column.getDataType())); - } + Type.fromTsDataType(column.getDataType()).setTo(val, column, startPos, endPos); column.setPositionCount(endPos); } From bae9a848b9170527eb8b4496e047f85f0efab762 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 09:35:12 +0800 Subject: [PATCH 082/131] add Type.serializeArray --- .../read/common/type/AbstractIntType.java | 8 ++++ .../read/common/type/AbstractLongType.java | 8 ++++ .../tsfile/read/common/type/AbstractType.java | 10 ++++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 8 ++++ .../tsfile/read/common/type/DateType.java | 15 ++++++ .../tsfile/read/common/type/DoubleType.java | 8 ++++ .../tsfile/read/common/type/FloatType.java | 8 ++++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 5 ++ .../tsfile/read/common/type/VectorType.java | 5 ++ .../tsfile/read/common/type/TypeTest.java | 47 +++++++++++++++++++ 13 files changed, 137 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index fbd087161..6f435784b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -351,6 +351,14 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + int[] values = (int[]) array; + for (int i = 0; i < length; i++) { + buffer.putInt(values[i]); + } + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { int[] values = new int[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 5e8c7ec55..6d040894a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -350,6 +350,14 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + long[] values = (long[]) array; + for (int i = 0; i < length; i++) { + buffer.putLong(values[i]); + } + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { long[] values = new long[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 27f0b0573..f13e16d3c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -102,6 +102,16 @@ protected void serializeBinaryValues(Object array, int rowSize, DataOutputStream } } + protected void serializeBinaryValues(Object array, int length, ByteBuffer buffer) { + Binary[] binaryValues = (Binary[]) array; + for (int i = 0; i < length; i++) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(binaryValues[i] != null), buffer); + if (binaryValues[i] != null) { + ReadWriteIOUtils.write(binaryValues[i], buffer); + } + } + } + protected Binary[] deserializeBinaryValues(ByteBuffer buffer, int rowSize) { Binary[] values = new Binary[rowSize]; for (int i = 0; i < rowSize; i++) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index bc16b3343..f550a36d7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -308,6 +308,11 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) serializeBinaryValues(array, rowSize, stream); } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + serializeBinaryValues(array, length, buffer); + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 5c36ef0fa..a5aef652c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -321,6 +321,11 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) serializeBinaryValues(array, rowSize, stream); } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + serializeBinaryValues(array, length, buffer); + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index b5990e5f5..cbe0a6cf1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -319,6 +319,14 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + boolean[] values = (boolean[]) array; + for (int i = 0; i < length; i++) { + buffer.put(BytesUtils.boolToByte(values[i])); + } + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { boolean[] values = new boolean[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 308f2b845..8a429966c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -171,6 +171,21 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + if (array instanceof int[]) { + super.serializeArray(array, length, buffer); + return; + } + LocalDate[] values = (LocalDate[]) array; + for (int i = 0; i < length; i++) { + buffer.putInt( + values[i] == null + ? DateUtils.EMPTY_DATE_INT + : DateUtils.parseDateExpressionToInt(values[i])); + } + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { LocalDate[] values = new LocalDate[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index ce8bb46d2..f70c12228 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -360,6 +360,14 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + double[] values = (double[]) array; + for (int i = 0; i < length; i++) { + buffer.putDouble(values[i]); + } + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { double[] values = new double[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 4e68efcfa..d689e01fd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -358,6 +358,14 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) } } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + float[] values = (float[]) array; + for (int i = 0; i < length; i++) { + buffer.putFloat(values[i]); + } + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { float[] values = new float[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index fb112fe92..a21eeeaa2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -315,6 +315,11 @@ public void serializeArray(Object array, int rowSize, DataOutputStream stream) serializeBinaryValues(array, rowSize, stream); } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + serializeBinaryValues(array, length, buffer); + } + @Override public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index d8548328c..e59a714cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -226,6 +226,11 @@ default void serializeArray(Object array, int rowSize, DataOutputStream stream) throw new UnsupportedOperationException(getClass().getName()); } + /** Serializes the first {@code length} entries in {@code array} to {@code buffer}. */ + default void serializeArray(Object array, int length, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Deserializes an array with {@code rowSize} entries from {@code buffer}. */ default Object deserializeArray(ByteBuffer buffer, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index d5a32887b..5828fea38 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -162,6 +162,11 @@ public void serializeValue(Object value, DataOutputStream stream) { throw new UnsupportedOperationException(getDisplayName()); } + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public Decoder getDecoder(TSEncoding encoding) { return switch (encoding) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 86b4920fc..0f8adb0e5 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -229,6 +229,53 @@ public void testDeserializeValue() throws IOException { } } + @Test + public void testSerializeArrayToByteBuffer() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new boolean[] {true, false}}, + {TSDataType.INT32, new int[] {1, 2}}, + {TSDataType.DATE, new int[] {20260716, 20260717}}, + {TSDataType.INT64, new long[] {3L, 4L}}, + {TSDataType.TIMESTAMP, new long[] {5L, 6L}}, + {TSDataType.FLOAT, new float[] {1.25F, 2.5F}}, + {TSDataType.DOUBLE, new double[] {2.5D, 5.0D}}, + {TSDataType.TEXT, new Binary[] {binary, null}}, + {TSDataType.STRING, new Binary[] {binary, null}}, + {TSDataType.BLOB, new Binary[] {binary, null}}, + {TSDataType.OBJECT, new Binary[] {binary, null}} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object array = testCase[1]; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + type.serializeArray(array, 2, new DataOutputStream(output)); + + ByteBuffer buffer = ByteBuffer.allocate(type.serializedSize(array, 2)); + type.serializeArray(array, 2, buffer); + Assert.assertArrayEquals( + output.toByteArray(), Arrays.copyOf(buffer.array(), buffer.position())); + } + + LocalDate[] dates = {LocalDate.of(2026, 7, 16), null}; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Type dateType = Type.fromTsDataType(TSDataType.DATE); + dateType.serializeArray(dates, dates.length, new DataOutputStream(output)); + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * dates.length); + dateType.serializeArray(dates, dates.length, buffer); + Assert.assertArrayEquals(output.toByteArray(), buffer.array()); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).serializeArray(new Object[0], 0, ByteBuffer.allocate(0)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + @Test public void testGetDataPoint() { Object[][] testCases = { From fe0a1189e984f8497b68ca87d8f48347c8809ce7 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 09:43:45 +0800 Subject: [PATCH 083/131] refactor updateWithoutTimeColumn --- .../tsfile/read/common/block/TsBlock.java | 46 ++----------------- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 5 ++ .../tsfile/read/common/type/VectorType.java | 5 ++ .../tsfile/read/common/type/TypeTest.java | 42 +++++++++++++++++ 12 files changed, 96 insertions(+), 42 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java index 658024c14..de6158409 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java @@ -26,10 +26,10 @@ import org.apache.tsfile.read.common.IBatchDataIterator; import org.apache.tsfile.read.common.block.column.ColumnFactory; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.IMeasurementSchema; import java.util.Arrays; @@ -560,47 +560,9 @@ public void updateWithoutTimeColumn(int updateIdx, TsBlock sourceTsBlock, int so valueColumns[i].isNull()[updateIdx] = true; continue; } - switch (valueColumns[i].getDataType()) { - case BOOLEAN: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getBooleans()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getBoolean(sourceIndex); - break; - case INT32: - case DATE: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getInts()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getInt(sourceIndex); - break; - case INT64: - case TIMESTAMP: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getLongs()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getLong(sourceIndex); - break; - case FLOAT: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getFloats()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getFloat(sourceIndex); - break; - case DOUBLE: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getDoubles()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getDouble(sourceIndex); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getBinaries()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getBinary(sourceIndex); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format( - "error.read.tsblock_unknown_datatype", valueColumns[i].getDataType())); - } + valueColumns[i].isNull()[updateIdx] = false; + Type.fromTsDataType(valueColumns[i].getDataType()) + .setTo(sourceTsBlock.getValueColumns()[i], sourceIndex, valueColumns[i], updateIdx); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 6f435784b..e26ed2dcd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -198,6 +198,11 @@ public void setTo(BatchData from, Column to, int index) { to.getInts()[index] = from.getInt(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getInts()[toIndex] = from.getInt(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((int[]) toArray)[toIndex] = from.getInt(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 6d040894a..895df1d23 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -197,6 +197,11 @@ public void setTo(BatchData from, Column to, int index) { to.getLongs()[index] = from.getLong(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getLongs()[toIndex] = from.getLong(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((long[]) toArray)[toIndex] = from.getLong(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index f550a36d7..32713b17a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -177,6 +177,11 @@ public void setTo(BatchData from, Column to, int index) { to.getBinaries()[index] = from.getBinary(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBinaries()[toIndex] = from.getBinary(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index a5aef652c..008680f10 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -185,6 +185,11 @@ public void setTo(BatchData from, Column to, int index) { to.getBinaries()[index] = from.getBinary(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBinaries()[toIndex] = from.getBinary(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index cbe0a6cf1..0f22b8312 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -189,6 +189,11 @@ public void setTo(BatchData from, Column to, int index) { to.getBooleans()[index] = from.getBoolean(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBooleans()[toIndex] = from.getBoolean(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((boolean[]) toArray)[toIndex] = from.getBoolean(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index f70c12228..4f666f695 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -200,6 +200,11 @@ public void setTo(BatchData from, Column to, int index) { to.getDoubles()[index] = from.getDouble(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getDoubles()[toIndex] = from.getDouble(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((double[]) toArray)[toIndex] = from.getDouble(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index d689e01fd..2c355b956 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -198,6 +198,11 @@ public void setTo(BatchData from, Column to, int index) { to.getFloats()[index] = from.getFloat(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getFloats()[toIndex] = from.getFloat(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((float[]) toArray)[toIndex] = from.getFloat(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index a21eeeaa2..4462e0bf7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -185,6 +185,11 @@ public void setTo(BatchData from, Column to, int index) { to.getBinaries()[index] = from.getBinary(); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBinaries()[toIndex] = from.getBinary(fromIndex); + } + @Override public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index e59a714cd..fbd3c0c22 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -306,6 +306,11 @@ default void setTo(BatchData from, Column to, int index) { Messages.format("error.common.unsupported_data_type", from.getDataType())); } + /** Copies a value from {@code from} at {@code fromIndex} to {@code to} at {@code toIndex}. */ + default void setTo(Column from, int fromIndex, Column to, int toIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** * Copies a value from {@code from} at {@code fromIndex} to {@code toArray} at {@code toIndex}. */ diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 5828fea38..2988a02b8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -89,6 +89,11 @@ public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { ((long[]) toArray)[toIndex] = from.getLong(fromIndex); } + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 0f8adb0e5..1d6f080ed 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -878,6 +878,48 @@ public void testSetToArray() { Assert.assertEquals(testCase[2], Array.get(target, 0)); } + Object[][] columnTestCases = { + {TSDataType.BOOLEAN, 1, true, new boolean[1]}, + {TSDataType.INT32, 1, 1, new int[1]}, + {TSDataType.DATE, 2, 20260715, new int[1]}, + {TSDataType.INT64, 1, 2L, new long[1]}, + {TSDataType.TIMESTAMP, 1, 2L, new long[1]}, + {TSDataType.FLOAT, 1, 1.25F, new float[1]}, + {TSDataType.DOUBLE, 1, 2.5D, new double[1]}, + {TSDataType.TEXT, 1, binary, new Binary[1]}, + {TSDataType.STRING, 1, binary, new Binary[1]}, + {TSDataType.BLOB, 1, binary, new Binary[1]}, + {TSDataType.OBJECT, 1, binary, new Binary[1]} + }; + + for (Object[] testCase : columnTestCases) { + Column targetColumn = Mockito.mock(Column.class); + Object target = testCase[3]; + if (target instanceof boolean[]) { + Mockito.when(targetColumn.getBooleans()).thenReturn((boolean[]) target); + } else if (target instanceof int[]) { + Mockito.when(targetColumn.getInts()).thenReturn((int[]) target); + } else if (target instanceof long[]) { + Mockito.when(targetColumn.getLongs()).thenReturn((long[]) target); + } else if (target instanceof float[]) { + Mockito.when(targetColumn.getFloats()).thenReturn((float[]) target); + } else if (target instanceof double[]) { + Mockito.when(targetColumn.getDoubles()).thenReturn((double[]) target); + } else { + Mockito.when(targetColumn.getBinaries()).thenReturn((Binary[]) target); + } + Type.fromTsDataType((TSDataType) testCase[0]) + .setTo(column, (int) testCase[1], targetColumn, 0); + Assert.assertEquals(testCase[2], Array.get(target, 0)); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).setTo(column, 1, column, 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + try { Type.fromTsDataType(TSDataType.UNKNOWN).setTo(column, 1, new Object[1], 0); Assert.fail("Expected UnsupportedOperationException"); From cf393a49a218aa3a74abe1d9cf655374053decef Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 10:00:19 +0800 Subject: [PATCH 084/131] refactor rewrite --- .../write/UnSupportedDataTypeException.java | 11 +++++ .../org/apache/tsfile/read/common/Chunk.java | 48 ++----------------- .../read/common/type/AbstractIntType.java | 2 +- .../read/common/type/AbstractLongType.java | 4 +- .../read/common/type/AbstractVarcharType.java | 6 +-- .../tsfile/read/common/type/BlobType.java | 6 +-- .../tsfile/read/common/type/BooleanType.java | 4 +- .../tsfile/read/common/type/DoubleType.java | 4 +- .../tsfile/read/common/type/FloatType.java | 4 +- .../tsfile/read/common/type/ObjectType.java | 6 +-- .../apache/tsfile/read/common/type/Type.java | 11 ++++- .../tsfile/read/common/type/VectorType.java | 10 ++++ .../tsfile/read/common/type/TypeTest.java | 22 ++++++++- 13 files changed, 72 insertions(+), 66 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java b/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java index a7ab7dac1..a60a105be 100644 --- a/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java +++ b/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java @@ -21,6 +21,8 @@ public class UnSupportedDataTypeException extends RuntimeException { + private boolean isChecked = false; + public UnSupportedDataTypeException(String message) { super("Unsupported dataType: " + message); } @@ -28,4 +30,13 @@ public UnSupportedDataTypeException(String message) { public UnSupportedDataTypeException(String message, Throwable e) { super(message + e.getMessage()); } + + public UnSupportedDataTypeException setChecked(boolean checked) { + isChecked = checked; + return this; + } + + public boolean isChecked() { + return isChecked; + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java index bf83a4539..6d2b39098 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java @@ -29,6 +29,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; @@ -257,51 +258,8 @@ public Chunk rewrite(TSDataType newType, Chunk timeChunk) throws IOException { chunkHeader.getDataType(), point.getValue().getVector()[0].getValue()); } long timestamp = point.getTimestamp(); - switch (newType) { - case BOOLEAN: - chunkWriter.write( - timestamp, - convertedValue == null ? true : (boolean) convertedValue, - convertedValue == null); - break; - case DATE: - case INT32: - chunkWriter.write( - timestamp, - convertedValue == null ? Integer.MAX_VALUE : (int) convertedValue, - convertedValue == null); - break; - case TIMESTAMP: - case INT64: - chunkWriter.write( - timestamp, - convertedValue == null ? (long) Integer.MAX_VALUE : (long) convertedValue, - convertedValue == null); - break; - case FLOAT: - chunkWriter.write( - timestamp, - convertedValue == null ? (float) Integer.MAX_VALUE : (float) convertedValue, - convertedValue == null); - break; - case DOUBLE: - chunkWriter.write( - timestamp, - convertedValue == null ? (double) Integer.MAX_VALUE : (double) convertedValue, - convertedValue == null); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - chunkWriter.write( - timestamp, - convertedValue == null ? Binary.EMPTY_VALUE : (Binary) convertedValue, - convertedValue == null); - break; - default: - throw new IOException(Messages.format("error.read.chunk_unsupported_type", newType)); - } + Type.fromTsDataType(newType) + .write(chunkWriter, timestamp, convertedValue, convertedValue == null); } chunkWriter.sealCurrentPage(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index e26ed2dcd..0e8155167 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -343,7 +343,7 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { + public int serializedSize(Object array, int rowSize) { return Math.multiplyExact(Integer.BYTES, rowSize); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 895df1d23..d81a6c889 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -229,7 +229,7 @@ public DataPoint getDataPoint(String measurementId, long value) { @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { - writer.write(time, (long) value, isNull); + writer.write(time, isNull ? 0L : (long) value, isNull); } @Override @@ -342,7 +342,7 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { + public int serializedSize(Object array, int rowSize) { return Math.multiplyExact(Long.BYTES, rowSize); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 32713b17a..1ac22d652 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -204,7 +204,7 @@ public DataPoint getDataPoint(String measurementId, String value) { @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { - writer.write(time, (Binary) value, isNull); + writer.write(time, isNull ? Binary.EMPTY_VALUE : (Binary) value, isNull); } @Override @@ -303,8 +303,8 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { - return serializedSizeOfBinaryValues(column, rowSize); + public int serializedSize(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, rowSize); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 008680f10..379b7f386 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -212,7 +212,7 @@ public DataPoint getDataPoint(String measurementId, String value) { @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { - writer.write(time, (Binary) value, isNull); + writer.write(time, isNull ? Binary.EMPTY_VALUE : (Binary) value, isNull); } @Override @@ -316,8 +316,8 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { - return serializedSizeOfBinaryValues(column, rowSize); + public int serializedSize(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, rowSize); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 0f22b8312..4380ac048 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -221,7 +221,7 @@ public DataPoint getDataPoint(String measurementId, long value) { @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { - writer.write(time, (boolean) value, isNull); + writer.write(time, !isNull && (boolean) value, isNull); } @Override @@ -311,7 +311,7 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { + public int serializedSize(Object array, int rowSize) { return rowSize; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 4f666f695..059c28fb9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -232,7 +232,7 @@ public DataPoint getDataPoint(String measurementId, long value) { @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { - writer.write(time, (double) value, isNull); + writer.write(time, isNull ? 0.0D : (double) value, isNull); } @Override @@ -352,7 +352,7 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { + public int serializedSize(Object array, int rowSize) { return Math.multiplyExact(Double.BYTES, rowSize); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 2c355b956..cc8f29eb7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -230,7 +230,7 @@ public DataPoint getDataPoint(String measurementId, long value) { @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { - writer.write(time, (float) value, isNull); + writer.write(time, isNull ? 0.0F : (float) value, isNull); } @Override @@ -350,7 +350,7 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { + public int serializedSize(Object array, int rowSize) { return Math.multiplyExact(Float.BYTES, rowSize); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 4462e0bf7..0ba0336d2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -212,7 +212,7 @@ public DataPoint getDataPoint(String measurementId, String value) { @Override public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { - writer.write(time, (Binary) value, isNull); + writer.write(time, isNull ? Binary.EMPTY_VALUE : (Binary) value, isNull); } @Override @@ -310,8 +310,8 @@ public long estimateValueSize() { } @Override - public int serializedSize(Object column, int rowSize) { - return serializedSizeOfBinaryValues(column, rowSize); + public int serializedSize(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, rowSize); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index fbd3c0c22..d7ff294cb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -46,6 +46,13 @@ import java.nio.ByteBuffer; import java.util.List; +/** + * Type provides common interfaces associated with a specific TsDataType to remove uses of + * "switch(dataType)", which make the extension of datatypes very difficult. + * + *

Interfaces that are less generic should be defined using TypeService and checked in a proper + * context. + */ public interface Type { static Type fromTsDataType(TSDataType tsDataType) { @@ -215,8 +222,8 @@ default long estimateValueSize() { throw new UnsupportedOperationException(getClass().getName()); } - /** Returns the serialized byte size of the array column with {@code rowSize} entries. */ - default int serializedSize(Object column, int rowSize) { + /** Returns the serialized byte size of the array with {@code rowSize} entries. */ + default int serializedSize(Object array, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 2988a02b8..92a8938bf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -205,6 +205,16 @@ public Object getCurrentValue(BatchData batchData) { return batchData.getVector(); } + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsVector(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsVector((TsPrimitiveType[]) value); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 1d6f080ed..ec98bdf46 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -382,8 +382,28 @@ public void testWriteValueChunk() { Mockito.reset(writer); } + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 8L, null, true); + Mockito.verify(writer).write(8L, false, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT64).write(writer, 9L, null, true); + Mockito.verify(writer).write(9L, 0L, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 10L, null, true); + Mockito.verify(writer).write(10L, 0.0F, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 11L, null, true); + Mockito.verify(writer).write(11L, 0.0D, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TEXT).write(writer, 12L, null, true); + Mockito.verify(writer).write(12L, Binary.EMPTY_VALUE, true); + Mockito.reset(writer); + try { - Type.fromTsDataType(TSDataType.VECTOR).write(writer, 8L, null, true); + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 13L, null, true); Assert.fail("Expected UnSupportedDataTypeException"); } catch (UnSupportedDataTypeException ignored) { // Expected. From e96b0ff52c2d3917e9548e311690be165b8f1610 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 10:13:50 +0800 Subject: [PATCH 085/131] enhance type check --- .../write/UnSupportedDataTypeException.java | 2 + .../apache/tsfile/read/common/type/Type.java | 5 ++ .../read/common/type/service/TypeService.java | 11 +++- .../org/apache/tsfile/utils/TypeServices.java | 60 +++++++++++-------- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java b/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java index a60a105be..2ec35339c 100644 --- a/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java +++ b/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java @@ -21,6 +21,8 @@ public class UnSupportedDataTypeException extends RuntimeException { + // true if the exception is raised by a datatype that is known to be unsupported, + // otherwise, it is caused by an unknown/uncovered datatype private boolean isChecked = false; public UnSupportedDataTypeException(String message) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index d7ff294cb..05181e3e4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -227,6 +227,11 @@ default int serializedSize(Object array, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } + /** Returns the serialized byte size of the array within the given range. */ + default int serializedSize(Object array, int startRow, int endRow) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Serializes the array column with {@code rowSize} entries. */ default void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java index b732ad86b..e8b3b3b3d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java @@ -20,6 +20,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.write.UnSupportedDataTypeException; /** * A TypeService wraps a switch(dataType) clause into a checked service class. Implementations @@ -42,7 +43,15 @@ default void check() { } if (type != null) { - call(type); + try { + call(type); + } catch (UnSupportedDataTypeException e) { + if (!e.isChecked()) { + // the exception is caused by an uncovered datatype, throw it to fail any tests + // involving this service + throw e; + } + } } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 82e48dbe3..a84ebd841 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -103,7 +103,8 @@ public final class TypeServices { }; case ROW, UNKNOWN, VECTOR -> (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { - throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); }; }; @@ -293,7 +294,8 @@ public final class TypeServices { allSatisfy, isDeleted, paginationController) -> { - throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); }; }; @@ -342,7 +344,8 @@ public final class TypeServices { }; case ROW, UNKNOWN, VECTOR -> (decoder, buffer, timestamp, isDeleted) -> { - throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); }; }; @@ -431,7 +434,8 @@ public final class TypeServices { }; case ROW, UNKNOWN, VECTOR -> (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { - throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); }; }; @@ -447,7 +451,8 @@ public final class TypeServices { case ROW, UNKNOWN, VECTOR -> writer -> { throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", type.getTypeEnum())); + Messages.format("error.write.type_not_supported", type.getTypeEnum())) + .setChecked(true); }; }; @@ -465,10 +470,11 @@ public final class TypeServices { case TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> maxPointNumber -> { throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.RLE, - type.getTypeEnum())); + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.RLE, + type.getTypeEnum())) + .setChecked(true); }; }; @@ -486,10 +492,11 @@ public final class TypeServices { case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> maxPointNumber -> { throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.TS_2DIFF, - type.getTypeEnum())); + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.TS_2DIFF, + type.getTypeEnum())) + .setChecked(true); }; }; @@ -511,10 +518,11 @@ public final class TypeServices { UNKNOWN, VECTOR -> throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.GORILLA_V1, - type.getTypeEnum())); + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.GORILLA_V1, + type.getTypeEnum())) + .setChecked(true); }; public static final TypeService REGULAR_ENCODER_SERVICE = @@ -524,10 +532,11 @@ public final class TypeServices { case INT64, TIMESTAMP -> new RegularDataEncoder.LongRegularEncoder(); case BOOLEAN, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.REGULAR, - type.getTypeEnum())); + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.REGULAR, + type.getTypeEnum())) + .setChecked(true); }; public static final TypeService GORILLA_V2_ENCODER_SERVICE = @@ -539,10 +548,11 @@ public final class TypeServices { case INT64, TIMESTAMP -> new LongGorillaEncoder(); case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.GORILLA, - type.getTypeEnum())); + Messages.format( + "error.encoding.ts_encoding_builder_unsupported_type", + TSEncoding.GORILLA, + type.getTypeEnum())) + .setChecked(true); }; private TypeServices() {} From 0b499648454902ea94681a7842374e632e6b15ee Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 10:21:49 +0800 Subject: [PATCH 086/131] add Type.serializedSize --- .../tsfile/read/common/type/AbstractLongType.java | 2 +- .../apache/tsfile/read/common/type/AbstractType.java | 10 +++++++--- .../tsfile/read/common/type/AbstractVarcharType.java | 5 +++++ .../org/apache/tsfile/read/common/type/BlobType.java | 5 +++++ .../org/apache/tsfile/read/common/type/ObjectType.java | 5 +++++ .../java/org/apache/tsfile/read/common/type/Type.java | 2 +- 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index d81a6c889..92c460da0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -345,7 +345,7 @@ public long estimateValueSize() { public int serializedSize(Object array, int rowSize) { return Math.multiplyExact(Long.BYTES, rowSize); } - + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index f13e16d3c..1390b07c4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -79,10 +79,14 @@ protected void checkValueType(Object value, Class expectedClass, String expec } } - protected int serializedSizeOfBinaryValues(Object column, int rowSize) { - Binary[] binaryValues = (Binary[]) column; + protected int serializedSizeOfBinaryValues(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, 0, rowSize); + } + + protected int serializedSizeOfBinaryValues(Object array, int startRow, int endRow) { + Binary[] binaryValues = (Binary[]) array; int size = 0; - for (int i = 0; i < rowSize; i++) { + for (int i = startRow; i < endRow; i++) { size = Math.addExact(size, Byte.BYTES); if (binaryValues[i] != null) { size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(binaryValues[i])); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 1ac22d652..e22152d21 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -307,6 +307,11 @@ public int serializedSize(Object array, int rowSize) { return serializedSizeOfBinaryValues(array, rowSize); } + @Override + public int serializedSize(Object array, int startRow, int endRow) { + return serializedSizeOfBinaryValues(array, startRow, endRow); + } + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 379b7f386..4b48345b4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -320,6 +320,11 @@ public int serializedSize(Object array, int rowSize) { return serializedSizeOfBinaryValues(array, rowSize); } + @Override + public int serializedSize(Object array, int startRow, int endRow) { + return serializedSizeOfBinaryValues(array, startRow, endRow); + } + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 0ba0336d2..53b9c9f2c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -314,6 +314,11 @@ public int serializedSize(Object array, int rowSize) { return serializedSizeOfBinaryValues(array, rowSize); } + @Override + public int serializedSize(Object array, int startRow, int endRow) { + return serializedSizeOfBinaryValues(array, startRow, endRow); + } + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 05181e3e4..4d01e6a43 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -229,7 +229,7 @@ default int serializedSize(Object array, int rowSize) { /** Returns the serialized byte size of the array within the given range. */ default int serializedSize(Object array, int startRow, int endRow) { - throw new UnsupportedOperationException(getClass().getName()); + return serializedSize(array, endRow - startRow); } /** Serializes the array column with {@code rowSize} entries. */ From f41019e7de1a28b2923e7ce88c2726c741252dfe Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 10:37:30 +0800 Subject: [PATCH 087/131] refactor rewrite --- .../org/apache/tsfile/read/common/Chunk.java | 29 +-------- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 7 ++- .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 6 ++ .../tsfile/read/common/type/VectorType.java | 6 ++ .../tsfile/read/common/type/TypeTest.java | 62 +++++++++++++++++++ 12 files changed, 116 insertions(+), 29 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java index 6d2b39098..d41ad14a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java @@ -34,7 +34,6 @@ import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.read.reader.chunk.TableChunkReader; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -304,33 +303,7 @@ public Chunk rewrite(TSDataType newType) throws IOException { if (convertedValue == null) { throw new IOException(Messages.format("error.read.chunk_non_aligned_null", timestamp)); } - switch (newType) { - case BOOLEAN: - chunkWriter.write(timestamp, (boolean) convertedValue); - break; - case DATE: - case INT32: - chunkWriter.write(timestamp, (int) convertedValue); - break; - case TIMESTAMP: - case INT64: - chunkWriter.write(timestamp, (long) convertedValue); - break; - case FLOAT: - chunkWriter.write(timestamp, (float) convertedValue); - break; - case DOUBLE: - chunkWriter.write(timestamp, (double) convertedValue); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - chunkWriter.write(timestamp, (Binary) convertedValue); - break; - default: - throw new IOException(Messages.format("error.read.chunk_unsupported_type", newType)); - } + Type.fromTsDataType(newType).write(chunkWriter, timestamp, convertedValue); } chunkWriter.sealCurrentPage(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 0e8155167..0c37f68d0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -261,6 +261,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((int[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (int) value); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 92c460da0..9300868a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -260,6 +260,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((long[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (long) value); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsLong(); @@ -345,7 +350,7 @@ public long estimateValueSize() { public int serializedSize(Object array, int rowSize) { return Math.multiplyExact(Long.BYTES, rowSize); } - + @Override public void serializeArray(Object array, int rowSize, DataOutputStream stream) throws IOException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index e22152d21..a834352e8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -235,6 +235,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((Binary[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (Binary) value); + } + @Override public TsPrimitiveType getTsPrimitiveType() { return new TsPrimitiveType.TsBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 4b48345b4..e6bf36050 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -243,6 +243,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((Binary[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (Binary) value); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BLOB); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 4380ac048..a60ce1ee6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -252,6 +252,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((boolean[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (boolean) value); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.BOOLEAN); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 059c28fb9..aeb41dcbe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -263,6 +263,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((double[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (double) value); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.DOUBLE); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index cc8f29eb7..caf43fdb9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -261,6 +261,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((float[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (float) value); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.FLOAT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 53b9c9f2c..d24c5123c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -237,6 +237,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((Binary[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (Binary) value); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.OBJECT); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 4d01e6a43..7d575f196 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -393,6 +393,12 @@ default void write(ChunkWriterImpl writer, long time, Object column, int rowInde Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes a value to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, Object value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Creates a column with {@code positionCount} capacity for this type. */ default Column createColumnWithMaxPosition(int positionCount) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 92a8938bf..ed0200b30 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -121,6 +121,12 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + @Override public void init(BatchData batchData) { batchData.initVectorValues(); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index ec98bdf46..61fd202f4 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -276,6 +276,24 @@ public void testSerializeArrayToByteBuffer() throws IOException { } } + @Test + public void testSerializedSizeRange() { + Assert.assertEquals( + Integer.BYTES * 2, + Type.fromTsDataType(TSDataType.INT32).serializedSize(new int[] {1, 2, 3}, 1, 3)); + + Binary[] binaries = { + new Binary("excluded", StandardCharsets.UTF_8), null, new Binary("x", StandardCharsets.UTF_8) + }; + int expectedSize = Byte.BYTES + Byte.BYTES + Integer.BYTES + 1; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type type = Type.fromTsDataType(dataType); + Assert.assertEquals(expectedSize, type.serializedSize(binaries, 1, 3)); + Assert.assertEquals(0, type.serializedSize(binaries, 2, 2)); + } + } + @Test public void testGetDataPoint() { Object[][] testCases = { @@ -689,6 +707,50 @@ public void testWriteNonAlignedTabletValueChunk() { } } + @Test + public void testWriteNonAlignedValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1L, true}, + {TSDataType.INT32, 2L, 1}, + {TSDataType.DATE, 3L, 20260717}, + {TSDataType.INT64, 4L, 2L}, + {TSDataType.TIMESTAMP, 5L, 3L}, + {TSDataType.FLOAT, 6L, 1.25F}, + {TSDataType.DOUBLE, 7L, 2.5D} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]).write(writer, (long) testCase[1], testCase[2]); + } + + Mockito.verify(writer).write(1L, true); + Mockito.verify(writer).write(2L, 1); + Mockito.verify(writer).write(3L, 20260717); + Mockito.verify(writer).write(4L, 2L); + Mockito.verify(writer).write(5L, 3L); + Mockito.verify(writer).write(6L, 1.25F); + Mockito.verify(writer).write(7L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 8L, binary); + Mockito.verify(writer).write(8L, binary); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(writer, 9L, 1L); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + @Test public void testToString() { Object[][] testCases = { From 69cef4c8381fce9a48140e5feb3f1c081b6d7519 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 10:47:26 +0800 Subject: [PATCH 088/131] refactor TsBlockBuilder --- .../read/common/block/TsBlockBuilder.java | 51 ++----------------- .../read/common/type/AbstractIntType.java | 7 +++ .../read/common/type/AbstractLongType.java | 7 +++ .../read/common/type/AbstractVarcharType.java | 7 +++ .../tsfile/read/common/type/BlobType.java | 7 +++ .../tsfile/read/common/type/BooleanType.java | 7 +++ .../tsfile/read/common/type/DateType.java | 9 ++++ .../tsfile/read/common/type/DoubleType.java | 7 +++ .../tsfile/read/common/type/FloatType.java | 7 +++ .../tsfile/read/common/type/ObjectType.java | 7 +++ .../apache/tsfile/read/common/type/Type.java | 7 +++ .../tsfile/read/common/type/UnknownType.java | 9 ++++ .../tsfile/read/common/type/VectorType.java | 9 ++++ 13 files changed, 94 insertions(+), 47 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java index 456167304..f61884868 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java @@ -34,6 +34,7 @@ import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import java.util.List; @@ -104,54 +105,10 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List Date: Fri, 17 Jul 2026 10:51:30 +0800 Subject: [PATCH 089/131] refactor buildValueColumnBuilders --- .../read/common/block/TsBlockBuilder.java | 38 ++----------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java index f61884868..c557b4951 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java @@ -26,12 +26,6 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; -import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; -import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; -import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; -import org.apache.tsfile.read.common.block.column.IntColumnBuilder; -import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.read.common.type.Type; @@ -141,35 +135,9 @@ public void buildValueColumnBuilders(List types) { valueColumnBuilders = new ColumnBuilder[types.size()]; int initialExpectedEntries = timeColumnBuilder.getPositionCount(); for (int i = 0; i < valueColumnBuilders.length; i++) { - switch (types.get(i)) { - case BOOLEAN: - valueColumnBuilders[i] = - new BooleanColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case INT32: - valueColumnBuilders[i] = - new IntColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), - initialExpectedEntries, - TSDataType.INT32); - break; - case DATE: - valueColumnBuilders[i] = - new IntColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), - initialExpectedEntries, - TSDataType.DATE); - break; - case INT64: - case TIMESTAMP: - valueColumnBuilders[i] = - new LongColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case FLOAT: - valueColumnBuilders[i] = - new FloatColumnBuilder( + valueColumnBuilders[i] = + Type.fromTsDataType(types.get(i)) + .createColumnBuilder( tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); break; case DOUBLE: From 36efcdbbd71f485c01cb4e2b27c603689a0bf394 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 10:51:35 +0800 Subject: [PATCH 090/131] refactor buildValueColumnBuilders --- .../read/common/block/TsBlockBuilder.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java index c557b4951..e8868d04b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java @@ -139,24 +139,6 @@ public void buildValueColumnBuilders(List types) { Type.fromTsDataType(types.get(i)) .createColumnBuilder( tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case DOUBLE: - valueColumnBuilders[i] = - new DoubleColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueColumnBuilders[i] = - new BinaryColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.tsblock_builder_unknown_type", types.get(i))); - } } } From c00177f8d6eb1a44945bc2bc8712614d80e8c26c Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 11:03:14 +0800 Subject: [PATCH 091/131] add createStatistics --- .../file/metadata/statistics/Statistics.java | 30 ++----------------- .../read/common/type/AbstractIntType.java | 6 ++++ .../read/common/type/AbstractLongType.java | 6 ++++ .../tsfile/read/common/type/AbstractType.java | 4 +-- .../read/common/type/AbstractVarcharType.java | 6 ++++ .../tsfile/read/common/type/BlobType.java | 6 ++++ .../tsfile/read/common/type/BooleanType.java | 6 ++++ .../tsfile/read/common/type/DateType.java | 7 +++++ .../tsfile/read/common/type/DoubleType.java | 6 ++++ .../tsfile/read/common/type/FloatType.java | 6 ++++ .../tsfile/read/common/type/ObjectType.java | 6 ++++ .../tsfile/read/common/type/StringType.java | 7 +++++ .../read/common/type/TimestampType.java | 7 +++++ .../apache/tsfile/read/common/type/Type.java | 15 ++++++++-- .../tsfile/read/common/type/VectorType.java | 6 ++++ .../tsfile/read/common/type/TypeTest.java | 30 +++++++++++++++++++ 16 files changed, 122 insertions(+), 32 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java index 323115aee..98743ae19 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java @@ -23,6 +23,7 @@ import org.apache.tsfile.exception.filter.StatisticsClassException; import org.apache.tsfile.exception.write.UnknownColumnTypeException; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; @@ -104,34 +105,7 @@ public abstract class Statistics { * @throws UnknownColumnTypeException if the type is unknown */ public static Statistics getStatsByType(TSDataType type) { - switch (type) { - case INT32: - return new IntegerStatistics(); - case INT64: - return new LongStatistics(); - case TEXT: - return new BinaryStatistics(); - case BOOLEAN: - return new BooleanStatistics(); - case DOUBLE: - return new DoubleStatistics(); - case FLOAT: - return new FloatStatistics(); - case VECTOR: - return new TimeStatistics(); - case DATE: - return new DateStatistics(); - case TIMESTAMP: - return new TimestampStatistics(); - case STRING: - return new StringStatistics(); - case BLOB: - return new BlobStatistics(); - case OBJECT: - return new ObjectStatistics(); - default: - throw new UnknownColumnTypeException(type.toString()); - } + return Type.fromTsDataType(type).createStatistics(); } public static long getSizeByType(TSDataType type) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index a93626162..28328a276 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.IntegerStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; @@ -144,6 +145,11 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES; } + @Override + public Statistics createStatistics() { + return new IntegerStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getInt()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index c7957a317..6a1b3c7bd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -34,6 +34,7 @@ import org.apache.tsfile.encoding.decoder.RegularDataDecoder; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.LongStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; @@ -143,6 +144,11 @@ public int getOneItemMaxSize(int valveLength) { return Long.BYTES; } + @Override + public Statistics createStatistics() { + return new LongStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getLong()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 1390b07c4..10fe2cac6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -161,8 +161,8 @@ protected boolean binaryArrayEquals(Object left, Object right, int rowSize) { } @Override - public Object getValue(Object column, int rowIndex) { - return Array.get(column, rowIndex); + public Object getValue(Object array, int rowIndex) { + return Array.get(array, rowIndex); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 6954b46cc..eb333db28 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.BinaryStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; @@ -123,6 +124,11 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; } + @Override + public Statistics createStatistics() { + return new BinaryStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index ff46d925a..99bb79953 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -26,6 +26,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.BlobStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; @@ -129,6 +130,11 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; } + @Override + public Statistics createStatistics() { + return new BlobStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index a645c37e8..78389449c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -28,6 +28,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.BooleanStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; @@ -135,6 +136,11 @@ public int getOneItemMaxSize(int valveLength) { return Byte.BYTES; } + @Override + public Statistics createStatistics() { + return new BooleanStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBoolean()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index f89c371e3..7cbd80551 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -23,6 +23,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.DateStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; @@ -65,6 +67,11 @@ public DataPoint getDataPoint(String measurementId, String value) { return new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); } + @Override + public Statistics createStatistics() { + return new DateStatistics(); + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 9c94c5332..126e3d971 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -34,6 +34,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.DoubleStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; @@ -146,6 +147,11 @@ public int getOneItemMaxSize(int valveLength) { return Double.BYTES; } + @Override + public Statistics createStatistics() { + return new DoubleStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getDouble()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 5871d7353..c6825044c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -33,6 +33,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.FloatStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; @@ -144,6 +145,11 @@ public int getOneItemMaxSize(int valveLength) { return Float.BYTES; } + @Override + public Statistics createStatistics() { + return new FloatStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getFloat()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 0587e0509..e62863d2d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -26,6 +26,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.ObjectStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; @@ -131,6 +132,11 @@ public int getOneItemMaxSize(int valveLength) { return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; } + @Override + public Statistics createStatistics() { + return new ObjectStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java index 58dac7f22..4315c98cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java @@ -20,6 +20,8 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.file.metadata.statistics.StringStatistics; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.utils.Binary; @@ -29,6 +31,11 @@ public class StringType extends AbstractVarcharType { private StringType() {} + @Override + public Statistics createStatistics() { + return new StringStatistics(); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.STRING); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java index 210058970..49f060e97 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java @@ -20,6 +20,8 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.file.metadata.statistics.TimestampStatistics; import org.apache.tsfile.read.common.Field; public class TimestampType extends AbstractLongType { @@ -28,6 +30,11 @@ public class TimestampType extends AbstractLongType { private TimestampType() {} + @Override + public Statistics createStatistics() { + return new TimestampStatistics(); + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.TIMESTAMP); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 17bc23d1f..0f4ac6b66 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -25,6 +25,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.UnknownColumnTypeException; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -269,8 +270,13 @@ default boolean arrayEquals(Object left, Object right, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } - /** Returns the value at {@code rowIndex} from an array column in Tablet. */ - default Object getValue(Object column, int rowIndex) { + /** Returns the value at {@code rowIndex} from an array. */ + default Object getValue(Object array, int rowIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the value at {@code rowIndex} from an array. */ + default TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { throw new UnsupportedOperationException(getClass().getName()); } @@ -436,6 +442,11 @@ default int getOneItemMaxSize(int valveLength) { throw new UnsupportedOperationException(getClass().getName()); } + /** Creates empty statistics for this type. */ + default Statistics createStatistics() { + throw new UnknownColumnTypeException(getTypeEnum().toString()); + } + /** Updates statistics with a timestamp-value pair. */ default void update(Statistics stats, long timestamp, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 5c5adbb43..a9085efca 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -31,6 +31,7 @@ import org.apache.tsfile.encoding.decoder.RegularDataDecoder; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.file.metadata.statistics.TimeStatistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; @@ -205,6 +206,11 @@ public int getOneItemMaxSize(int valveLength) { throw new UnsupportedOperationException(getDisplayName()); } + @Override + public Statistics createStatistics() { + return new TimeStatistics(); + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { throw new UnsupportedOperationException(getDisplayName()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 61fd202f4..259ab8b4f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -23,6 +23,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.encoder.PlainEncoder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.UnknownColumnTypeException; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; @@ -1377,6 +1378,35 @@ public void testGetOneItemMaxSize() { } } + @Test + public void testCreateStatistics() { + for (TSDataType dataType : + new TSDataType[] { + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.DATE, + TSDataType.INT64, + TSDataType.TIMESTAMP, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.OBJECT, + TSDataType.VECTOR + }) { + Assert.assertEquals(dataType, Type.fromTsDataType(dataType).createStatistics().getType()); + Assert.assertEquals(dataType, Statistics.getStatsByType(dataType).getType()); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).createStatistics(); + Assert.fail("Expected UnknownColumnTypeException"); + } catch (UnknownColumnTypeException ignored) { + // Expected. + } + } + @Test public void testUpdateStatistics() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From fe3dd2e2ab125cca1be9ba99686252b75800df6f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 11:18:04 +0800 Subject: [PATCH 092/131] add getValueAsTsPrimitiveType --- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DateType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../tsfile/read/common/type/RowType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 2 +- .../tsfile/read/common/type/VectorType.java | 5 ++ .../tsfile/read/common/type/TypeTest.java | 49 +++++++++++++++++++ 13 files changed, 105 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 28328a276..461100053 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -283,6 +283,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsInt((int) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsInt(((int[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return c.getInt(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 6a1b3c7bd..baf68d472 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -282,6 +282,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsLong((long) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsLong(((long[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return (int) c.getLong(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index eb333db28..5e289837d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -257,6 +257,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsBinary((Binary) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsBinary(((Binary[]) array)[rowIndex]); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 99bb79953..469b4f96d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -272,6 +272,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsBinary((Binary) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsBinary(((Binary[]) array)[rowIndex]); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 78389449c..333926ef9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -281,6 +281,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsBoolean((boolean) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsBoolean(((boolean[]) array)[rowIndex]); + } + @Override public boolean getBoolean(Column c, int position) { return c.getBoolean(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 7cbd80551..dae8ae01b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -62,6 +62,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsInt((int) value, TSDataType.DATE); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsInt(getDateInt(array, rowIndex), TSDataType.DATE); + } + @Override public DataPoint getDataPoint(String measurementId, String value) { return new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 126e3d971..bbac32be7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -292,6 +292,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsDouble((double) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsDouble(((double[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return (int) c.getDouble(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index c6825044c..7e3ab914e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -290,6 +290,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsFloat((float) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsFloat(((float[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return (int) c.getFloat(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index e62863d2d..f37277198 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -266,6 +266,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsBinary((Binary) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsBinary(((Binary[]) array)[rowIndex]); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java index 455aaa639..c9339afd8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java @@ -84,6 +84,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsVector((TsPrimitiveType[]) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsVector(((TsPrimitiveType[][]) array)[rowIndex]); + } + public static Field field(String name, Type type) { return new Field(Optional.of(name), type); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 0f4ac6b66..331e24edf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -275,7 +275,7 @@ default Object getValue(Object array, int rowIndex) { throw new UnsupportedOperationException(getClass().getName()); } - /** Returns the value at {@code rowIndex} from an array. */ + /** Returns the value at {@code rowIndex} from an array as a TsPrimitiveType. */ default TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index a9085efca..cc01f6851 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -236,6 +236,11 @@ public TsPrimitiveType getTsPrimitiveType(Object value) { return new TsPrimitiveType.TsVector((TsPrimitiveType[]) value); } + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsVector(((TsPrimitiveType[][]) array)[rowIndex]); + } + @Override public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 259ab8b4f..63f758e3a 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -59,6 +59,55 @@ public class TypeTest { private static final int OFFSET = 2; private static final int VALUE_LENGTH = 10; + @Test + public void testGetValueAsTsPrimitiveType() { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + TsPrimitiveType[] vector = {new TsPrimitiveType.TsInt(1), null}; + Object[][] testCases = { + {TSDataType.BOOLEAN, new boolean[] {false, true}, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new int[] {0, 1}, new TsPrimitiveType.TsInt(1)}, + { + TSDataType.DATE, + new int[] {0, 20260717}, + new TsPrimitiveType.TsInt(20260717, TSDataType.DATE) + }, + { + TSDataType.DATE, + new LocalDate[] {null, LocalDate.of(2026, 7, 17)}, + new TsPrimitiveType.TsInt(20260717, TSDataType.DATE) + }, + {TSDataType.INT64, new long[] {0L, 1L}, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.TIMESTAMP, new long[] {0L, 1L}, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.FLOAT, new float[] {0.0F, 1.25F}, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, new double[] {0.0D, 2.5D}, new TsPrimitiveType.TsDouble(2.5D)}, + {TSDataType.TEXT, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.STRING, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.BLOB, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.OBJECT, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + { + TSDataType.VECTOR, + new TsPrimitiveType[][] {null, vector}, + new TsPrimitiveType.TsVector(vector) + } + }; + + for (Object[] testCase : testCases) { + TsPrimitiveType expected = (TsPrimitiveType) testCase[2]; + TsPrimitiveType actual = + Type.fromTsDataType((TSDataType) testCase[0]).getValueAsTsPrimitiveType(testCase[1], 1); + Assert.assertEquals(expected, actual); + Assert.assertEquals(expected.getDataType(), actual.getDataType()); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN) + .getValueAsTsPrimitiveType(new Object[] {new Object()}, 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + @Test public void testToBytes() { byte[] valueBytes = new byte[16]; From 7df7dfd79356671700d8317d01fd19b6c1bcbade Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 11:23:30 +0800 Subject: [PATCH 093/131] refactor getSizeByType --- .../file/metadata/statistics/Statistics.java | 29 +------------------ .../read/common/type/AbstractIntType.java | 5 ++++ .../read/common/type/AbstractLongType.java | 5 ++++ .../read/common/type/AbstractVarcharType.java | 5 ++++ .../tsfile/read/common/type/BlobType.java | 5 ++++ .../tsfile/read/common/type/BooleanType.java | 5 ++++ .../tsfile/read/common/type/DateType.java | 5 ++++ .../tsfile/read/common/type/DoubleType.java | 5 ++++ .../tsfile/read/common/type/FloatType.java | 5 ++++ .../tsfile/read/common/type/ObjectType.java | 5 ++++ .../tsfile/read/common/type/StringType.java | 5 ++++ .../read/common/type/TimestampType.java | 5 ++++ .../apache/tsfile/read/common/type/Type.java | 5 ++++ .../tsfile/read/common/type/VectorType.java | 5 ++++ .../read/common/type/service/TypeService.java | 3 ++ .../tsfile/read/common/type/TypeTest.java | 12 +++++++- 16 files changed, 80 insertions(+), 29 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java index 98743ae19..2b9ff6f29 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java @@ -109,34 +109,7 @@ public static Statistics getStatsByType(TSDataType type) } public static long getSizeByType(TSDataType type) { - switch (type) { - case INT32: - return IntegerStatistics.INSTANCE_SIZE; - case INT64: - return LongStatistics.INSTANCE_SIZE; - case TEXT: - return BinaryStatistics.INSTANCE_SIZE; - case BOOLEAN: - return BooleanStatistics.INSTANCE_SIZE; - case DOUBLE: - return DoubleStatistics.INSTANCE_SIZE; - case FLOAT: - return FloatStatistics.INSTANCE_SIZE; - case VECTOR: - return TimeStatistics.INSTANCE_SIZE; - case DATE: - return DateStatistics.INSTANCE_SIZE; - case TIMESTAMP: - return TimestampStatistics.INSTANCE_SIZE; - case STRING: - return StringStatistics.INSTANCE_SIZE; - case BLOB: - return BlobStatistics.INSTANCE_SIZE; - case OBJECT: - return ObjectStatistics.INSTANCE_SIZE; - default: - throw new UnknownColumnTypeException(type.toString()); - } + return Type.fromTsDataType(type).getStatisticsInstanceSize(); } public abstract TSDataType getType(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 461100053..5a2c07665 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -150,6 +150,11 @@ public Statistics createStatistics() { return new IntegerStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return IntegerStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getInt()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index baf68d472..d69c02b3a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -149,6 +149,11 @@ public Statistics createStatistics() { return new LongStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return LongStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getLong()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 5e289837d..8fc9c959d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -129,6 +129,11 @@ public Statistics createStatistics() { return new BinaryStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return BinaryStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 469b4f96d..f7827be83 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -135,6 +135,11 @@ public Statistics createStatistics() { return new BlobStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return BlobStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 333926ef9..5f3677fba 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -141,6 +141,11 @@ public Statistics createStatistics() { return new BooleanStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return BooleanStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBoolean()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index dae8ae01b..ed18c7a50 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -77,6 +77,11 @@ public Statistics createStatistics() { return new DateStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return DateStatistics.INSTANCE_SIZE; + } + @Override public void write( ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index bbac32be7..50d2a47ca 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -152,6 +152,11 @@ public Statistics createStatistics() { return new DoubleStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return DoubleStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getDouble()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 7e3ab914e..687408a7d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -150,6 +150,11 @@ public Statistics createStatistics() { return new FloatStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return FloatStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getFloat()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index f37277198..916c2d42e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -137,6 +137,11 @@ public Statistics createStatistics() { return new ObjectStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return ObjectStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { stats.update(timestamp, value.getBinary()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java index 4315c98cd..f5d54ed7d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java @@ -36,6 +36,11 @@ public Statistics createStatistics() { return new StringStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return StringStatistics.INSTANCE_SIZE; + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.STRING); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java index 49f060e97..143c577f0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java @@ -35,6 +35,11 @@ public Statistics createStatistics() { return new TimestampStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return TimestampStatistics.INSTANCE_SIZE; + } + @Override public Field getField(Object value) { Field field = new Field(TSDataType.TIMESTAMP); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 331e24edf..f2411c27b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -447,6 +447,11 @@ default Statistics createStatistics() { throw new UnknownColumnTypeException(getTypeEnum().toString()); } + /** Returns the estimated instance size of statistics for this type. */ + default long getStatisticsInstanceSize() { + throw new UnknownColumnTypeException(getTypeEnum().toString()); + } + /** Updates statistics with a timestamp-value pair. */ default void update(Statistics stats, long timestamp, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index cc01f6851..dccccebde 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -211,6 +211,11 @@ public Statistics createStatistics() { return new TimeStatistics(); } + @Override + public long getStatisticsInstanceSize() { + return TimeStatistics.INSTANCE_SIZE; + } + @Override public void update(Statistics stats, long timestamp, TsPrimitiveType value) { throw new UnsupportedOperationException(getDisplayName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java index e8b3b3b3d..7c4b28615 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java @@ -27,6 +27,9 @@ * should cover all Types that can be instanced from TsDataTypes, and throw * UnsupportedDataTypeException when an unrecognized type is encountered. * + *

Use enhanced-switch and list all branches in the implementation to expose uncovered datatypes + * during compilation. + * * @param the return type of the service. */ public interface TypeService { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 63f758e3a..fe843f3a6 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -1444,8 +1444,11 @@ public void testCreateStatistics() { TSDataType.OBJECT, TSDataType.VECTOR }) { - Assert.assertEquals(dataType, Type.fromTsDataType(dataType).createStatistics().getType()); + Type type = Type.fromTsDataType(dataType); + Assert.assertEquals(dataType, type.createStatistics().getType()); Assert.assertEquals(dataType, Statistics.getStatsByType(dataType).getType()); + Assert.assertTrue(type.getStatisticsInstanceSize() > 0); + Assert.assertEquals(type.getStatisticsInstanceSize(), Statistics.getSizeByType(dataType)); } try { @@ -1454,6 +1457,13 @@ public void testCreateStatistics() { } catch (UnknownColumnTypeException ignored) { // Expected. } + + try { + Statistics.getSizeByType(TSDataType.UNKNOWN); + Assert.fail("Expected UnknownColumnTypeException"); + } catch (UnknownColumnTypeException ignored) { + // Expected. + } } @Test From 5f9f1429c3cbc9f2b2c8de3cdb392c99d381171e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 11:55:27 +0800 Subject: [PATCH 094/131] refactor ValueFilterApi --- .../read/filter/factory/ValueFilterApi.java | 384 +++--------------- .../tsfile/utils/ValueFilterTypeServices.java | 381 +++++++++++++++++ 2 files changed, 431 insertions(+), 334 deletions(-) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java index 4423792f5..2b25915d5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java @@ -21,20 +21,12 @@ import org.apache.tsfile.common.regexp.LikePattern; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; -import org.apache.tsfile.read.filter.operator.BinaryFilterOperators; -import org.apache.tsfile.read.filter.operator.BooleanFilterOperators; -import org.apache.tsfile.read.filter.operator.DoubleFilterOperators; import org.apache.tsfile.read.filter.operator.ExtractTimeFilterOperators; import org.apache.tsfile.read.filter.operator.ExtractValueFilterOperators; -import org.apache.tsfile.read.filter.operator.FloatFilterOperators; -import org.apache.tsfile.read.filter.operator.IntegerFilterOperators; -import org.apache.tsfile.read.filter.operator.LongFilterOperators; -import org.apache.tsfile.read.filter.operator.StringFilterOperators; import org.apache.tsfile.read.filter.operator.ValueIsNotNullOperator; import org.apache.tsfile.read.filter.operator.ValueIsNullOperator; -import org.apache.tsfile.utils.Binary; import java.time.ZoneId; import java.util.Objects; @@ -42,6 +34,21 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_BETWEEN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_GT_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_GT_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_IN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_LIKE_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_LT_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_LT_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_BETWEEN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_IN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_LIKE_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_REGEXP_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_REGEXP_FILTER_SERVICE; + public class ValueFilterApi { public static final int DEFAULT_MEASUREMENT_INDEX = 0; @@ -56,166 +63,43 @@ private ValueFilterApi() { public static Filter gt(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueGt(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueGt(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueGt(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueGt(measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueGt(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueGt(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueGt(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_GT_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, value); } public static Filter gtEq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueGtEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueGtEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueGtEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueGtEq( - measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueGtEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueGtEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueGtEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_GT_EQ_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value); } public static Filter lt(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueLt(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueLt(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueLt(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueLt(measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueLt(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueLt(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueLt(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_LT_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, value); } public static Filter ltEq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueLtEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueLtEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueLtEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueLtEq( - measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueLtEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueLtEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueLtEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_LT_EQ_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value); } public static Filter eq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueEq(measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_EQ_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, value); } public static Filter notEq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueNotEq( - measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueNotEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueNotEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_EQ_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value); } public static Filter isNull(int measurementIndex) { @@ -231,35 +115,9 @@ public static Filter between( // constant cannot be null Objects.requireNonNull(value1, CONSTANT_CANNOT_BE_NULL_MSG); Objects.requireNonNull(value2, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueBetweenAnd( - measurementIndex, (boolean) value1, (boolean) value2); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).intValue(), ((Number) value2).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).longValue(), ((Number) value2).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).floatValue(), ((Number) value2).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - case STRING: - return new StringFilterOperators.ValueBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_BETWEEN_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value1, value2); } public static Filter notBetween( @@ -267,195 +125,53 @@ public static Filter notBetween( // constant cannot be null Objects.requireNonNull(value1, CONSTANT_CANNOT_BE_NULL_MSG); Objects.requireNonNull(value2, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotBetweenAnd( - measurementIndex, (boolean) value1, (boolean) value2); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).intValue(), ((Number) value2).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).longValue(), ((Number) value2).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).floatValue(), ((Number) value2).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - case STRING: - return new StringFilterOperators.ValueNotBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_BETWEEN_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value1, value2); } public static Filter like(int measurementIndex, LikePattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueLike(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueLike(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueLike(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueLike(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueLike(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueLike(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueLike(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_LIKE_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static Filter notLike(int measurementIndex, LikePattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotLike(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotLike(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotLike(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueNotLike(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueNotLike(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotLike(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueNotLike(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_LIKE_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static Filter regexp(int measurementIndex, Pattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueRegexp(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueRegexp(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueRegexp(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueRegexp(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueRegexp(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueRegexp(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueRegexp(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_REGEXP_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static Filter notRegexp(int measurementIndex, Pattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueNotRegexp(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_REGEXP_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static > Filter in( int measurementIndex, Set values, TSDataType type) { // constants cannot be null Objects.requireNonNull(values, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueIn(measurementIndex, (Set) values); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueIn(measurementIndex, (Set) values); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueIn(measurementIndex, (Set) values); - case FLOAT: - return new FloatFilterOperators.ValueIn(measurementIndex, (Set) values); - case DOUBLE: - return new DoubleFilterOperators.ValueIn(measurementIndex, (Set) values); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueIn(measurementIndex, (Set) values); - case STRING: - return new StringFilterOperators.ValueIn(measurementIndex, (Set) values); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_IN_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, values); } public static > Filter notIn( int measurementIndex, Set values, TSDataType type) { // constants cannot be null Objects.requireNonNull(values, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case FLOAT: - return new FloatFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case DOUBLE: - return new DoubleFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case STRING: - return new StringFilterOperators.ValueNotIn(measurementIndex, (Set) values); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_IN_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, values); } public static Filter extractValueGt( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java new file mode 100644 index 000000000..d7770856d --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.apache.tsfile.common.regexp.LikePattern; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.filter.operator.BinaryFilterOperators; +import org.apache.tsfile.read.filter.operator.BooleanFilterOperators; +import org.apache.tsfile.read.filter.operator.DoubleFilterOperators; +import org.apache.tsfile.read.filter.operator.FloatFilterOperators; +import org.apache.tsfile.read.filter.operator.IntegerFilterOperators; +import org.apache.tsfile.read.filter.operator.LongFilterOperators; +import org.apache.tsfile.read.filter.operator.StringFilterOperators; + +import java.util.Set; +import java.util.regex.Pattern; + +@SuppressWarnings("unchecked") +public final class ValueFilterTypeServices { + + public static final TypeService VALUE_GT_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueGt(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueGt(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueGt(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueGt(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueGt(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueGt(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueGt(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_GT_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueGtEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueGtEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueGtEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueGtEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueGtEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueGtEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueGtEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_LT_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueLt(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueLt(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueLt(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueLt(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueLt(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueLt(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueLt(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_LT_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueLtEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueLtEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueLtEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueLtEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueLtEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueLtEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueLtEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueNotEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueNotEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueNotEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueNotEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueNotEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueNotEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueNotEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_BETWEEN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value1, value2) -> + new BooleanFilterOperators.ValueBetweenAnd( + index, (boolean) value1, (boolean) value2); + case INT32, DATE -> + (index, value1, value2) -> + new IntegerFilterOperators.ValueBetweenAnd( + index, ((Number) value1).intValue(), ((Number) value2).intValue()); + case INT64, TIMESTAMP -> + (index, value1, value2) -> + new LongFilterOperators.ValueBetweenAnd( + index, ((Number) value1).longValue(), ((Number) value2).longValue()); + case FLOAT -> + (index, value1, value2) -> + new FloatFilterOperators.ValueBetweenAnd( + index, ((Number) value1).floatValue(), ((Number) value2).floatValue()); + case DOUBLE -> + (index, value1, value2) -> + new DoubleFilterOperators.ValueBetweenAnd( + index, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); + case TEXT, BLOB -> + (index, value1, value2) -> + new BinaryFilterOperators.ValueBetweenAnd( + index, (Binary) value1, (Binary) value2); + case STRING -> + (index, value1, value2) -> + new StringFilterOperators.ValueBetweenAnd( + index, (Binary) value1, (Binary) value2); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value1, value2) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_BETWEEN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value1, value2) -> + new BooleanFilterOperators.ValueNotBetweenAnd( + index, (boolean) value1, (boolean) value2); + case INT32, DATE -> + (index, value1, value2) -> + new IntegerFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).intValue(), ((Number) value2).intValue()); + case INT64, TIMESTAMP -> + (index, value1, value2) -> + new LongFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).longValue(), ((Number) value2).longValue()); + case FLOAT -> + (index, value1, value2) -> + new FloatFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).floatValue(), ((Number) value2).floatValue()); + case DOUBLE -> + (index, value1, value2) -> + new DoubleFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); + case TEXT, BLOB -> + (index, value1, value2) -> + new BinaryFilterOperators.ValueNotBetweenAnd( + index, (Binary) value1, (Binary) value2); + case STRING -> + (index, value1, value2) -> + new StringFilterOperators.ValueNotBetweenAnd( + index, (Binary) value1, (Binary) value2); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value1, value2) -> unsupportedType(type); + }; + + public static final TypeService VALUE_LIKE_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueLike::new; + case INT32, DATE -> IntegerFilterOperators.ValueLike::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueLike::new; + case FLOAT -> FloatFilterOperators.ValueLike::new; + case DOUBLE -> DoubleFilterOperators.ValueLike::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueLike::new; + case STRING -> StringFilterOperators.ValueLike::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_LIKE_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueNotLike::new; + case INT32, DATE -> IntegerFilterOperators.ValueNotLike::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueNotLike::new; + case FLOAT -> FloatFilterOperators.ValueNotLike::new; + case DOUBLE -> DoubleFilterOperators.ValueNotLike::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueNotLike::new; + case STRING -> StringFilterOperators.ValueNotLike::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_REGEXP_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueRegexp::new; + case INT32, DATE -> IntegerFilterOperators.ValueRegexp::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueRegexp::new; + case FLOAT -> FloatFilterOperators.ValueRegexp::new; + case DOUBLE -> DoubleFilterOperators.ValueRegexp::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueRegexp::new; + case STRING -> StringFilterOperators.ValueRegexp::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_REGEXP_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueNotRegexp::new; + case INT32, DATE -> IntegerFilterOperators.ValueNotRegexp::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueNotRegexp::new; + case FLOAT -> FloatFilterOperators.ValueNotRegexp::new; + case DOUBLE -> DoubleFilterOperators.ValueNotRegexp::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueNotRegexp::new; + case STRING -> StringFilterOperators.ValueNotRegexp::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_IN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, values) -> new BooleanFilterOperators.ValueIn(index, (Set) values); + case INT32, DATE -> + (index, values) -> new IntegerFilterOperators.ValueIn(index, (Set) values); + case INT64, TIMESTAMP -> + (index, values) -> new LongFilterOperators.ValueIn(index, (Set) values); + case FLOAT -> + (index, values) -> new FloatFilterOperators.ValueIn(index, (Set) values); + case DOUBLE -> + (index, values) -> new DoubleFilterOperators.ValueIn(index, (Set) values); + case TEXT, BLOB -> + (index, values) -> new BinaryFilterOperators.ValueIn(index, (Set) values); + case STRING -> + (index, values) -> new StringFilterOperators.ValueIn(index, (Set) values); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, values) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_IN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, values) -> + new BooleanFilterOperators.ValueNotIn(index, (Set) values); + case INT32, DATE -> + (index, values) -> + new IntegerFilterOperators.ValueNotIn(index, (Set) values); + case INT64, TIMESTAMP -> + (index, values) -> new LongFilterOperators.ValueNotIn(index, (Set) values); + case FLOAT -> + (index, values) -> new FloatFilterOperators.ValueNotIn(index, (Set) values); + case DOUBLE -> + (index, values) -> + new DoubleFilterOperators.ValueNotIn(index, (Set) values); + case TEXT, BLOB -> + (index, values) -> + new BinaryFilterOperators.ValueNotIn(index, (Set) values); + case STRING -> + (index, values) -> + new StringFilterOperators.ValueNotIn(index, (Set) values); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, values) -> unsupportedType(type); + }; + + private ValueFilterTypeServices() {} + + private static Filter unsupportedType(Type type) { + throw new UnsupportedOperationException( + Messages.format("error.read.filter_api_unsupported_type", type.getTypeEnum())); + } + + @FunctionalInterface + public interface SingleValueFilterFactory { + Filter create(int measurementIndex, Object value); + } + + @FunctionalInterface + public interface BetweenValueFilterFactory { + Filter create(int measurementIndex, Object value1, Object value2); + } + + @FunctionalInterface + public interface LikeFilterFactory { + Filter create(int measurementIndex, LikePattern pattern); + } + + @FunctionalInterface + public interface RegexpFilterFactory { + Filter create(int measurementIndex, Pattern pattern); + } + + @FunctionalInterface + public interface SetFilterFactory { + Filter create(int measurementIndex, Set values); + } +} From 126f9eb758c2ca0caa1548cc0754483141662faa Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 12:05:41 +0800 Subject: [PATCH 095/131] refactor generate --- .../java/org/apache/tsfile/DataGenerator.java | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java b/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java index 159e2ea65..ee9e72d14 100644 --- a/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java +++ b/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java @@ -21,32 +21,34 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import java.time.LocalDate; public class DataGenerator { + + private static final TypeService GENERATE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> index -> index % 2 == 0; + case INT32 -> index -> index; + case INT64, TIMESTAMP -> index -> (long) index; + case FLOAT -> index -> (float) index; + case DOUBLE -> index -> (double) index; + case DATE -> index -> LocalDate.of(2024, 1, index % 30 + 1); + case TEXT, STRING, BLOB -> + index -> new Binary(String.valueOf(index), TSFileConfig.STRING_CHARSET); + case OBJECT, ROW, UNKNOWN, VECTOR -> index -> null; + }; + public static Object generate(TSDataType type, int index) { - switch (type) { - case INT32: - return index; - case INT64: - case TIMESTAMP: - return (long) index; - case FLOAT: - return (float) index; - case DOUBLE: - return (double) index; - case BOOLEAN: - return index % 2 == 0; - case DATE: - return LocalDate.of(2024, 1, index % 30 + 1); - case TEXT: - case STRING: - case BLOB: - return new Binary(String.valueOf(index), TSFileConfig.STRING_CHARSET); - default: - return null; - } + return GENERATE_VALUE_SERVICE.call(Type.fromTsDataType(type)).generate(index); + } + + @FunctionalInterface + private interface ValueGenerator { + Object generate(int index); } } From c34ecc8cdd1081896aefc3a2a5fa7ea9823a140f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 12:13:49 +0800 Subject: [PATCH 096/131] refactor TsFileSequenceRead --- .../java/org/apache/tsfile/DataGenerator.java | 4 ++ .../org/apache/tsfile/TsFileSequenceRead.java | 56 ++++++++++--------- .../tsfile/utils/ValueFilterTypeServices.java | 17 ++++++ 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java b/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java index ee9e72d14..d33f822c2 100644 --- a/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java +++ b/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java @@ -43,6 +43,10 @@ public class DataGenerator { case OBJECT, ROW, UNKNOWN, VECTOR -> index -> null; }; + static { + GENERATE_VALUE_SERVICE.check(); + } + public static Object generate(TSDataType type, int index) { return GENERATE_VALUE_SERVICE.call(Type.fromTsDataType(type)).generate(index); } diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java b/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java index ecd3fdd27..5873784e7 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java @@ -34,6 +34,8 @@ import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.reader.page.PageReader; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -51,6 +53,25 @@ public class TsFileSequenceRead { public static final String POINT_IN_PAGE = "\t\tpoints in the page: "; private static int MASK = 0x80; + private static final TypeService PAGE_VALUE_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> Decoder::readBoolean; + case INT32, DATE -> Decoder::readInt; + case INT64, TIMESTAMP -> Decoder::readLong; + case FLOAT -> Decoder::readFloat; + case DOUBLE -> Decoder::readDouble; + case TEXT, STRING, BLOB -> Decoder::readBinary; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (decoder, buffer) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + }; + }; + + static { + PAGE_VALUE_READER_SERVICE.check(); + } + @SuppressWarnings({ "squid:S3776", "squid:S106" @@ -133,6 +154,8 @@ public static void main(String[] args) throws IOException { } } else if ((header.getChunkType() & TsFileConstant.VALUE_COLUMN_MASK) == TsFileConstant.VALUE_COLUMN_MASK) { // Value Chunk + PageValueReader valueReader = + PAGE_VALUE_READER_SERVICE.call(Type.fromTsDataType(header.getDataType())); int pointNum = 0; byte[] bitmap = null; if (pageData.hasRemaining()) { @@ -149,33 +172,7 @@ public static void main(String[] args) throws IOException { } continue; } - Object value; - switch (header.getDataType()) { - case BOOLEAN: - value = valueDecoder.readBoolean(pageData); - break; - case INT32: - case DATE: - value = valueDecoder.readInt(pageData); - break; - case INT64: - case TIMESTAMP: - value = valueDecoder.readLong(pageData); - break; - case FLOAT: - value = valueDecoder.readFloat(pageData); - break; - case DOUBLE: - value = valueDecoder.readDouble(pageData); - break; - case TEXT: - case STRING: - case BLOB: - value = valueDecoder.readBinary(pageData); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(header.getDataType())); - } + Object value = valueReader.read(valueDecoder, pageData); if (printDetail) { System.out.println("\t\t\tvalue: " + value); } @@ -240,4 +237,9 @@ public static void main(String[] args) throws IOException { } } } + + @FunctionalInterface + private interface PageValueReader { + Object read(Decoder decoder, ByteBuffer buffer); + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java index d7770856d..a868ae8a5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java @@ -347,6 +347,23 @@ public final class ValueFilterTypeServices { case OBJECT, ROW, UNKNOWN, VECTOR -> (index, values) -> unsupportedType(type); }; + static { + VALUE_GT_FILTER_SERVICE.check(); + VALUE_GT_EQ_FILTER_SERVICE.check(); + VALUE_LT_FILTER_SERVICE.check(); + VALUE_LT_EQ_FILTER_SERVICE.check(); + VALUE_EQ_FILTER_SERVICE.check(); + VALUE_NOT_EQ_FILTER_SERVICE.check(); + VALUE_BETWEEN_FILTER_SERVICE.check(); + VALUE_NOT_BETWEEN_FILTER_SERVICE.check(); + VALUE_LIKE_FILTER_SERVICE.check(); + VALUE_NOT_LIKE_FILTER_SERVICE.check(); + VALUE_REGEXP_FILTER_SERVICE.check(); + VALUE_NOT_REGEXP_FILTER_SERVICE.check(); + VALUE_IN_FILTER_SERVICE.check(); + VALUE_NOT_IN_FILTER_SERVICE.check(); + } + private ValueFilterTypeServices() {} private static Filter unsupportedType(Type type) { From 597de8d435ce26576c0479204899190d0396f5bd Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 12:17:35 +0800 Subject: [PATCH 097/131] refactor ValueConverter --- .../apache/tsfile/tools/ValueConverter.java | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java b/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java index a45de52f9..e75179e4e 100644 --- a/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java +++ b/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java @@ -19,6 +19,8 @@ package org.apache.tsfile.tools; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import java.nio.charset.StandardCharsets; @@ -30,6 +32,30 @@ public class ValueConverter { + private static final TypeService FROM_STRING_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (value, isMeasurement, timePrecision) -> Boolean.valueOf(value); + case INT32 -> (value, isMeasurement, timePrecision) -> Integer.valueOf(value); + case INT64 -> (value, isMeasurement, timePrecision) -> Long.valueOf(value); + case FLOAT -> (value, isMeasurement, timePrecision) -> Float.valueOf(value); + case DOUBLE -> (value, isMeasurement, timePrecision) -> Double.valueOf(value); + case TEXT, STRING -> + (value, isMeasurement, timePrecision) -> + isMeasurement ? new Binary(value, StandardCharsets.UTF_8) : value; + case BLOB -> + (value, isMeasurement, timePrecision) -> + new Binary(value.getBytes(StandardCharsets.UTF_8)); + case DATE -> (value, isMeasurement, timePrecision) -> parseDate(value); + case TIMESTAMP -> + (value, isMeasurement, timePrecision) -> parseTimestamp(value, timePrecision); + case OBJECT, ROW, UNKNOWN, VECTOR -> (value, isMeasurement, timePrecision) -> value; + }; + + static { + FROM_STRING_SERVICE.check(); + } + public static Object convert(Object value, TSDataType targetType, boolean isMeasurement) { return convert(value, targetType, isMeasurement, "ms"); } @@ -47,32 +73,9 @@ public static Object convert( private static Object fromString( String value, TSDataType targetType, boolean isMeasurement, String timePrecision) { - switch (targetType) { - case BOOLEAN: - return Boolean.valueOf(value); - case INT32: - return Integer.valueOf(value); - case INT64: - return Long.valueOf(value); - case FLOAT: - return Float.valueOf(value); - case DOUBLE: - return Double.valueOf(value); - case TEXT: - case STRING: - if (isMeasurement) { - return new Binary(value, StandardCharsets.UTF_8); - } - return value; - case BLOB: - return new Binary(value.getBytes(StandardCharsets.UTF_8)); - case DATE: - return parseDate(value); - case TIMESTAMP: - return parseTimestamp(value, timePrecision); - default: - return value; - } + return FROM_STRING_SERVICE + .call(Type.fromTsDataType(targetType)) + .convert(value, isMeasurement, timePrecision); } private static Object fromObject( @@ -220,4 +223,9 @@ private static long toEpochLong(Object value, String timePrecision) { } return parseTimestamp(value.toString(), timePrecision); } + + @FunctionalInterface + private interface StringConverter { + Object convert(String value, boolean isMeasurement, String timePrecision); + } } From 63f8763846f9210c9366eb120509bbed8fec75fd Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 12:29:57 +0800 Subject: [PATCH 098/131] refactor ValueConverter --- .../apache/tsfile/tools/ValueConverter.java | 145 ++++++++++-------- 1 file changed, 80 insertions(+), 65 deletions(-) diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java b/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java index e75179e4e..75145f5e0 100644 --- a/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java +++ b/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java @@ -52,8 +52,80 @@ public class ValueConverter { case OBJECT, ROW, UNKNOWN, VECTOR -> (value, isMeasurement, timePrecision) -> value; }; + private static final TypeService FROM_OBJECT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (value, isMeasurement, timePrecision) -> + value instanceof Boolean ? value : Boolean.valueOf(value.toString()); + case INT32 -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Integer) { + return value; + } + if (value instanceof Number) { + return ((Number) value).intValue(); + } + return Integer.valueOf(value.toString()); + }; + case INT64 -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Long) { + return value; + } + if (value instanceof Number) { + return ((Number) value).longValue(); + } + return Long.valueOf(value.toString()); + }; + case FLOAT -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Float) { + return value; + } + if (value instanceof Number) { + return ((Number) value).floatValue(); + } + return Float.valueOf(value.toString()); + }; + case DOUBLE -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Double) { + return value; + } + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + return Double.valueOf(value.toString()); + }; + case TEXT, STRING -> + (value, isMeasurement, timePrecision) -> { + if (isMeasurement) { + return value instanceof Binary + ? value + : new Binary(value.toString(), StandardCharsets.UTF_8); + } + return value instanceof String ? value : value.toString(); + }; + case BLOB -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Binary) { + return value; + } + if (value instanceof byte[]) { + return new Binary((byte[]) value); + } + return new Binary(value.toString().getBytes(StandardCharsets.UTF_8)); + }; + case DATE -> (value, isMeasurement, timePrecision) -> toLocalDate(value); + case TIMESTAMP -> + (value, isMeasurement, timePrecision) -> toEpochLong(value, timePrecision); + case OBJECT, ROW, UNKNOWN, VECTOR -> (value, isMeasurement, timePrecision) -> value; + }; + static { FROM_STRING_SERVICE.check(); + FROM_OBJECT_SERVICE.check(); } public static Object convert(Object value, TSDataType targetType, boolean isMeasurement) { @@ -80,71 +152,9 @@ private static Object fromString( private static Object fromObject( Object value, TSDataType targetType, boolean isMeasurement, String timePrecision) { - switch (targetType) { - case BOOLEAN: - if (value instanceof Boolean) { - return value; - } - return Boolean.valueOf(value.toString()); - case INT32: - if (value instanceof Integer) { - return value; - } - if (value instanceof Number) { - return ((Number) value).intValue(); - } - return Integer.valueOf(value.toString()); - case INT64: - if (value instanceof Long) { - return value; - } - if (value instanceof Number) { - return ((Number) value).longValue(); - } - return Long.valueOf(value.toString()); - case FLOAT: - if (value instanceof Float) { - return value; - } - if (value instanceof Number) { - return ((Number) value).floatValue(); - } - return Float.valueOf(value.toString()); - case DOUBLE: - if (value instanceof Double) { - return value; - } - if (value instanceof Number) { - return ((Number) value).doubleValue(); - } - return Double.valueOf(value.toString()); - case TEXT: - case STRING: - if (isMeasurement) { - if (value instanceof Binary) { - return value; - } - return new Binary(value.toString(), StandardCharsets.UTF_8); - } - if (value instanceof String) { - return value; - } - return value.toString(); - case BLOB: - if (value instanceof Binary) { - return value; - } - if (value instanceof byte[]) { - return new Binary((byte[]) value); - } - return new Binary(value.toString().getBytes(StandardCharsets.UTF_8)); - case DATE: - return toLocalDate(value); - case TIMESTAMP: - return toEpochLong(value, timePrecision); - default: - return value; - } + return FROM_OBJECT_SERVICE + .call(Type.fromTsDataType(targetType)) + .convert(value, isMeasurement, timePrecision); } private static LocalDate parseDate(String value) { @@ -228,4 +238,9 @@ private static long toEpochLong(Object value, String timePrecision) { private interface StringConverter { Object convert(String value, boolean isMeasurement, String timePrecision); } + + @FunctionalInterface + private interface ObjectConverter { + Object convert(Object value, boolean isMeasurement, String timePrecision); + } } From 5a078a885dcf3cd644789984621b0cfb242469f2 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 14:42:47 +0800 Subject: [PATCH 099/131] refactor TSEncodingBuilder --- .../encoding/encoder/TSEncodingBuilder.java | 74 ++----- .../tsfile/utils/EncodingTypeServices.java | 198 ++++++++++++++++++ .../org/apache/tsfile/utils/TypeServices.java | 118 ----------- 3 files changed, 210 insertions(+), 180 deletions(-) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/utils/EncodingTypeServices.java diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index f15ceedf5..0db1cc211 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -26,7 +26,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.EncodingTypeServices; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; @@ -145,7 +145,9 @@ public static class Rle extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - return TypeServices.RLE_ENCODER_SERVICE.call(Type.fromTsDataType(type)).build(maxPointNumber); + return EncodingTypeServices.RLE_ENCODER_SERVICE + .call(Type.fromTsDataType(type)) + .build(maxPointNumber); } /** @@ -186,7 +188,7 @@ public static class Ts2Diff extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - return TypeServices.TS_2DIFF_ENCODER_SERVICE + return EncodingTypeServices.TS_2DIFF_ENCODER_SERVICE .call(Type.fromTsDataType(type)) .build(maxPointNumber); } @@ -227,7 +229,7 @@ public static class GorillaV1 extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - return TypeServices.GORILLA_V1_ENCODER_SERVICE.call(Type.fromTsDataType(type)); + return EncodingTypeServices.GORILLA_V1_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -259,7 +261,7 @@ public static class Regular extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - return TypeServices.REGULAR_ENCODER_SERVICE.call(Type.fromTsDataType(type)); + return EncodingTypeServices.REGULAR_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -273,7 +275,7 @@ public static class GorillaV2 extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - return TypeServices.GORILLA_V2_ENCODER_SERVICE.call(Type.fromTsDataType(type)); + return EncodingTypeServices.GORILLA_V2_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -285,21 +287,7 @@ public void initFromProps(Map props) { public static class Sprintz extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new IntSprintzEncoder(); - case INT64: - case TIMESTAMP: - return new LongSprintzEncoder(); - case FLOAT: - return new FloatSprintzEncoder(); - case DOUBLE: - return new DoubleSprintzEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.SPRINTZ, type)); - } + return EncodingTypeServices.SPRINTZ_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -312,21 +300,7 @@ public static class RLBE extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new IntRLBE(); - case INT64: - case TIMESTAMP: - return new LongRLBE(); - case FLOAT: - return new FloatRLBE(); - case DOUBLE: - return new DoubleRLBE(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.RLBE, type)); - } + return EncodingTypeServices.RLBE_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -356,17 +330,7 @@ public static class Zigzag extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new IntZigzagEncoder(); - case INT64: - case TIMESTAMP: - return new LongZigzagEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.ZIGZAG, type)); - } + return EncodingTypeServices.ZIGZAG_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -380,21 +344,7 @@ public static class Chimp extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case FLOAT: - return new SinglePrecisionChimpEncoder(); - case DOUBLE: - return new DoublePrecisionChimpEncoder(); - case INT32: - case DATE: - return new IntChimpEncoder(); - case INT64: - case TIMESTAMP: - return new LongChimpEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.CHIMP, type)); - } + return EncodingTypeServices.CHIMP_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/EncodingTypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/EncodingTypeServices.java new file mode 100644 index 000000000..145f5abd0 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/EncodingTypeServices.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.apache.tsfile.encoding.encoder.DeltaBinaryEncoder; +import org.apache.tsfile.encoding.encoder.DoublePrecisionChimpEncoder; +import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV1; +import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV2; +import org.apache.tsfile.encoding.encoder.DoubleRLBE; +import org.apache.tsfile.encoding.encoder.DoubleSprintzEncoder; +import org.apache.tsfile.encoding.encoder.Encoder; +import org.apache.tsfile.encoding.encoder.FloatEncoder; +import org.apache.tsfile.encoding.encoder.FloatRLBE; +import org.apache.tsfile.encoding.encoder.FloatSprintzEncoder; +import org.apache.tsfile.encoding.encoder.IntChimpEncoder; +import org.apache.tsfile.encoding.encoder.IntGorillaEncoder; +import org.apache.tsfile.encoding.encoder.IntRLBE; +import org.apache.tsfile.encoding.encoder.IntRleEncoder; +import org.apache.tsfile.encoding.encoder.IntSprintzEncoder; +import org.apache.tsfile.encoding.encoder.IntZigzagEncoder; +import org.apache.tsfile.encoding.encoder.LongChimpEncoder; +import org.apache.tsfile.encoding.encoder.LongGorillaEncoder; +import org.apache.tsfile.encoding.encoder.LongRLBE; +import org.apache.tsfile.encoding.encoder.LongRleEncoder; +import org.apache.tsfile.encoding.encoder.LongSprintzEncoder; +import org.apache.tsfile.encoding.encoder.LongZigzagEncoder; +import org.apache.tsfile.encoding.encoder.RegularDataEncoder; +import org.apache.tsfile.encoding.encoder.SinglePrecisionChimpEncoder; +import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV1; +import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV2; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +public final class EncodingTypeServices { + + private static final String ERROR_MSG_KEY = "error.encoding.ts_encoding_builder_unsupported_type"; + + public static final TypeService RLE_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE, BOOLEAN -> maxPointNumber -> new IntRleEncoder(); + case INT64, TIMESTAMP -> maxPointNumber -> new LongRleEncoder(); + case FLOAT -> + maxPointNumber -> + new FloatEncoder(TSEncoding.RLE, TSDataType.FLOAT, maxPointNumber); + case DOUBLE -> + maxPointNumber -> + new FloatEncoder(TSEncoding.RLE, TSDataType.DOUBLE, maxPointNumber); + case TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + maxPointNumber -> { + throw unsupported(TSEncoding.RLE, type.getTypeEnum()); + }; + }; + + public static final TypeService TS_2DIFF_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> maxPointNumber -> new DeltaBinaryEncoder.IntDeltaEncoder(); + case INT64, TIMESTAMP -> maxPointNumber -> new DeltaBinaryEncoder.LongDeltaEncoder(); + case FLOAT -> + maxPointNumber -> + new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT, maxPointNumber); + case DOUBLE -> + maxPointNumber -> + new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.DOUBLE, maxPointNumber); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + maxPointNumber -> { + throw unsupported(TSEncoding.TS_2DIFF, type.getTypeEnum()); + }; + }; + + public static final TypeService GORILLA_V1_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionEncoderV1(); + case DOUBLE -> new DoublePrecisionEncoderV1(); + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + TEXT, + BLOB, + STRING, + OBJECT, + ROW, + UNKNOWN, + VECTOR -> + throw unsupported(TSEncoding.GORILLA_V1, type.getTypeEnum()); + }; + + public static final TypeService REGULAR_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new RegularDataEncoder.IntRegularEncoder(); + case INT64, TIMESTAMP -> new RegularDataEncoder.LongRegularEncoder(); + case BOOLEAN, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.REGULAR, type.getTypeEnum()); + }; + + public static final TypeService GORILLA_V2_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionEncoderV2(); + case DOUBLE -> new DoublePrecisionEncoderV2(); + case INT32, DATE -> new IntGorillaEncoder(); + case INT64, TIMESTAMP -> new LongGorillaEncoder(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.GORILLA, type.getTypeEnum()); + }; + + public static final TypeService SPRINTZ_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new IntSprintzEncoder(); + case INT64, TIMESTAMP -> new LongSprintzEncoder(); + case FLOAT -> new FloatSprintzEncoder(); + case DOUBLE -> new DoubleSprintzEncoder(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.SPRINTZ, type.getTypeEnum()); + }; + + public static final TypeService RLBE_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new IntRLBE(); + case INT64, TIMESTAMP -> new LongRLBE(); + case FLOAT -> new FloatRLBE(); + case DOUBLE -> new DoubleRLBE(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.RLBE, type.getTypeEnum()); + }; + + public static final TypeService ZIGZAG_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new IntZigzagEncoder(); + case INT64, TIMESTAMP -> new LongZigzagEncoder(); + case BOOLEAN, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.ZIGZAG, type.getTypeEnum()); + }; + + public static final TypeService CHIMP_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionChimpEncoder(); + case DOUBLE -> new DoublePrecisionChimpEncoder(); + case INT32, DATE -> new IntChimpEncoder(); + case INT64, TIMESTAMP -> new LongChimpEncoder(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.CHIMP, type.getTypeEnum()); + }; + + static { + RLE_ENCODER_SERVICE.check(); + TS_2DIFF_ENCODER_SERVICE.check(); + GORILLA_V1_ENCODER_SERVICE.check(); + REGULAR_ENCODER_SERVICE.check(); + GORILLA_V2_ENCODER_SERVICE.check(); + SPRINTZ_ENCODER_SERVICE.check(); + RLBE_ENCODER_SERVICE.check(); + ZIGZAG_ENCODER_SERVICE.check(); + CHIMP_ENCODER_SERVICE.check(); + } + + private EncodingTypeServices() {} + + private static UnSupportedDataTypeException unsupported(TSEncoding encoding, Object type) { + return new UnSupportedDataTypeException(Messages.format(ERROR_MSG_KEY, encoding, type)) + .setChecked(true); + } + + @FunctionalInterface + public interface EncoderBuilder { + + Encoder build(int maxPointNumber); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index a84ebd841..18f644bba 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -21,20 +21,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; -import org.apache.tsfile.encoding.encoder.DeltaBinaryEncoder; -import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV1; -import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV2; -import org.apache.tsfile.encoding.encoder.Encoder; -import org.apache.tsfile.encoding.encoder.FloatEncoder; -import org.apache.tsfile.encoding.encoder.IntGorillaEncoder; -import org.apache.tsfile.encoding.encoder.IntRleEncoder; -import org.apache.tsfile.encoding.encoder.LongGorillaEncoder; -import org.apache.tsfile.encoding.encoder.LongRleEncoder; -import org.apache.tsfile.encoding.encoder.RegularDataEncoder; -import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV1; -import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV2; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.TsBlockBuilder; @@ -456,105 +443,6 @@ public final class TypeServices { }; }; - public static final TypeService RLE_ENCODER_SERVICE = - type -> - switch (type.getTypeEnum()) { - case INT32, DATE, BOOLEAN -> maxPointNumber -> new IntRleEncoder(); - case INT64, TIMESTAMP -> maxPointNumber -> new LongRleEncoder(); - case FLOAT -> - maxPointNumber -> - new FloatEncoder(TSEncoding.RLE, TSDataType.FLOAT, maxPointNumber); - case DOUBLE -> - maxPointNumber -> - new FloatEncoder(TSEncoding.RLE, TSDataType.DOUBLE, maxPointNumber); - case TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> - maxPointNumber -> { - throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.RLE, - type.getTypeEnum())) - .setChecked(true); - }; - }; - - public static final TypeService TS_2DIFF_ENCODER_SERVICE = - type -> - switch (type.getTypeEnum()) { - case INT32, DATE -> maxPointNumber -> new DeltaBinaryEncoder.IntDeltaEncoder(); - case INT64, TIMESTAMP -> maxPointNumber -> new DeltaBinaryEncoder.LongDeltaEncoder(); - case FLOAT -> - maxPointNumber -> - new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT, maxPointNumber); - case DOUBLE -> - maxPointNumber -> - new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.DOUBLE, maxPointNumber); - case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> - maxPointNumber -> { - throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.TS_2DIFF, - type.getTypeEnum())) - .setChecked(true); - }; - }; - - public static final TypeService GORILLA_V1_ENCODER_SERVICE = - type -> - switch (type.getTypeEnum()) { - case FLOAT -> new SinglePrecisionEncoderV1(); - case DOUBLE -> new DoublePrecisionEncoderV1(); - case BOOLEAN, - INT32, - DATE, - INT64, - TIMESTAMP, - TEXT, - BLOB, - STRING, - OBJECT, - ROW, - UNKNOWN, - VECTOR -> - throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.GORILLA_V1, - type.getTypeEnum())) - .setChecked(true); - }; - - public static final TypeService REGULAR_ENCODER_SERVICE = - type -> - switch (type.getTypeEnum()) { - case INT32, DATE -> new RegularDataEncoder.IntRegularEncoder(); - case INT64, TIMESTAMP -> new RegularDataEncoder.LongRegularEncoder(); - case BOOLEAN, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> - throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.REGULAR, - type.getTypeEnum())) - .setChecked(true); - }; - - public static final TypeService GORILLA_V2_ENCODER_SERVICE = - type -> - switch (type.getTypeEnum()) { - case FLOAT -> new SinglePrecisionEncoderV2(); - case DOUBLE -> new DoublePrecisionEncoderV2(); - case INT32, DATE -> new IntGorillaEncoder(); - case INT64, TIMESTAMP -> new LongGorillaEncoder(); - case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> - throw new UnSupportedDataTypeException( - Messages.format( - "error.encoding.ts_encoding_builder_unsupported_type", - TSEncoding.GORILLA, - type.getTypeEnum())) - .setChecked(true); - }; - private TypeServices() {} @FunctionalInterface @@ -608,12 +496,6 @@ public interface EmptyValueChunkWriter { void write(ValueChunkWriter writer); } - @FunctionalInterface - public interface EncoderBuilder { - - Encoder build(int maxPointNumber); - } - public enum PageDataReadStatus { CONTINUE, FILTERED, From 4c80fa62c38d22305dccca3f30fe73c439b0d76e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 14:53:42 +0800 Subject: [PATCH 100/131] refactor StatisticsTest --- .../metadata/statistics/StatisticsTest.java | 107 +++++++++++------- 1 file changed, 63 insertions(+), 44 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java index ba12c6358..7b202894b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java @@ -20,6 +20,8 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.junit.Test; @@ -34,6 +36,58 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public class StatisticsTest { + private static final TypeService CHECK_STATISTICS_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, FLOAT, DOUBLE, TIMESTAMP, DATE -> + (statistics, min, max, sum) -> { + assertEquals(min, ((Number) statistics.getMinValue()).intValue()); + assertEquals(max, ((Number) statistics.getMaxValue()).intValue()); + assertEquals(min, ((Number) statistics.getFirstValue()).intValue()); + assertEquals(max, ((Number) statistics.getLastValue()).intValue()); + assertEquals(sum, statistics.getSumDoubleValue(), 0.001); + }; + case BOOLEAN -> + (statistics, min, max, sum) -> { + assertEquals(min % 2 == 1, statistics.getFirstValue()); + assertEquals(max % 2 == 1, statistics.getLastValue()); + assertEquals(sum, statistics.getSumDoubleValue(), 0.001); + }; + case TEXT -> + (statistics, min, max, sum) -> { + assertEquals( + new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), + statistics.getFirstValue()); + assertEquals( + new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), + statistics.getLastValue()); + }; + case STRING -> + (statistics, min, max, sum) -> { + assertEquals( + new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), + statistics.getMinValue()); + assertEquals( + new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), + statistics.getMaxValue()); + assertEquals( + new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), + statistics.getFirstValue()); + assertEquals( + new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), + statistics.getLastValue()); + }; + case BLOB, OBJECT -> (statistics, min, max, sum) -> {}; + case ROW, UNKNOWN, VECTOR -> + (statistics, min, max, sum) -> { + throw new IllegalArgumentException(type.getTypeEnum().toString()); + }; + }; + + static { + CHECK_STATISTICS_SERVICE.check(); + } + @Test public void testCrossTypeMerge() { Set dataTypes = new HashSet<>(); @@ -69,50 +123,9 @@ public void testCrossTypeMerge() { private static void checkStatistics(Statistics statistics, int min, int max, double sum) { assertEquals(min, statistics.getStartTime()); assertEquals(max, statistics.getEndTime()); - switch (statistics.getType()) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case TIMESTAMP: - case DATE: - assertEquals(min, ((Number) statistics.getMinValue()).intValue()); - assertEquals(max, ((Number) statistics.getMaxValue()).intValue()); - assertEquals(min, ((Number) statistics.getFirstValue()).intValue()); - assertEquals(max, ((Number) statistics.getLastValue()).intValue()); - assertEquals(sum, statistics.getSumDoubleValue(), 0.001); - break; - case BOOLEAN: - assertEquals(min % 2 == 1, statistics.getFirstValue()); - assertEquals(max % 2 == 1, statistics.getLastValue()); - assertEquals(sum, statistics.getSumDoubleValue(), 0.001); - break; - case TEXT: - assertEquals( - new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), - statistics.getFirstValue()); - assertEquals( - new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), - statistics.getLastValue()); - break; - case STRING: - assertEquals( - new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), statistics.getMinValue()); - assertEquals( - new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), statistics.getMaxValue()); - assertEquals( - new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), - statistics.getFirstValue()); - assertEquals( - new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), - statistics.getLastValue()); - break; - case BLOB: - case OBJECT: - break; - default: - throw new IllegalArgumentException(statistics.getType().toString()); - } + CHECK_STATISTICS_SERVICE + .call(Type.fromTsDataType(statistics.getType())) + .check(statistics, min, max, sum); } private static Statistics genStatistics(TSDataType dataType, int val) { @@ -185,4 +198,10 @@ private static Statistics genStatistics(TSDataType dataType, int val) { result.setEmpty(false); return result; } + + @FunctionalInterface + private interface StatisticsChecker { + + void check(Statistics statistics, int min, int max, double sum); + } } From c91071a4cb0eed903cbf62c5ab6cac547c018e23 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 16:15:26 +0800 Subject: [PATCH 101/131] refactor StatisticsTest --- .../metadata/statistics/StatisticsTest.java | 145 ++++++++++-------- 1 file changed, 81 insertions(+), 64 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java index 7b202894b..47ad21a2d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java @@ -84,8 +84,81 @@ public class StatisticsTest { }; }; + private static final TypeService GENERATE_STATISTICS_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + value -> { + IntegerStatistics statistics = new IntegerStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case INT64 -> + value -> { + LongStatistics statistics = new LongStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case FLOAT -> + value -> { + FloatStatistics statistics = new FloatStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case DOUBLE -> + value -> { + DoubleStatistics statistics = new DoubleStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case TEXT -> + value -> { + BinaryStatistics statistics = new BinaryStatistics(); + statistics.initializeStats( + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + return statistics; + }; + case STRING -> + value -> { + StringStatistics statistics = new StringStatistics(); + statistics.initializeStats( + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + return statistics; + }; + case BOOLEAN -> + value -> { + BooleanStatistics statistics = new BooleanStatistics(); + boolean booleanValue = value % 2 == 1; + statistics.initializeStats(booleanValue, booleanValue, booleanValue ? 1 : 0); + return statistics; + }; + case BLOB -> value -> new BlobStatistics(); + case OBJECT -> value -> new ObjectStatistics(); + case DATE -> + value -> { + DateStatistics statistics = new DateStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case TIMESTAMP -> + value -> { + TimestampStatistics statistics = new TimestampStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case ROW, UNKNOWN, VECTOR -> + value -> { + throw new IllegalArgumentException(type.getTypeEnum().toString()); + }; + }; + static { CHECK_STATISTICS_SERVICE.check(); + GENERATE_STATISTICS_SERVICE.check(); } @Test @@ -129,70 +202,8 @@ private static void checkStatistics(Statistics statistics, int min, int max, dou } private static Statistics genStatistics(TSDataType dataType, int val) { - Statistics result; - switch (dataType) { - case INT32: - IntegerStatistics intStat = new IntegerStatistics(); - intStat.initializeStats(val, val, val, val, val); - result = intStat; - break; - case INT64: - LongStatistics longStat = new LongStatistics(); - longStat.initializeStats(val, val, val, val, val); - result = longStat; - break; - case FLOAT: - FloatStatistics floatStat = new FloatStatistics(); - floatStat.initializeStats(val, val, val, val, val); - result = floatStat; - break; - case DOUBLE: - DoubleStatistics doubleStat = new DoubleStatistics(); - doubleStat.initializeStats(val, val, val, val, val); - result = doubleStat; - break; - case TEXT: - BinaryStatistics binaryStat = new BinaryStatistics(); - binaryStat.initializeStats( - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET)); - result = binaryStat; - break; - case STRING: - StringStatistics stringStat = new StringStatistics(); - stringStat.initializeStats( - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET)); - result = stringStat; - break; - case BOOLEAN: - BooleanStatistics boolStat = new BooleanStatistics(); - boolStat.initializeStats(val % 2 == 1, val % 2 == 1, val % 2 == 1 ? 1 : 0); - result = boolStat; - break; - case BLOB: - BlobStatistics blobStat = new BlobStatistics(); - result = blobStat; - break; - case OBJECT: - ObjectStatistics objectStat = new ObjectStatistics(); - result = objectStat; - break; - case DATE: - DateStatistics dateStat = new DateStatistics(); - dateStat.initializeStats(val, val, val, val, val); - result = dateStat; - break; - case TIMESTAMP: - TimestampStatistics timestampStat = new TimestampStatistics(); - timestampStat.initializeStats(val, val, val, val, val); - result = timestampStat; - break; - default: - throw new IllegalArgumentException(dataType.toString()); - } + Statistics result = + GENERATE_STATISTICS_SERVICE.call(Type.fromTsDataType(dataType)).generate(val); result.setStartTime(val); result.setEndTime(val); result.setEmpty(false); @@ -204,4 +215,10 @@ private interface StatisticsChecker { void check(Statistics statistics, int min, int max, double sum); } + + @FunctionalInterface + private interface StatisticsGenerator { + + Statistics generate(int value); + } } From 25d94149708c3cbd2b7b953411a1dde4a5c1083d Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 16:26:45 +0800 Subject: [PATCH 102/131] add write --- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 6 +++ .../tsfile/read/common/type/VectorType.java | 6 +++ .../tsfile/read/common/type/TypeTest.java | 46 +++++++++++++++++++ 11 files changed, 98 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 5a2c07665..f5f427615 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -273,6 +273,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((int[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getInt()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (int) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index d69c02b3a..eeeea0cac 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -272,6 +272,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((long[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getLong()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (long) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 8fc9c959d..a633c4771 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -247,6 +247,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((Binary[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBinary()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (Binary) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index f7827be83..4fbe5b95f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -255,6 +255,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((Binary[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBinary()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (Binary) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 5f3677fba..1e2d9763d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -264,6 +264,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((boolean[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBoolean()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (boolean) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 50d2a47ca..e7a8887c1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -275,6 +275,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((double[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getDouble()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (double) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 687408a7d..040b37b06 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -273,6 +273,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((float[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getFloat()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (float) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 916c2d42e..1ae37bff6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -249,6 +249,11 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex writer.write(time, ((Binary[]) column)[rowIndex]); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBinary()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (Binary) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index f2411c27b..b5faf76d6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -400,6 +400,12 @@ default void write(ChunkWriterImpl writer, long time, Object column, int rowInde Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes a TsPrimitiveType value to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Writes a value to a non-aligned chunk. */ default void write(ChunkWriterImpl writer, long time, Object value) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index dccccebde..f79bde536 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -124,6 +124,12 @@ public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index fe843f3a6..a9ac1ac58 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -801,6 +801,52 @@ public void testWriteNonAlignedValueChunk() { } } + @Test + public void testWriteTsPrimitiveTypeToNonAlignedValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1L, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, 2L, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, 3L, new TsPrimitiveType.TsInt(20260717, TSDataType.DATE)}, + {TSDataType.INT64, 4L, new TsPrimitiveType.TsLong(2L)}, + {TSDataType.TIMESTAMP, 5L, new TsPrimitiveType.TsLong(3L)}, + {TSDataType.FLOAT, 6L, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, 7L, new TsPrimitiveType.TsDouble(2.5D)} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]) + .write(writer, (long) testCase[1], (TsPrimitiveType) testCase[2]); + } + + Mockito.verify(writer).write(1L, true); + Mockito.verify(writer).write(2L, 1); + Mockito.verify(writer).write(3L, 20260717); + Mockito.verify(writer).write(4L, 2L); + Mockito.verify(writer).write(5L, 3L); + Mockito.verify(writer).write(6L, 1.25F); + Mockito.verify(writer).write(7L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType binaryValue = new TsPrimitiveType.TsBinary(binary); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 8L, binaryValue); + Mockito.verify(writer).write(8L, binary); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(writer, 9L, new TsPrimitiveType.TsLong(1L)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + @Test public void testToString() { Object[][] testCases = { From 6ff37f949d9090304bfd0deaceff8e7c9a50132f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 16:33:16 +0800 Subject: [PATCH 103/131] refactor BatchDataTest --- .../tsfile/read/common/BatchDataTest.java | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java index 055109f4d..ea976100f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java @@ -41,6 +41,36 @@ public class BatchDataTest { + private static final TypeService ASSERT_SERIALIZED_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (inputStream, expectedValue) -> + assertEquals(expectedValue, inputStream.readBoolean()); + case INT32, DATE -> + (inputStream, expectedValue) -> assertEquals(expectedValue, inputStream.readInt()); + case INT64, TIMESTAMP -> + (inputStream, expectedValue) -> assertEquals(expectedValue, inputStream.readLong()); + case FLOAT -> + (inputStream, expectedValue) -> + assertEquals((float) expectedValue, inputStream.readFloat(), 0); + case DOUBLE -> + (inputStream, expectedValue) -> + assertEquals((double) expectedValue, inputStream.readDouble(), 0); + case TEXT, STRING, BLOB, OBJECT -> + (inputStream, expectedValue) -> { + Binary binary = (Binary) expectedValue; + assertArrayEquals( + binary.getValues(), inputStream.readNBytes(inputStream.readInt())); + }; + case ROW, UNKNOWN, VECTOR -> + (inputStream, expectedValue) -> fail("Unexpected data type: " + type.getTypeEnum()); + }; + + static { + ASSERT_SERIALIZED_VALUE_SERVICE.check(); + } + @Test public void testInt() { BatchData batchData = new BatchData(TSDataType.INT32); @@ -163,33 +193,14 @@ private DataInputStream serialize(BatchData batchData) throws IOException { private void assertSerializedValue( DataInputStream inputStream, TSDataType dataType, Object expectedValue) throws IOException { - switch (dataType) { - case BOOLEAN: - assertEquals(expectedValue, inputStream.readBoolean()); - break; - case INT32: - case DATE: - assertEquals(expectedValue, inputStream.readInt()); - break; - case INT64: - case TIMESTAMP: - assertEquals(expectedValue, inputStream.readLong()); - break; - case FLOAT: - assertEquals((float) expectedValue, inputStream.readFloat(), 0); - break; - case DOUBLE: - assertEquals((double) expectedValue, inputStream.readDouble(), 0); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary binary = (Binary) expectedValue; - assertArrayEquals(binary.getValues(), inputStream.readNBytes(inputStream.readInt())); - break; - default: - fail("Unexpected data type: " + dataType); - } + ASSERT_SERIALIZED_VALUE_SERVICE + .call(Type.fromTsDataType(dataType)) + .assertValue(inputStream, expectedValue); + } + + @FunctionalInterface + private interface SerializedValueAsserter { + + void assertValue(DataInputStream inputStream, Object expectedValue) throws IOException; } } From bc87a506e27b137b57a73da804687c75e5edd81a Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 16:33:22 +0800 Subject: [PATCH 104/131] refactor BatchDataTest --- .../test/java/org/apache/tsfile/read/common/BatchDataTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java index ea976100f..fde2d0b9c 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java @@ -21,6 +21,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; From c1dd57b2e896de75d8e4b86c03ae6126463770d8 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 16:36:32 +0800 Subject: [PATCH 105/131] refactor TsBlockFilterTest --- .../tsfile/read/filter/TsBlockFilterTest.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java index a4b7f3c8d..a6c6e0599 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.IntTypeService; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.ValueFilterApi; import org.apache.tsfile.utils.Binary; @@ -41,6 +43,21 @@ public class TsBlockFilterTest { private static final int COL_NUM = 3; private static final int ROW_NUM = 7; + private static final IntTypeService GET_MEASUREMENT_ID_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> 0; + case INT32 -> 1; + case INT64 -> 2; + case FLOAT -> 3; + case DOUBLE -> 4; + case TEXT -> 5; + case BLOB, DATE, OBJECT, ROW, STRING, TIMESTAMP, UNKNOWN, VECTOR -> -1; + }; + + static { + GET_MEASUREMENT_ID_SERVICE.check(); + } private TsBlock tsBlock; @@ -49,22 +66,7 @@ public class TsBlockFilterTest { private boolean[] selection = new boolean[COL_NUM]; public int getMeasurementId(TSDataType type) { - switch (type) { - case BOOLEAN: - return 0; - case INT32: - return 1; - case INT64: - return 2; - case FLOAT: - return 3; - case DOUBLE: - return 4; - case TEXT: - return 5; - default: - return -1; - } + return GET_MEASUREMENT_ID_SERVICE.call(Type.fromTsDataType(type)); } @Before From b1aacf499b3618046c58fe778c50743f9d828b8e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 16:51:28 +0800 Subject: [PATCH 106/131] refactor PageReaderTest --- .../tsfile/read/reader/PageReaderTest.java | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java index 7b0258f2d..7c9a38bd8 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java @@ -36,6 +36,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.reader.page.PageReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.page.PageWriter; @@ -51,6 +53,22 @@ public class PageReaderTest { private static final int POINTS_COUNT_IN_ONE_PAGE = 1000000; + private static final TypeService WRITE_PAGE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (writer, time, value) -> writer.write(time, (Boolean) value); + case INT32 -> (writer, time, value) -> writer.write(time, (Integer) value); + case INT64 -> (writer, time, value) -> writer.write(time, (Long) value); + case FLOAT -> (writer, time, value) -> writer.write(time, (Float) value); + case DOUBLE -> (writer, time, value) -> writer.write(time, (Double) value); + case TEXT -> (writer, time, value) -> writer.write(time, (Binary) value); + case BLOB, DATE, OBJECT, ROW, STRING, TIMESTAMP, UNKNOWN, VECTOR -> + (writer, time, value) -> {}; + }; + + static { + WRITE_PAGE_VALUE_SERVICE.check(); + } @Test public void testLong() { @@ -272,33 +290,21 @@ public void testDelete(TSDataType dataType) { } private void writeData() { + PageValueWriter valueWriter = WRITE_PAGE_VALUE_SERVICE.call(Type.fromTsDataType(dataType)); for (int i = 0; i < count; i++) { - switch (dataType) { - case BOOLEAN: - pageWriter.write(i, (Boolean) generateValueByIndex(i)); - break; - case INT32: - pageWriter.write(i, (Integer) generateValueByIndex(i)); - break; - case INT64: - pageWriter.write(i, (Long) generateValueByIndex(i)); - break; - case FLOAT: - pageWriter.write(i, (Float) generateValueByIndex(i)); - break; - case DOUBLE: - pageWriter.write(i, (Double) generateValueByIndex(i)); - break; - case TEXT: - pageWriter.write(i, (Binary) generateValueByIndex(i)); - break; - } + valueWriter.write(pageWriter, i, generateValueByIndex(i)); } } public abstract Object generateValueByIndex(int i); } + @FunctionalInterface + private interface PageValueWriter { + + void write(PageWriter writer, long time, Object value); + } + @Test public void testPageDelete() { LoopWriteReadTest test = From b8a355f16d278ab31b0a7897d5b58e57e4ba7499 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 17 Jul 2026 17:58:36 +0800 Subject: [PATCH 107/131] refactor TsFileLastReaderTest --- .../read/reader/TsFileLastReaderTest.java | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java index d8f25ea97..efa9871b4 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java @@ -25,6 +25,7 @@ import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; @@ -62,6 +63,34 @@ public class TsFileLastReaderTest { Arrays.asList(TSDataType.INT64, TSDataType.BLOB); private static final Map typeAddValueFunctions = new HashMap<>(); + private static final TypeService ADD_OFFSET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT64 -> + (tablet, row, column, offset) -> tablet.addValue(row, column, (long) row + offset); + case BLOB -> + (tablet, row, column, offset) -> + tablet.addValue( + row, + column, + Long.toBinaryString(row + offset).getBytes(StandardCharsets.UTF_8)); + case BOOLEAN, + DATE, + DOUBLE, + FLOAT, + INT32, + OBJECT, + ROW, + STRING, + TEXT, + TIMESTAMP, + UNKNOWN, + VECTOR -> + (tablet, row, column, offset) -> { + throw new IllegalArgumentException( + "Unsupported TSDataType " + type.getTypeEnum()); + }; + }; static { typeAddValueFunctions.put( @@ -71,6 +100,7 @@ public class TsFileLastReaderTest { ((tablet, row, column) -> tablet.addValue( row, column, Long.toBinaryString(row).getBytes(StandardCharsets.UTF_8)))); + ADD_OFFSET_VALUE_SERVICE.check(); } private final String filePath = "target/test.tsfile"; @@ -143,20 +173,10 @@ private void createFileWithLastEmptyChunks(int deviceNum, int measurementNum, in } for (int j = 0; j < measurementNum / 2; j++) { TSDataType tsDataType = dataTypes.get(j % dataTypes.size()); + OffsetTabletAddValueFunction addValueFunction = + ADD_OFFSET_VALUE_SERVICE.call(Type.fromTsDataType(tsDataType)); for (int k = 0; k < seriesPointNum; k++) { - switch (tsDataType) { - case INT64: - tablet.addValue(k, j, (long) k + seriesPointNum / 2); - break; - case BLOB: - tablet.addValue( - k, - j, - Long.toBinaryString(k + seriesPointNum / 2).getBytes(StandardCharsets.UTF_8)); - break; - default: - throw new IllegalArgumentException("Unsupported TSDataType " + tsDataType); - } + addValueFunction.addValue(tablet, k, j, seriesPointNum / 2); } } writer.writeTree(tablet); @@ -164,6 +184,12 @@ private void createFileWithLastEmptyChunks(int deviceNum, int measurementNum, in } } + @FunctionalInterface + private interface OffsetTabletAddValueFunction { + + void addValue(Tablet tablet, int row, int column, int offset); + } + private void createAlignedFileWithObject(int seriesPointNum) throws IOException, WriteProcessException { try (TsFileWriter writer = new TsFileWriter(file)) { From 366468413a90bea3be456ba8ab6aa3edcc907c00 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 10:31:27 +0800 Subject: [PATCH 108/131] refactor TableViewTest --- .../tsfile/tableview/TableViewTest.java | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java index b55473681..434fd50cf 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java @@ -32,6 +32,8 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.tsfile.read.expression.QueryExpression; @@ -68,6 +70,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.IntFunction; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -81,6 +84,29 @@ public class TableViewTest { private static final int measurementSchemaNum = 5; private static TableSchema testTableSchema; private static int numTimestampPerDevice = 10; + private static final TypeService> GET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT64 -> i -> (long) i; + case TEXT -> i -> new Binary(String.valueOf(i), StandardCharsets.UTF_8); + case BLOB, + BOOLEAN, + DATE, + DOUBLE, + FLOAT, + INT32, + OBJECT, + ROW, + STRING, + TIMESTAMP, + UNKNOWN, + VECTOR -> + i -> i; + }; + + static { + GET_VALUE_SERVICE.check(); + } @Before public void setUp() throws Exception { @@ -505,14 +531,7 @@ public static Tablet genTablet(TableSchema tableSchema, int offset, int deviceNu } public static Object getValue(TSDataType dataType, int i) { - switch (dataType) { - case INT64: - return (long) i; - case TEXT: - return new Binary(String.valueOf(i), StandardCharsets.UTF_8); - default: - return i; - } + return GET_VALUE_SERVICE.call(Type.fromTsDataType(dataType)).apply(i); } public static TableSchema genTableSchema(int tableNum) { From e750e7befcb3743f8c053bf1f95b4de75f10d89a Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 11:37:58 +0800 Subject: [PATCH 109/131] refactor TabletTest --- .../tsfile/write/record/TabletTest.java | 210 ++++++++++-------- 1 file changed, 121 insertions(+), 89 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java index ab4bf377b..660fe8534 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java @@ -24,6 +24,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; @@ -59,6 +61,98 @@ public class TabletTest { + private static final TypeService SAMPLE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (row, col, variableBinaryLength) -> (row + col) % 2 == 0; + case INT32 -> (row, col, variableBinaryLength) -> row + col * 100; + case INT64, TIMESTAMP -> + (row, col, variableBinaryLength) -> (long) (valueOffset(row, col) * 1_000_000L); + case FLOAT -> (row, col, variableBinaryLength) -> (row + col) * 1.5f; + case DOUBLE -> (row, col, variableBinaryLength) -> (row + col) * 2.5; + case TEXT, STRING -> + (row, col, variableBinaryLength) -> + stringOfLength(binaryPayloadLength(variableBinaryLength, row, col)); + case BLOB -> + (row, col, variableBinaryLength) -> + binaryOfLength(binaryPayloadLength(variableBinaryLength, row, col)); + case DATE -> + (row, col, variableBinaryLength) -> + LocalDate.of(2000 + (row % 20), (col % 12) + 1, (row % 28) + 1); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (row, col, variableBinaryLength) -> { + throw new IllegalArgumentException( + "Unsupported type in test: " + type.getTypeEnum()); + }; + }; + + private static final TypeService WRITE_TABLET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (tablet, row, col, value) -> tablet.addValue(row, col, value); + case INT64, TIMESTAMP -> + (tablet, row, col, value) -> tablet.addValue(row, col, (long) value); + case FLOAT -> (tablet, row, col, value) -> tablet.addValue(row, col, value * 1.0f); + case DOUBLE -> (tablet, row, col, value) -> tablet.addValue(row, col, value * 1.0); + case BOOLEAN -> (tablet, row, col, value) -> tablet.addValue(row, col, value % 2 == 0); + case BLOB, STRING, TEXT -> + (tablet, row, col, value) -> tablet.addValue(row, col, String.valueOf(value)); + case OBJECT -> + (tablet, row, col, value) -> + tablet.addValue(row, col, value % 2 == 0, value, new byte[] {(byte) row}); + case DATE -> + (tablet, row, col, value) -> tablet.addValue(row, col, LocalDate.of(value, 1, 1)); + case ROW, UNKNOWN, VECTOR -> + (tablet, row, col, value) -> { + throw new IllegalArgumentException( + "Unsupported type in test: " + type.getTypeEnum()); + }; + }; + + private static final TypeService CHECK_TABLET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (tablet, row, col) -> assertEquals(row, tablet.getValue(row, col)); + case INT64, TIMESTAMP -> + (tablet, row, col) -> assertEquals((long) row, tablet.getValue(row, col)); + case FLOAT -> + (tablet, row, col) -> + assertEquals(row * 1.0f, (float) tablet.getValue(row, col), 0.0001f); + case DOUBLE -> + (tablet, row, col) -> + assertEquals(row * 1.0, (double) tablet.getValue(row, col), 0.0001); + case BOOLEAN -> + (tablet, row, col) -> assertEquals(row % 2 == 0, tablet.getValue(row, col)); + case BLOB, STRING, TEXT -> + (tablet, row, col) -> + assertEquals( + new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8)), + tablet.getValue(row, col)); + case OBJECT -> + (tablet, row, col) -> { + byte[] content = new byte[] {(byte) row}; + byte[] expected = new byte[content.length + 9]; + expected[0] = (byte) (row % 2); + System.arraycopy(BytesUtils.longToBytes(row), 0, expected, 1, 8); + System.arraycopy(content, 0, expected, 9, content.length); + assertEquals(new Binary(expected), tablet.getValue(row, col)); + }; + case DATE -> + (tablet, row, col) -> + assertEquals(LocalDate.of(row, 1, 1), tablet.getValue(row, col)); + case ROW, UNKNOWN, VECTOR -> + (tablet, row, col) -> { + throw new IllegalArgumentException( + "Unsupported type in test: " + type.getTypeEnum()); + }; + }; + + static { + SAMPLE_VALUE_SERVICE.check(); + WRITE_TABLET_VALUE_SERVICE.check(); + CHECK_TABLET_VALUE_SERVICE.check(); + } + @Test public void testAddValue() { Tablet tablet = @@ -694,28 +788,9 @@ private static int binaryPayloadLength(boolean variableBinaryLength, int row, in } private Object sampleValue(TSDataType type, int row, int col, boolean variableBinaryLength) { - switch (type) { - case BOOLEAN: - return (row + col) % 2 == 0; - case INT32: - return row + col * 100; - case INT64: - case TIMESTAMP: - return (long) (valueOffset(row, col) * 1_000_000L); - case FLOAT: - return (row + col) * 1.5f; - case DOUBLE: - return (row + col) * 2.5; - case TEXT: - case STRING: - return stringOfLength(binaryPayloadLength(variableBinaryLength, row, col)); - case BLOB: - return binaryOfLength(binaryPayloadLength(variableBinaryLength, row, col)); - case DATE: - return LocalDate.of(2000 + (row % 20), (col % 12) + 1, (row % 28) + 1); - default: - throw new IllegalArgumentException("Unsupported type in test: " + type); - } + return SAMPLE_VALUE_SERVICE + .call(Type.fromTsDataType(type)) + .generate(row, col, variableBinaryLength); } private static int valueOffset(int row, int col) { @@ -799,35 +874,9 @@ private void fillTablet(Tablet t, int valueOffset, int length) { for (int i = 0; i < length; i++) { t.addTimestamp(i, i + valueOffset); for (int j = 0; j < t.getSchemas().size(); j++) { - switch (t.getSchemas().get(j).getType()) { - case INT32: - t.addValue(i, j, i + valueOffset); - break; - case TIMESTAMP: - case INT64: - t.addValue(i, j, (long) (i + valueOffset)); - break; - case FLOAT: - t.addValue(i, j, (i + valueOffset) * 1.0f); - break; - case DOUBLE: - t.addValue(i, j, (i + valueOffset) * 1.0); - break; - case BOOLEAN: - t.addValue(i, j, (i + valueOffset) % 2 == 0); - break; - case TEXT: - case STRING: - case BLOB: - t.addValue(i, j, String.valueOf(i + valueOffset)); - break; - case OBJECT: - t.addValue(i, j, (i + valueOffset) % 2 == 0, i + valueOffset, new byte[] {(byte) i}); - break; - case DATE: - t.addValue(i, j, LocalDate.of(i + valueOffset, 1, 1)); - break; - } + WRITE_TABLET_VALUE_SERVICE + .call(Type.fromTsDataType(t.getSchemas().get(j).getType())) + .write(t, i, j, i + valueOffset); } } } @@ -1030,45 +1079,28 @@ private void checkAppendedTablet( } assertFalse(result.isNull(i, j)); - switch (result.getSchemas().get(j).getType()) { - case INT32: - assertEquals(i, result.getValue(i, j)); - break; - case TIMESTAMP: - case INT64: - assertEquals((long) i, result.getValue(i, j)); - break; - case FLOAT: - assertEquals(i * 1.0f, (float) result.getValue(i, j), 0.0001f); - break; - case DOUBLE: - assertEquals(i * 1.0, (double) result.getValue(i, j), 0.0001); - break; - case BOOLEAN: - assertEquals(i % 2 == 0, result.getValue(i, j)); - break; - case TEXT: - case BLOB: - case STRING: - assertEquals( - new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)), - result.getValue(i, j)); - break; - case OBJECT: - { - byte[] content = new byte[] {(byte) i}; - byte[] expected = new byte[content.length + 9]; - expected[0] = (byte) (i % 2); - System.arraycopy(BytesUtils.longToBytes(i), 0, expected, 1, 8); - System.arraycopy(content, 0, expected, 9, content.length); - assertEquals(new Binary(expected), result.getValue(i, j)); - } - break; - case DATE: - assertEquals(LocalDate.of(i, 1, 1), result.getValue(i, j)); - break; - } + CHECK_TABLET_VALUE_SERVICE + .call(Type.fromTsDataType(result.getSchemas().get(j).getType())) + .check(result, i, j); } } } + + @FunctionalInterface + private interface SampleValueGenerator { + + Object generate(int row, int col, boolean variableBinaryLength); + } + + @FunctionalInterface + private interface TabletValueWriter { + + void write(Tablet tablet, int row, int col, int value); + } + + @FunctionalInterface + private interface TabletValueChecker { + + void check(Tablet tablet, int row, int col); + } } From 7e8d5fe15d102878ac55c12bd7aa4d2fd98c4008 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 11:43:23 +0800 Subject: [PATCH 110/131] refactor TypeCastTest --- .../org/apache/tsfile/utils/TypeCastTest.java | 298 +++++++++--------- 1 file changed, 156 insertions(+), 142 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java index 6398eac75..68384314d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java @@ -20,6 +20,8 @@ package org.apache.tsfile.utils; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.junit.Test; @@ -28,6 +30,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.function.IntFunction; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -36,6 +39,129 @@ public class TypeCastTest { + private static final TypeService CHECK_ARRAY_CAST_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (from, to, array) -> + assertArrayEquals( + (int[]) genValueArray(to), (int[]) to.castFromArray(from, array)); + case INT64, TIMESTAMP -> + (from, to, array) -> + assertArrayEquals( + (long[]) genValueArray(to), (long[]) to.castFromArray(from, array)); + case BOOLEAN -> + (from, to, array) -> + assertArrayEquals( + (boolean[]) genValueArray(to), (boolean[]) to.castFromArray(from, array)); + case BLOB, STRING, TEXT -> + (from, to, array) -> + assertArrayEquals( + TypeCastTest.EXPECTED_BINARY_ARRAY_SERVICE + .call(Type.fromTsDataType(from)) + .generate(array), + (Binary[]) to.castFromArray(from, array)); + case FLOAT -> + (from, to, array) -> + assertArrayEquals( + (float[]) genValueArray(to), (float[]) to.castFromArray(from, array), 0.1f); + case DOUBLE -> + (from, to, array) -> + assertArrayEquals( + (double[]) genValueArray(to), + (double[]) to.castFromArray(from, array), + 0.1); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (from, to, array) -> fail("Unexpected type: " + to); + }; + + private static final TypeService EXPECTED_BINARY_ARRAY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + array -> { + int[] values = (int[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case DATE -> + array -> { + int[] values = (int[]) array; + return toBinaryArray( + values.length, i -> TSDataType.getDateStringValue(values[i])); + }; + case INT64, TIMESTAMP -> + array -> { + long[] values = (long[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case FLOAT -> + array -> { + float[] values = (float[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case DOUBLE -> + array -> { + double[] values = (double[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case BOOLEAN -> + array -> { + boolean[] values = (boolean[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case BLOB, STRING, TEXT -> array -> (Binary[]) array; + case OBJECT, ROW, UNKNOWN, VECTOR -> + array -> { + throw new IllegalArgumentException( + "Unsupported data type: " + type.getTypeEnum()); + }; + }; + + private static final TypeService GENERATE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> () -> 1; + case INT64, TIMESTAMP -> () -> 1L; + case BOOLEAN -> () -> false; + case FLOAT -> () -> 1.0f; + case DOUBLE -> () -> 1.0; + case BLOB, OBJECT, STRING, TEXT -> () -> new Binary("1", StandardCharsets.UTF_8); + case ROW, UNKNOWN, VECTOR -> + () -> { + throw new IllegalArgumentException( + "Unsupported data type: " + type.getTypeEnum()); + }; + }; + + private static final TypeService GENERATE_VALUE_ARRAY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> () -> new int[] {1, 2, 3}; + case INT64, TIMESTAMP -> () -> new long[] {1, 2, 3}; + case BOOLEAN -> () -> new boolean[] {true, false}; + case FLOAT -> () -> new float[] {1.0f, 2.0f, 3.0f}; + case DOUBLE -> () -> new double[] {1.0, 2.0, 3.0}; + case BLOB, OBJECT, STRING, TEXT -> + () -> + new Binary[] { + new Binary("1", StandardCharsets.UTF_8), + new Binary("2", StandardCharsets.UTF_8), + new Binary("3", StandardCharsets.UTF_8) + }; + case ROW, UNKNOWN, VECTOR -> + () -> { + throw new IllegalArgumentException( + "Unsupported data type: " + type.getTypeEnum()); + }; + }; + + static { + CHECK_ARRAY_CAST_SERVICE.check(); + EXPECTED_BINARY_ARRAY_SERVICE.check(); + GENERATE_VALUE_SERVICE.check(); + GENERATE_VALUE_ARRAY_SERVICE.check(); + } + @Test public void testSingleCast() { Set dataTypes = new HashSet<>(); @@ -85,154 +211,42 @@ public void testArrayCast() { assertThrows(ClassCastException.class, () -> to.castFromArray(from, array)); continue; } - switch (to) { - case INT32: - case DATE: - assertArrayEquals((int[]) genValueArray(to), (int[]) to.castFromArray(from, array)); - break; - case INT64: - case TIMESTAMP: - assertArrayEquals((long[]) genValueArray(to), (long[]) to.castFromArray(from, array)); - break; - case BOOLEAN: - assertArrayEquals( - (boolean[]) genValueArray(to), (boolean[]) to.castFromArray(from, array)); - break; - case STRING: - case BLOB: - case TEXT: - switch (from) { - case BLOB: - case STRING: - assertArrayEquals((Binary[]) array, (Binary[]) to.castFromArray(from, array)); - break; - case INT32: - int[] tmpInt = (int[]) array; - Binary[] intResult = new Binary[tmpInt.length]; - for (int i = 0; i < tmpInt.length; i++) { - intResult[i] = new Binary(String.valueOf(tmpInt[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(intResult, (Binary[]) to.castFromArray(from, array)); - break; - case DATE: - int[] tmpDate = (int[]) array; - Binary[] dateResult = new Binary[tmpDate.length]; - for (int i = 0; i < tmpDate.length; i++) { - dateResult[i] = - new Binary(TSDataType.getDateStringValue(tmpDate[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(dateResult, (Binary[]) to.castFromArray(from, array)); - break; - case INT64: - case TIMESTAMP: - long[] tmpLong = (long[]) array; - Binary[] longResult = new Binary[tmpLong.length]; - for (int i = 0; i < tmpLong.length; i++) { - longResult[i] = new Binary(String.valueOf(tmpLong[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(longResult, (Binary[]) to.castFromArray(from, array)); - break; - case FLOAT: - float[] tmpFloat = (float[]) array; - Binary[] floatResult = new Binary[tmpFloat.length]; - for (int i = 0; i < tmpFloat.length; i++) { - floatResult[i] = new Binary(String.valueOf(tmpFloat[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(floatResult, (Binary[]) to.castFromArray(from, array)); - break; - case DOUBLE: - double[] tmpDouble = (double[]) array; - Binary[] doubleResult = new Binary[tmpDouble.length]; - for (int i = 0; i < tmpDouble.length; i++) { - doubleResult[i] = - new Binary(String.valueOf(tmpDouble[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(doubleResult, (Binary[]) to.castFromArray(from, array)); - break; - case BOOLEAN: - boolean[] tmpBoolean = (boolean[]) array; - Binary[] booleanResult = new Binary[tmpBoolean.length]; - for (int i = 0; i < tmpBoolean.length; i++) { - booleanResult[i] = - new Binary(String.valueOf(tmpBoolean[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(booleanResult, (Binary[]) to.castFromArray(from, array)); - break; - default: - break; - } - break; - case FLOAT: - assertArrayEquals( - (float[]) genValueArray(to), (float[]) to.castFromArray(from, array), 0.1f); - break; - case DOUBLE: - assertArrayEquals( - (double[]) genValueArray(to), (double[]) to.castFromArray(from, array), 0.1); - break; - case UNKNOWN: - case VECTOR: - default: - fail("Unexpected type: " + to); - } + CHECK_ARRAY_CAST_SERVICE.call(Type.fromTsDataType(to)).check(from, to, array); } } } - private Object genValue(TSDataType dataType) { - int i = 1; - switch (dataType) { - case INT32: - case DATE: - return i; - case TIMESTAMP: - case INT64: - return (long) i; - case BOOLEAN: - return false; - case FLOAT: - return i * 1.0f; - case DOUBLE: - return i * 1.0; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - return new Binary(Integer.toString(i), StandardCharsets.UTF_8); - case UNKNOWN: - case VECTOR: - default: - throw new IllegalArgumentException("Unsupported data type: " + dataType); - } + private static Object genValue(TSDataType dataType) { + return GENERATE_VALUE_SERVICE.call(Type.fromTsDataType(dataType)).generate(); + } + + private static Object genValueArray(TSDataType dataType) { + return GENERATE_VALUE_ARRAY_SERVICE.call(Type.fromTsDataType(dataType)).generate(); } - private Object genValueArray(TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return new int[] {1, 2, 3}; - case TIMESTAMP: - case INT64: - return new long[] {1, 2, 3}; - case BOOLEAN: - return new boolean[] {true, false}; - case FLOAT: - return new float[] {1.0f, 2.0f, 3.0f}; - case DOUBLE: - return new double[] {1.0, 2.0, 3.0}; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - return new Binary[] { - new Binary(Integer.toString(1), StandardCharsets.UTF_8), - new Binary(Integer.toString(2), StandardCharsets.UTF_8), - new Binary(Integer.toString(3), StandardCharsets.UTF_8) - }; - case UNKNOWN: - case VECTOR: - default: - throw new IllegalArgumentException("Unsupported data type: " + dataType); + private static Binary[] toBinaryArray(int length, IntFunction valueProvider) { + Binary[] result = new Binary[length]; + for (int i = 0; i < length; i++) { + result[i] = new Binary(valueProvider.apply(i), StandardCharsets.UTF_8); } + return result; + } + + @FunctionalInterface + private interface ArrayCastChecker { + + void check(TSDataType from, TSDataType to, Object array); + } + + @FunctionalInterface + private interface BinaryArrayGenerator { + + Binary[] generate(Object array); + } + + @FunctionalInterface + private interface ValueGenerator { + + Object generate(); } } From 3fd8785edbad576d0ec525ff7b29b5322bc93579 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 11:51:24 +0800 Subject: [PATCH 111/131] refactor RecordUtils --- .../apache/tsfile/i18n/messages.properties | 12 +++++ .../apache/tsfile/i18n/messages_zh.properties | 12 +++++ .../org/apache/tsfile/utils/RecordUtils.java | 45 ++++--------------- 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties index 98909f7a6..59679f6fd 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties @@ -1320,6 +1320,18 @@ log.utils.bytes_int_to_bytes_error = tsfile-common BytesUtils: cannot convert an # BytesUtils.longToBytes — cannot convert long to byte array (3 args: srcNum, pos, width) log.utils.bytes_long_to_bytes_error = tsfile-common BytesUtils: cannot convert a long {} to a byte array, pos {}, width {} +# RecordUtils.parseSimpleTupleRecord — invalid timestamp +log.utils.record_timestamp_illegal = Given timestamp is illegal: {} + +# RecordUtils.parseSimpleTupleRecord — measurement type not found +log.utils.record_measurement_type_not_found = Type not found for measurement ID {}; skipping + +# RecordUtils.parseSimpleTupleRecord — failed to parse measurement +log.utils.record_parse_measurement_error = Failed to parse measurement; omitting it + +# RecordUtils.parseSimpleTupleRecord — unsupported data type +log.utils.record_unsupported_data_type = Unsupported data type: {} + # TsFileSketchTool — cannot load file (filename arg) error.utils.sketch_tool_cannot_load = Cannot load file %1$s because the file has crashed. diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties index a1a437cfb..c2406b634 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties @@ -1320,6 +1320,18 @@ log.utils.bytes_int_to_bytes_error = tsfile-common BytesUtils: 无法将 integer # BytesUtils.longToBytes — cannot convert long to byte array (3 args: srcNum, pos, width) log.utils.bytes_long_to_bytes_error = tsfile-common BytesUtils: 无法将 long {} 转换为 byte 数组，pos {}，width {} +# RecordUtils.parseSimpleTupleRecord — 时间戳非法 +log.utils.record_timestamp_illegal = 给定的时间戳非法：{} + +# RecordUtils.parseSimpleTupleRecord — 未找到测点类型 +log.utils.record_measurement_type_not_found = 未找到 measurement ID {} 的类型，跳过该测点 + +# RecordUtils.parseSimpleTupleRecord — 测点解析失败 +log.utils.record_parse_measurement_error = 解析测点时出错，已忽略 + +# RecordUtils.parseSimpleTupleRecord — 不支持的数据类型 +log.utils.record_unsupported_data_type = 不支持的数据类型：{} + # TsFileSketchTool — cannot load file (filename arg) error.utils.sketch_tool_cannot_load = 无法加载文件 %1$s，因为文件已损坏。 diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java index 7e623590b..f67f82a0b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java @@ -21,13 +21,10 @@ import org.apache.tsfile.common.constant.JsonFormatConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.TSRecord; -import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; -import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; -import org.apache.tsfile.write.record.datapoint.FloatDataPoint; -import org.apache.tsfile.write.record.datapoint.IntDataPoint; -import org.apache.tsfile.write.record.datapoint.LongDataPoint; -import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.Schema; @@ -56,7 +53,7 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { try { timestamp = Long.parseLong(items[1].trim()); } catch (NumberFormatException e) { - LOG.warn("given timestamp is illegal:{}", str); + LOG.warn(Messages.get("log.utils.record_timestamp_illegal"), str); // return a TSRecord without any data points return new TSRecord(deviceId, -1); } @@ -75,7 +72,7 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { ? null : measurementGroup.getMeasurementSchemaMap().get(measurementId); if (measurementSchema == null) { - LOG.warn("measurementId:{},type not found, pass", measurementId); + LOG.warn(Messages.get("log.utils.record_measurement_type_not_found"), measurementId); continue; } type = measurementSchema.getType(); @@ -84,35 +81,11 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { // TSRecord if (!"".equals(value)) { try { - switch (type) { - case INT32: - case DATE: - ret.addTuple(new IntDataPoint(measurementId, Integer.parseInt(value))); - break; - case INT64: - case TIMESTAMP: - ret.addTuple(new LongDataPoint(measurementId, Long.parseLong(value))); - break; - case FLOAT: - ret.addTuple(new FloatDataPoint(measurementId, Float.parseFloat(value))); - break; - case DOUBLE: - ret.addTuple(new DoubleDataPoint(measurementId, Double.parseDouble(value))); - break; - case BOOLEAN: - ret.addTuple(new BooleanDataPoint(measurementId, Boolean.parseBoolean(value))); - break; - case TEXT: - case BLOB: - case STRING: - ret.addTuple(new StringDataPoint(measurementId, BytesUtils.valueOf(items[i + 1]))); - break; - default: - LOG.warn("unsupported data type:{}", type); - break; - } + ret.addTuple(Type.fromTsDataType(type).getDataPoint(measurementId, value)); } catch (NumberFormatException e) { - LOG.warn("parsing measurement meets error, omit it", e); + LOG.warn(Messages.get("log.utils.record_parse_measurement_error"), e); + } catch (UnSupportedDataTypeException e) { + LOG.warn(Messages.get("log.utils.record_unsupported_data_type"), type); } } } From ff4843f21b4e864c19a0a81561ae053bbaec6c4b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 12:05:36 +0800 Subject: [PATCH 112/131] refactor castFromSingleValue --- .../org/apache/tsfile/enums/TSDataType.java | 229 ++++++++---------- 1 file changed, 102 insertions(+), 127 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index f97291046..3e39ad6b7 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -39,45 +39,47 @@ public enum TSDataType { /** BOOLEAN. */ - BOOLEAN((byte) 0), + BOOLEAN((byte) 0, TSDataType::castIdenticalSingleValue), /** INT32. */ - INT32((byte) 1), + INT32((byte) 1, TSDataType::castIdenticalSingleValue), /** INT64. */ - INT64((byte) 2), + INT64((byte) 2, TSDataType::castToLongSingleValue), /** FLOAT. */ - FLOAT((byte) 3), + FLOAT((byte) 3, TSDataType::castToFloatSingleValue), /** DOUBLE. */ - DOUBLE((byte) 4), + DOUBLE((byte) 4, TSDataType::castToDoubleSingleValue), /** TEXT. */ - TEXT((byte) 5), + TEXT((byte) 5, TSDataType::castToTextSingleValue), /** VECTOR. */ - VECTOR((byte) 6), + VECTOR((byte) 6, TSDataType::unsupportedSingleValueCast), /** UNKNOWN. */ - UNKNOWN((byte) 7), + UNKNOWN((byte) 7, TSDataType::unsupportedSingleValueCast), /** TIMESTAMP. */ - TIMESTAMP((byte) 8), + TIMESTAMP((byte) 8, TSDataType::castToTimestampSingleValue), /** DATE. */ - DATE((byte) 9), + DATE((byte) 9, TSDataType::castIdenticalSingleValue), /** BLOB. */ - BLOB((byte) 10), + BLOB((byte) 10, TSDataType::castToBlobSingleValue), /** STRING */ - STRING((byte) 11), + STRING((byte) 11, TSDataType::castToTextSingleValue), /** OBJECT */ - OBJECT((byte) 12); + OBJECT((byte) 12, TSDataType::castIdenticalSingleValue); + private static final Object UNSUPPORTED_CAST = new Object(); private final byte type; + private final SingleValueCaster singleValueCaster; private static final Map> compatibleTypes; static { @@ -147,8 +149,9 @@ public enum TSDataType { compatibleTypes.put(OBJECT, Collections.emptySet()); } - TSDataType(byte type) { + TSDataType(byte type, SingleValueCaster singleValueCaster) { this.type = type; + this.singleValueCaster = singleValueCaster; } /** @@ -211,124 +214,90 @@ public Object castFromSingleValue(TSDataType sourceType, Object value) { if (Objects.isNull(value)) { return null; } - switch (this) { - case BOOLEAN: - if (sourceType == TSDataType.BOOLEAN) { - return value; - } else { - break; - } - case INT32: - if (sourceType == TSDataType.INT32) { - return value; - } else { - break; - } - case INT64: - if (sourceType == TSDataType.INT64) { - return value; - } else if (sourceType == INT32) { - return (long) ((int) value); - } else if (sourceType == TIMESTAMP) { - return value; - } else { - break; - } - case FLOAT: - if (sourceType == TSDataType.FLOAT) { - return value; - } else if (sourceType == INT32) { - return (float) ((int) value); - } else { - break; - } - case DOUBLE: - if (sourceType == TSDataType.DOUBLE) { - return value; - } else if (sourceType == INT32) { - return (double) ((int) value); - } else if (sourceType == INT64) { - return (double) ((long) value); - } else if (sourceType == FLOAT) { - return (double) ((float) value); - } else if (sourceType == TIMESTAMP) { - return (double) ((long) value); - } else { - break; - } - case TEXT: - if (sourceType == TSDataType.TEXT || sourceType == TSDataType.STRING) { - return value; - } else if (sourceType == TSDataType.INT32 - || sourceType == TSDataType.INT64 - || sourceType == TSDataType.FLOAT - || sourceType == TSDataType.DOUBLE - || sourceType == TSDataType.BOOLEAN - || sourceType == TSDataType.TIMESTAMP) { - return new Binary(String.valueOf(value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.DATE) { - return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.BLOB) { - return new Binary(value.toString(), StandardCharsets.UTF_8); - } else { - break; - } - case TIMESTAMP: - if (sourceType == TSDataType.TIMESTAMP) { - return value; - } else if (sourceType == INT32) { - return (long) ((int) value); - } else if (sourceType == INT64) { - return value; - } else { - break; - } - case DATE: - if (sourceType == TSDataType.DATE) { - return value; - } else { - break; - } - case BLOB: - if (sourceType == TSDataType.BLOB - || sourceType == TSDataType.STRING - || sourceType == TSDataType.TEXT) { - return value; - } else { - break; - } - case STRING: - if (sourceType == TSDataType.STRING || sourceType == TSDataType.TEXT) { - return value; - } else if (sourceType == TSDataType.INT32 - || sourceType == TSDataType.INT64 - || sourceType == TSDataType.FLOAT - || sourceType == TSDataType.DOUBLE - || sourceType == TSDataType.BOOLEAN - || sourceType == TSDataType.TIMESTAMP) { - return new Binary(String.valueOf(value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.DATE) { - return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.BLOB) { - return new Binary(value.toString(), StandardCharsets.UTF_8); - } else { - break; - } - case OBJECT: - if (sourceType == TSDataType.OBJECT) { - return value; - } else { - break; - } - case VECTOR: - case UNKNOWN: - default: - break; + Object result = singleValueCaster.cast(this, sourceType, value); + if (result != UNSUPPORTED_CAST) { + return result; } throw new ClassCastException( Messages.format("error.common.unsupported_cast", sourceType, this)); } + private static Object castIdenticalSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + return sourceType == targetType ? value : UNSUPPORTED_CAST; + } + + private static Object castToLongSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return value; + } + return sourceType == INT32 ? (long) ((int) value) : UNSUPPORTED_CAST; + } + + private static Object castToFloatSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == FLOAT) { + return value; + } + return sourceType == INT32 ? (float) ((int) value) : UNSUPPORTED_CAST; + } + + private static Object castToDoubleSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == DOUBLE) { + return value; + } + if (sourceType == INT32) { + return (double) ((int) value); + } + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return (double) ((long) value); + } + return sourceType == FLOAT ? (double) ((float) value) : UNSUPPORTED_CAST; + } + + private static Object castToTextSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == TEXT || sourceType == STRING) { + return value; + } + if (sourceType == INT32 + || sourceType == INT64 + || sourceType == FLOAT + || sourceType == DOUBLE + || sourceType == BOOLEAN + || sourceType == TIMESTAMP) { + return new Binary(String.valueOf(value), StandardCharsets.UTF_8); + } + if (sourceType == DATE) { + return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8); + } + return sourceType == BLOB + ? new Binary(value.toString(), StandardCharsets.UTF_8) + : UNSUPPORTED_CAST; + } + + private static Object castToTimestampSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == TIMESTAMP || sourceType == INT64) { + return value; + } + return sourceType == INT32 ? (long) ((int) value) : UNSUPPORTED_CAST; + } + + private static Object castToBlobSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + return sourceType == BLOB || sourceType == STRING || sourceType == TEXT + ? value + : UNSUPPORTED_CAST; + } + + private static Object unsupportedSingleValueCast( + TSDataType targetType, TSDataType sourceType, Object value) { + return UNSUPPORTED_CAST; + } + @SuppressWarnings({"java:S3012", "java:S3776", "java:S6541"}) public Object castFromArray(TSDataType sourceType, Object array) { switch (this) { @@ -603,4 +572,10 @@ public boolean hasNoValueInStatistics() { public static String getDateStringValue(int value) { return String.format("%04d-%02d-%02d", value / 10000, (value % 10000) / 100, value % 100); } + + @FunctionalInterface + private interface SingleValueCaster { + + Object cast(TSDataType targetType, TSDataType sourceType, Object value); + } } From 6938be35201b319963072d63902d89c8ddbbedbe Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 12:10:47 +0800 Subject: [PATCH 113/131] refactor castFromArray --- .../org/apache/tsfile/enums/TSDataType.java | 311 +++++++++--------- 1 file changed, 156 insertions(+), 155 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index 3e39ad6b7..08540bd9f 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -39,47 +39,48 @@ public enum TSDataType { /** BOOLEAN. */ - BOOLEAN((byte) 0, TSDataType::castIdenticalSingleValue), + BOOLEAN((byte) 0, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** INT32. */ - INT32((byte) 1, TSDataType::castIdenticalSingleValue), + INT32((byte) 1, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** INT64. */ - INT64((byte) 2, TSDataType::castToLongSingleValue), + INT64((byte) 2, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), /** FLOAT. */ - FLOAT((byte) 3, TSDataType::castToFloatSingleValue), + FLOAT((byte) 3, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), /** DOUBLE. */ - DOUBLE((byte) 4, TSDataType::castToDoubleSingleValue), + DOUBLE((byte) 4, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), /** TEXT. */ - TEXT((byte) 5, TSDataType::castToTextSingleValue), + TEXT((byte) 5, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** VECTOR. */ - VECTOR((byte) 6, TSDataType::unsupportedSingleValueCast), + VECTOR((byte) 6, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), /** UNKNOWN. */ - UNKNOWN((byte) 7, TSDataType::unsupportedSingleValueCast), + UNKNOWN((byte) 7, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), /** TIMESTAMP. */ - TIMESTAMP((byte) 8, TSDataType::castToTimestampSingleValue), + TIMESTAMP((byte) 8, TSDataType::castToTimestampSingleValue, TSDataType::castToTimestampArray), /** DATE. */ - DATE((byte) 9, TSDataType::castIdenticalSingleValue), + DATE((byte) 9, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** BLOB. */ - BLOB((byte) 10, TSDataType::castToBlobSingleValue), + BLOB((byte) 10, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), /** STRING */ - STRING((byte) 11, TSDataType::castToTextSingleValue), + STRING((byte) 11, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** OBJECT */ - OBJECT((byte) 12, TSDataType::castIdenticalSingleValue); + OBJECT((byte) 12, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray); private static final Object UNSUPPORTED_CAST = new Object(); private final byte type; private final SingleValueCaster singleValueCaster; + private final ArrayCaster arrayCaster; private static final Map> compatibleTypes; static { @@ -149,9 +150,10 @@ public enum TSDataType { compatibleTypes.put(OBJECT, Collections.emptySet()); } - TSDataType(byte type, SingleValueCaster singleValueCaster) { + TSDataType(byte type, SingleValueCaster singleValueCaster, ArrayCaster arrayCaster) { this.type = type; this.singleValueCaster = singleValueCaster; + this.arrayCaster = arrayCaster; } /** @@ -298,153 +300,146 @@ private static Object unsupportedSingleValueCast( return UNSUPPORTED_CAST; } - @SuppressWarnings({"java:S3012", "java:S3776", "java:S6541"}) public Object castFromArray(TSDataType sourceType, Object array) { - switch (this) { - case BOOLEAN: - if (sourceType == TSDataType.BOOLEAN) { - return array; - } else { - break; - } - case INT32: - if (sourceType == TSDataType.INT32) { - return array; - } else { - break; - } - case INT64: - if (sourceType == TSDataType.INT64) { - return array; - } else if (sourceType == INT32) { - return Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray(); - } else if (sourceType == TIMESTAMP) { - return array; - } else { - break; - } - case FLOAT: - if (sourceType == TSDataType.FLOAT) { - return array; - } else if (sourceType == INT32) { - int[] tmp = (int[]) array; - float[] result = new float[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = tmp[i]; - } - return result; - } else { - break; - } - case DOUBLE: - if (sourceType == TSDataType.DOUBLE) { - return array; - } else if (sourceType == INT32) { - return Arrays.stream((int[]) array).mapToDouble(Double::valueOf).toArray(); - } else if (sourceType == INT64) { - return Arrays.stream((long[]) array).mapToDouble(Double::valueOf).toArray(); - } else if (sourceType == FLOAT) { - float[] tmp = (float[]) array; - double[] result = new double[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = tmp[i]; - } - return result; - } else if (sourceType == TIMESTAMP) { - return Arrays.stream((long[]) array).mapToDouble(Double::valueOf).toArray(); - } else { - break; - } - case TIMESTAMP: - if (sourceType == TSDataType.TIMESTAMP) { - return array; - } else if (sourceType == INT32) { - return Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray(); - } else if (sourceType == INT64) { - return array; - } else { - break; - } - case DATE: - if (sourceType == TSDataType.DATE) { - return array; - } else { - break; - } - case BLOB: - if (sourceType == TSDataType.BLOB - || sourceType == TSDataType.STRING - || sourceType == TSDataType.TEXT) { - return array; - } else { - break; - } - case TEXT: - case STRING: - if (sourceType == TSDataType.STRING - || sourceType == TSDataType.TEXT - || sourceType == TSDataType.BLOB) { - return array; - } else if (sourceType == TSDataType.INT32) { - int[] tmp = (int[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.DATE) { - int[] tmp = (int[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(TSDataType.getDateStringValue(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.INT64 || sourceType == TSDataType.TIMESTAMP) { - long[] tmp = (long[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.FLOAT) { - float[] tmp = (float[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.DOUBLE) { - double[] tmp = (double[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.BOOLEAN) { - boolean[] tmp = (boolean[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else { - break; - } - case OBJECT: - if (sourceType == TSDataType.OBJECT) { - return array; - } else { - break; - } - case VECTOR: - case UNKNOWN: - default: - break; + Object result = arrayCaster.cast(this, sourceType, array); + if (result != UNSUPPORTED_CAST) { + return result; } throw new ClassCastException( Messages.format("error.common.unsupported_cast", sourceType, this)); } + private static Object castIdenticalArray( + TSDataType targetType, TSDataType sourceType, Object array) { + return sourceType == targetType ? array : UNSUPPORTED_CAST; + } + + private static Object castToLongArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return array; + } + return sourceType == INT32 + ? Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray() + : UNSUPPORTED_CAST; + } + + private static Object castToFloatArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == FLOAT) { + return array; + } + if (sourceType != INT32) { + return UNSUPPORTED_CAST; + } + int[] values = (int[]) array; + float[] result = new float[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = values[i]; + } + return result; + } + + private static Object castToDoubleArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == DOUBLE) { + return array; + } + if (sourceType == INT32) { + return Arrays.stream((int[]) array).mapToDouble(Double::valueOf).toArray(); + } + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return Arrays.stream((long[]) array).mapToDouble(Double::valueOf).toArray(); + } + if (sourceType != FLOAT) { + return UNSUPPORTED_CAST; + } + float[] values = (float[]) array; + double[] result = new double[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = values[i]; + } + return result; + } + + private static Object castToTimestampArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == TIMESTAMP || sourceType == INT64) { + return array; + } + return sourceType == INT32 + ? Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray() + : UNSUPPORTED_CAST; + } + + private static Object castToBlobArray( + TSDataType targetType, TSDataType sourceType, Object array) { + return sourceType == BLOB || sourceType == STRING || sourceType == TEXT + ? array + : UNSUPPORTED_CAST; + } + + private static Object castToTextArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == STRING || sourceType == TEXT || sourceType == BLOB) { + return array; + } + if (sourceType == INT32) { + int[] values = (int[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == DATE) { + int[] values = (int[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(getDateStringValue(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == INT64 || sourceType == TIMESTAMP) { + long[] values = (long[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == FLOAT) { + float[] values = (float[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == DOUBLE) { + double[] values = (double[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == BOOLEAN) { + boolean[] values = (boolean[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + return UNSUPPORTED_CAST; + } + + private static Object unsupportedArrayCast( + TSDataType targetType, TSDataType sourceType, Object array) { + return UNSUPPORTED_CAST; + } + public static TSDataType deserializeFrom(ByteBuffer buffer) { return deserialize(buffer.get()); } @@ -578,4 +573,10 @@ private interface SingleValueCaster { Object cast(TSDataType targetType, TSDataType sourceType, Object value); } + + @FunctionalInterface + private interface ArrayCaster { + + Object cast(TSDataType targetType, TSDataType sourceType, Object array); + } } From b36c4fee1e088c0417d0918e702538fbf26e08df Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 14:10:20 +0800 Subject: [PATCH 114/131] refactor getDataTypeSize --- .../org/apache/tsfile/enums/TSDataType.java | 58 ++++++++----------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index 08540bd9f..002d56487 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -39,46 +39,52 @@ public enum TSDataType { /** BOOLEAN. */ - BOOLEAN((byte) 0, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + BOOLEAN((byte) 0, 1, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** INT32. */ - INT32((byte) 1, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + INT32((byte) 1, 4, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** INT64. */ - INT64((byte) 2, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), + INT64((byte) 2, 8, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), /** FLOAT. */ - FLOAT((byte) 3, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), + FLOAT((byte) 3, 4, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), /** DOUBLE. */ - DOUBLE((byte) 4, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), + DOUBLE((byte) 4, 8, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), /** TEXT. */ - TEXT((byte) 5, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), + TEXT((byte) 5, 8, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** VECTOR. */ - VECTOR((byte) 6, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), + VECTOR((byte) 6, 8, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), /** UNKNOWN. */ - UNKNOWN((byte) 7, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), + UNKNOWN( + (byte) 7, + TSDataType.UNSUPPORTED_DATA_TYPE_SIZE, + TSDataType::unsupportedSingleValueCast, + TSDataType::unsupportedArrayCast), /** TIMESTAMP. */ - TIMESTAMP((byte) 8, TSDataType::castToTimestampSingleValue, TSDataType::castToTimestampArray), + TIMESTAMP((byte) 8, 8, TSDataType::castToTimestampSingleValue, TSDataType::castToTimestampArray), /** DATE. */ - DATE((byte) 9, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + DATE((byte) 9, 4, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** BLOB. */ - BLOB((byte) 10, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), + BLOB((byte) 10, 8, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), /** STRING */ - STRING((byte) 11, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), + STRING((byte) 11, 8, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** OBJECT */ - OBJECT((byte) 12, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray); + OBJECT((byte) 12, 8, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray); + private static final int UNSUPPORTED_DATA_TYPE_SIZE = -1; private static final Object UNSUPPORTED_CAST = new Object(); private final byte type; + private final int dataTypeSize; private final SingleValueCaster singleValueCaster; private final ArrayCaster arrayCaster; private static final Map> compatibleTypes; @@ -150,8 +156,10 @@ public enum TSDataType { compatibleTypes.put(OBJECT, Collections.emptySet()); } - TSDataType(byte type, SingleValueCaster singleValueCaster, ArrayCaster arrayCaster) { + TSDataType( + byte type, int dataTypeSize, SingleValueCaster singleValueCaster, ArrayCaster arrayCaster) { this.type = type; + this.dataTypeSize = dataTypeSize; this.singleValueCaster = singleValueCaster; this.arrayCaster = arrayCaster; } @@ -465,26 +473,10 @@ public void serializeTo(FileOutputStream outputStream) throws IOException { } public int getDataTypeSize() { - switch (this) { - case BOOLEAN: - return 1; - case INT32: - case FLOAT: - case DATE: - return 4; - // For text: return the size of reference here - case TEXT: - case INT64: - case DOUBLE: - case VECTOR: - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - return 8; - default: - throw new UnSupportedDataTypeException(this.toString()); + if (dataTypeSize == UNSUPPORTED_DATA_TYPE_SIZE) { + throw new UnSupportedDataTypeException(this.toString()); } + return dataTypeSize; } /** From b9033a72bd0916ecb938513091830b3d1da927ea Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 14:13:46 +0800 Subject: [PATCH 115/131] refactor isNumeric --- .../org/apache/tsfile/enums/TSDataType.java | 56 +++++++++---------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index 002d56487..285971374 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -39,52 +39,56 @@ public enum TSDataType { /** BOOLEAN. */ - BOOLEAN((byte) 0, 1, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + BOOLEAN((byte) 0, 1, false, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** INT32. */ - INT32((byte) 1, 4, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + INT32((byte) 1, 4, true, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** INT64. */ - INT64((byte) 2, 8, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), + INT64((byte) 2, 8, true, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), /** FLOAT. */ - FLOAT((byte) 3, 4, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), + FLOAT((byte) 3, 4, true, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), /** DOUBLE. */ - DOUBLE((byte) 4, 8, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), + DOUBLE((byte) 4, 8, true, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), /** TEXT. */ - TEXT((byte) 5, 8, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), + TEXT((byte) 5, 8, false, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** VECTOR. */ - VECTOR((byte) 6, 8, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), + VECTOR( + (byte) 6, 8, false, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), /** UNKNOWN. */ UNKNOWN( (byte) 7, TSDataType.UNSUPPORTED_DATA_TYPE_SIZE, + false, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), /** TIMESTAMP. */ - TIMESTAMP((byte) 8, 8, TSDataType::castToTimestampSingleValue, TSDataType::castToTimestampArray), + TIMESTAMP( + (byte) 8, 8, false, TSDataType::castToTimestampSingleValue, TSDataType::castToTimestampArray), /** DATE. */ - DATE((byte) 9, 4, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + DATE((byte) 9, 4, false, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), /** BLOB. */ - BLOB((byte) 10, 8, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), + BLOB((byte) 10, 8, false, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), /** STRING */ - STRING((byte) 11, 8, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), + STRING((byte) 11, 8, false, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** OBJECT */ - OBJECT((byte) 12, 8, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray); + OBJECT((byte) 12, 8, false, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray); private static final int UNSUPPORTED_DATA_TYPE_SIZE = -1; private static final Object UNSUPPORTED_CAST = new Object(); private final byte type; private final int dataTypeSize; + private final boolean numeric; private final SingleValueCaster singleValueCaster; private final ArrayCaster arrayCaster; private static final Map> compatibleTypes; @@ -157,9 +161,14 @@ public enum TSDataType { } TSDataType( - byte type, int dataTypeSize, SingleValueCaster singleValueCaster, ArrayCaster arrayCaster) { + byte type, + int dataTypeSize, + boolean numeric, + SingleValueCaster singleValueCaster, + ArrayCaster arrayCaster) { this.type = type; this.dataTypeSize = dataTypeSize; + this.numeric = numeric; this.singleValueCaster = singleValueCaster; this.arrayCaster = arrayCaster; } @@ -495,25 +504,10 @@ public byte serialize() { * @throws UnSupportedDataTypeException when meets unSupported DataType */ public boolean isNumeric() { - switch (this) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - return true; - // For text: return the size of reference here - case BLOB: - case TIMESTAMP: - case DATE: - case STRING: - case BOOLEAN: - case TEXT: - case VECTOR: - case OBJECT: - return false; - default: - throw new UnSupportedDataTypeException(this.toString()); + if (this == UNKNOWN) { + throw new UnSupportedDataTypeException(this.toString()); } + return numeric; } /** From 8ad7980725c13dc4d7e169d225308caee395598b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 20 Jul 2026 17:20:22 +0800 Subject: [PATCH 116/131] refactor public boolean isComparable() { --- .../org/apache/tsfile/enums/TSDataType.java | 83 ++++++++++++------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index 285971374..eaa09be73 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -39,56 +39,93 @@ public enum TSDataType { /** BOOLEAN. */ - BOOLEAN((byte) 0, 1, false, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + BOOLEAN( + (byte) 0, + 1, + false, + true, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray), /** INT32. */ - INT32((byte) 1, 4, true, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + INT32( + (byte) 1, + 4, + true, + true, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray), /** INT64. */ - INT64((byte) 2, 8, true, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), + INT64((byte) 2, 8, true, true, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), /** FLOAT. */ - FLOAT((byte) 3, 4, true, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), + FLOAT((byte) 3, 4, true, true, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), /** DOUBLE. */ - DOUBLE((byte) 4, 8, true, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), + DOUBLE( + (byte) 4, 8, true, true, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), /** TEXT. */ - TEXT((byte) 5, 8, false, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), + TEXT((byte) 5, 8, false, true, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** VECTOR. */ VECTOR( - (byte) 6, 8, false, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), + (byte) 6, + 8, + false, + false, + TSDataType::unsupportedSingleValueCast, + TSDataType::unsupportedArrayCast), /** UNKNOWN. */ UNKNOWN( (byte) 7, TSDataType.UNSUPPORTED_DATA_TYPE_SIZE, false, + false, TSDataType::unsupportedSingleValueCast, TSDataType::unsupportedArrayCast), /** TIMESTAMP. */ TIMESTAMP( - (byte) 8, 8, false, TSDataType::castToTimestampSingleValue, TSDataType::castToTimestampArray), + (byte) 8, + 8, + false, + true, + TSDataType::castToTimestampSingleValue, + TSDataType::castToTimestampArray), /** DATE. */ - DATE((byte) 9, 4, false, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray), + DATE( + (byte) 9, + 4, + false, + true, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray), /** BLOB. */ - BLOB((byte) 10, 8, false, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), + BLOB((byte) 10, 8, false, false, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), /** STRING */ - STRING((byte) 11, 8, false, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), + STRING((byte) 11, 8, false, true, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** OBJECT */ - OBJECT((byte) 12, 8, false, TSDataType::castIdenticalSingleValue, TSDataType::castIdenticalArray); + OBJECT( + (byte) 12, + 8, + false, + false, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray); private static final int UNSUPPORTED_DATA_TYPE_SIZE = -1; private static final Object UNSUPPORTED_CAST = new Object(); private final byte type; private final int dataTypeSize; private final boolean numeric; + private final boolean comparable; private final SingleValueCaster singleValueCaster; private final ArrayCaster arrayCaster; private static final Map> compatibleTypes; @@ -164,11 +201,13 @@ public enum TSDataType { byte type, int dataTypeSize, boolean numeric, + boolean comparable, SingleValueCaster singleValueCaster, ArrayCaster arrayCaster) { this.type = type; this.dataTypeSize = dataTypeSize; this.numeric = numeric; + this.comparable = comparable; this.singleValueCaster = singleValueCaster; this.arrayCaster = arrayCaster; } @@ -517,24 +556,10 @@ public boolean isNumeric() { * @throws UnSupportedDataTypeException when meets unSupported DataType */ public boolean isComparable() { - switch (this) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case TEXT: - case BOOLEAN: - case TIMESTAMP: - case DATE: - case STRING: - return true; - case VECTOR: - case BLOB: - case OBJECT: - return false; - default: - throw new UnSupportedDataTypeException(this.toString()); + if (this == UNKNOWN) { + throw new UnSupportedDataTypeException(this.toString()); } + return comparable; } public boolean isBinary() { From 184a2c50aec33814f698c04cfe1b8fc96724d8c1 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 21 Jul 2026 10:33:32 +0800 Subject: [PATCH 117/131] add update --- .../read/common/type/AbstractIntType.java | 6 ++++++ .../read/common/type/AbstractLongType.java | 6 ++++++ .../read/common/type/AbstractVarcharType.java | 6 ++++++ .../tsfile/read/common/type/BlobType.java | 6 ++++++ .../tsfile/read/common/type/BooleanType.java | 6 ++++++ .../tsfile/read/common/type/DoubleType.java | 6 ++++++ .../tsfile/read/common/type/FloatType.java | 6 ++++++ .../tsfile/read/common/type/ObjectType.java | 6 ++++++ .../apache/tsfile/read/common/type/Type.java | 6 ++++++ .../tsfile/read/common/type/VectorType.java | 6 ++++++ .../tsfile/read/common/type/TypeTest.java | 19 +++++++++++++++++++ 11 files changed, 79 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index f5f427615..e75a55a83 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -39,6 +39,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; @@ -165,6 +166,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getInt()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getInt(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index eeeea0cac..bc3df4b39 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -38,6 +38,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; @@ -164,6 +165,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getLong()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getLong(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index a633c4771..5fa6149a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -30,6 +30,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; @@ -144,6 +145,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBinary()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBinary(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 4fbe5b95f..c532a24b4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -31,6 +31,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; @@ -150,6 +151,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBinary()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBinary(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 1e2d9763d..64a5bcdff 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -32,6 +32,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import org.apache.tsfile.read.common.block.column.ColumnEncoder; @@ -156,6 +157,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBoolean()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBoolean(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index e7a8887c1..d55e25448 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -38,6 +38,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; @@ -167,6 +168,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getDouble()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getDouble(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 040b37b06..366ce4b9b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -37,6 +37,7 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; @@ -165,6 +166,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getFloat()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getFloat(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 1ae37bff6..5a1e84272 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -31,6 +31,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; @@ -152,6 +153,11 @@ public void update(Statistics stats, BatchData batchData) { stats.update(batchData.currentTime(), batchData.getBinary()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBinary(rowIndex)); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index b5faf76d6..b20b3e4c3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -32,6 +32,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; @@ -468,6 +469,11 @@ default void update(Statistics stats, BatchData batchData) { throw new UnsupportedOperationException(getClass().getName()); } + /** Updates statistics with the value at {@code rowIndex} in a TsBlock column. */ + default void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Returns the current value in {@code batchData}. */ default Object getCurrentValue(BatchData batchData) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index f79bde536..6fb1293d8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -35,6 +35,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; @@ -232,6 +233,11 @@ public void update(Statistics stats, BatchData batchData) { throw new UnsupportedOperationException(getDisplayName()); } + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + throw new UnsupportedOperationException(getDisplayName()); + } + @Override public Object getCurrentValue(BatchData batchData) { return batchData.getVector(); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index a9ac1ac58..ee18315cc 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -28,6 +28,8 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; @@ -1550,6 +1552,23 @@ public void testUpdateStatistics() { Assert.assertEquals(200L, batchStatistics.getStartTime()); Assert.assertEquals(200L, batchStatistics.getEndTime()); Assert.assertFalse(batchStatistics.isEmpty()); + + TsBlockBuilder blockBuilder = new TsBlockBuilder(Arrays.asList(dataType, dataType)); + blockBuilder.getTimeColumnBuilder().writeLong(250L); + blockBuilder.getColumnBuilder(0).writeObject(value.getValue()); + blockBuilder.getColumnBuilder(1).writeObject(value.getValue()); + blockBuilder.declarePosition(); + blockBuilder.getTimeColumnBuilder().writeLong(300L); + blockBuilder.getColumnBuilder(0).writeObject(value.getValue()); + blockBuilder.getColumnBuilder(1).writeObject(value.getValue()); + blockBuilder.declarePosition(); + TsBlock block = blockBuilder.build(); + Statistics blockStatistics = Statistics.getStatsByType(dataType); + type.update(blockStatistics, block, 1, 1); + Assert.assertEquals(1, blockStatistics.getCount()); + Assert.assertEquals(300L, blockStatistics.getStartTime()); + Assert.assertEquals(300L, blockStatistics.getEndTime()); + Assert.assertFalse(blockStatistics.isEmpty()); } } From 96c7d4e3c26f02678d4f75694c70cc7ee3d00ba7 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 21 Jul 2026 11:11:55 +0800 Subject: [PATCH 118/131] fix null binary conversion --- .../read/common/type/AbstractVarcharType.java | 3 ++- .../apache/tsfile/read/common/type/BlobType.java | 4 ++++ .../apache/tsfile/read/common/type/TypeTest.java | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 5fa6149a4..c4a521238 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -177,7 +177,8 @@ public void setTo(TsPrimitiveType from, Column to, int index) { @Override public void setTo(Object value, Column to, int startIndex, int endIndex) { - Arrays.fill(to.getBinaries(), startIndex, endIndex, toBinaryValue(value)); + Arrays.fill( + to.getBinaries(), startIndex, endIndex, value == null ? null : toBinaryValue(value)); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index c532a24b4..866f6c426 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -183,6 +183,10 @@ public void setTo(TsPrimitiveType from, Column to, int index) { @Override public void setTo(Object value, Column to, int startIndex, int endIndex) { + if (value == null) { + Arrays.fill(to.getBinaries(), startIndex, endIndex, null); + return; + } Binary[] values = new Binary[1]; addValue(0, value, values); Arrays.fill(to.getBinaries(), startIndex, endIndex, values[0]); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index ee18315cc..e40838d3e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -1108,6 +1108,20 @@ public void testSetToArray() { } } + @Test + public void testSetNullToBinaryColumnRange() { + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Binary[] values = {Binary.EMPTY_VALUE, Binary.EMPTY_VALUE}; + Column column = Mockito.mock(Column.class); + Mockito.when(column.getBinaries()).thenReturn(values); + + Type.fromTsDataType(dataType).setTo(null, column, 0, values.length); + + Assert.assertArrayEquals(new Binary[] {null, null}, values); + } + } + @Test public void testDateAddValueWithIntArray() throws IOException { Type dateType = Type.fromTsDataType(TSDataType.DATE); From c08c0af1a248546e317e52dee0fe790832078e86 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 21 Jul 2026 11:51:07 +0800 Subject: [PATCH 119/131] add write --- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 5 ++ .../tsfile/read/common/type/VectorType.java | 5 ++ .../tsfile/read/common/type/TypeTest.java | 46 +++++++++++++++++++ 11 files changed, 96 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index e75a55a83..375ca27f9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -324,6 +324,11 @@ public double getDouble(Column c, int position) { return c.getInt(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeInt(value.getInt()); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeInt(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index bc3df4b39..c770edeaf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -323,6 +323,11 @@ public double getDouble(Column c, int position) { return c.getLong(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeLong(value.getLong()); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeLong(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index c4a521238..c379aa4f7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -284,6 +284,11 @@ public Binary getBinary(Column c, int position) { return c.getBinary(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBinary(value.getBinary()); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 866f6c426..dea4d84cc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -302,6 +302,11 @@ public Binary getBinary(Column c, int position) { return c.getBinary(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBinary(value.getBinary()); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 64a5bcdff..963b21217 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -307,6 +307,11 @@ public boolean getBoolean(Column c, int position) { return c.getBoolean(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBoolean(value.getBoolean()); + } + @Override public void writeBoolean(ColumnBuilder builder, boolean value) { builder.writeBoolean(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index d55e25448..b21b0ffa4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -333,6 +333,11 @@ public double getDouble(Column c, int position) { return c.getDouble(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeDouble(value.getDouble()); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeDouble(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 366ce4b9b..4def2fc46 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -331,6 +331,11 @@ public double getDouble(Column c, int position) { return c.getFloat(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeFloat(value.getFloat()); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeFloat(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 5a1e84272..9076ec805 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -292,6 +292,11 @@ public Binary getBinary(Column c, int position) { return c.getBinary(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBinary(value.getBinary()); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index b20b3e4c3..a5c9ddf39 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -155,6 +155,11 @@ default void serializeValue(Object value, DataOutputStream stream) throws IOExce throw new UnsupportedOperationException(getClass().getName()); } + /** Writes a TsPrimitiveType value to the current entry. */ + default void write(ColumnBuilder builder, TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 6fb1293d8..8d5dc065f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -83,6 +83,11 @@ public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((long[]) array, newLength); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index e40838d3e..80ba2a575 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.encoding.encoder.PlainEncoder; import org.apache.tsfile.enums.TSDataType; @@ -61,6 +62,51 @@ public class TypeTest { private static final int OFFSET = 2; private static final int VALUE_LENGTH = 10; + @Test + public void testWriteTsPrimitiveTypeToColumnBuilder() { + ColumnBuilder builder = Mockito.mock(ColumnBuilder.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260721, TSDataType.DATE)}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L)}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L)}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D)} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]).write(builder, (TsPrimitiveType) testCase[1]); + } + + Mockito.verify(builder).writeBoolean(true); + Mockito.verify(builder).writeInt(1); + Mockito.verify(builder).writeInt(20260721); + Mockito.verify(builder).writeLong(2L); + Mockito.verify(builder).writeLong(3L); + Mockito.verify(builder).writeFloat(1.25F); + Mockito.verify(builder).writeDouble(2.5D); + Mockito.reset(builder); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType binaryValue = new TsPrimitiveType.TsBinary(binary); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(builder, binaryValue); + Mockito.verify(builder).writeBinary(binary); + Mockito.reset(builder); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(builder, new TsPrimitiveType.TsLong(1L)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + @Test public void testGetValueAsTsPrimitiveType() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From 03cdf35e3f0ceabd67aba3a9b064e5a145175541 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 22 Jul 2026 15:07:31 +0800 Subject: [PATCH 120/131] return serialized size --- .../read/common/type/AbstractIntType.java | 12 +++-- .../read/common/type/AbstractLongType.java | 12 +++-- .../read/common/type/AbstractVarcharType.java | 17 ++++--- .../tsfile/read/common/type/BlobType.java | 17 ++++--- .../tsfile/read/common/type/BooleanType.java | 12 +++-- .../tsfile/read/common/type/DoubleType.java | 12 +++-- .../tsfile/read/common/type/FloatType.java | 12 +++-- .../tsfile/read/common/type/ObjectType.java | 17 ++++--- .../apache/tsfile/read/common/type/Type.java | 16 +++---- .../tsfile/read/common/type/VectorType.java | 17 +++++-- .../tsfile/read/common/type/TypeTest.java | 47 +++++++++++++++++-- 11 files changed, 137 insertions(+), 54 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 375ca27f9..54102b549 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -91,28 +91,32 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); outputStream.writeInt(batchData.getIntByIndex(index)); } + return Math.multiplyExact(Long.BYTES + Integer.BYTES, batchData.length()); } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { stream.writeInt(value.getInt()); + return Integer.BYTES; } @Override - public void serializeValue(Object value, ByteBuffer buffer) { + public int serializeValue(Object value, ByteBuffer buffer) { buffer.putInt((int) value); + return Integer.BYTES; } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { + public int serializeValue(Object value, DataOutputStream stream) throws IOException { stream.writeInt((int) value); + return Integer.BYTES; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index c770edeaf..319276bab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -90,28 +90,32 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); outputStream.writeLong(batchData.getLongByIndex(index)); } + return Math.multiplyExact(Long.BYTES + Long.BYTES, batchData.length()); } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { stream.writeLong(value.getLong()); + return Long.BYTES; } @Override - public void serializeValue(Object value, ByteBuffer buffer) { + public int serializeValue(Object value, ByteBuffer buffer) { buffer.putLong((long) value); + return Long.BYTES; } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { + public int serializeValue(Object value, DataOutputStream stream) throws IOException { stream.writeLong((long) value); + return Long.BYTES; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index c379aa4f7..de5951e6e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -82,32 +82,37 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + int size = 0; for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); Binary binary = batchData.getBinaryByIndex(index); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); + size = Math.addExact(size, binary.getLength()); } + return size; } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { Binary binary = value.getBinary(); stream.writeInt(binary.getLength()); stream.write(binary.getValues()); + return Math.addExact(Integer.BYTES, binary.getLength()); } @Override - public void serializeValue(Object value, ByteBuffer buffer) { - ReadWriteIOUtils.write((Binary) value, buffer); + public int serializeValue(Object value, ByteBuffer buffer) { + return ReadWriteIOUtils.write((Binary) value, buffer); } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write((Binary) value, stream); + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + return ReadWriteIOUtils.write((Binary) value, stream); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index dea4d84cc..901fbd961 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -88,32 +88,37 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + int size = 0; for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); Binary binary = batchData.getBinaryByIndex(index); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); + size = Math.addExact(size, binary.getLength()); } + return size; } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { Binary binary = value.getBinary(); stream.writeInt(binary.getLength()); stream.write(binary.getValues()); + return Math.addExact(Integer.BYTES, binary.getLength()); } @Override - public void serializeValue(Object value, ByteBuffer buffer) { - ReadWriteIOUtils.write((Binary) value, buffer); + public int serializeValue(Object value, ByteBuffer buffer) { + return ReadWriteIOUtils.write((Binary) value, buffer); } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write((Binary) value, stream); + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + return ReadWriteIOUtils.write((Binary) value, stream); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 963b21217..2fd080166 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -89,28 +89,32 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); outputStream.writeBoolean(batchData.getBooleanByIndex(index)); } + return Math.multiplyExact(Long.BYTES + Byte.BYTES, batchData.length()); } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { stream.writeBoolean(value.getBoolean()); + return Byte.BYTES; } @Override - public void serializeValue(Object value, ByteBuffer buffer) { + public int serializeValue(Object value, ByteBuffer buffer) { buffer.put((boolean) value ? (byte) 1 : (byte) 0); + return Byte.BYTES; } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { + public int serializeValue(Object value, DataOutputStream stream) throws IOException { stream.writeBoolean((boolean) value); + return Byte.BYTES; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index b21b0ffa4..e017f23c3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -94,28 +94,32 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); outputStream.writeDouble(batchData.getDoubleByIndex(index)); } + return Math.multiplyExact(Long.BYTES + Double.BYTES, batchData.length()); } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { stream.writeDouble(value.getDouble()); + return Double.BYTES; } @Override - public void serializeValue(Object value, ByteBuffer buffer) { + public int serializeValue(Object value, ByteBuffer buffer) { buffer.putDouble((double) value); + return Double.BYTES; } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { + public int serializeValue(Object value, DataOutputStream stream) throws IOException { stream.writeDouble((double) value); + return Double.BYTES; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 4def2fc46..28cf3b0dd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -93,28 +93,32 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); outputStream.writeFloat(batchData.getFloatByIndex(index)); } + return Math.multiplyExact(Long.BYTES + Float.BYTES, batchData.length()); } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { stream.writeFloat(value.getFloat()); + return Float.BYTES; } @Override - public void serializeValue(Object value, ByteBuffer buffer) { + public int serializeValue(Object value, ByteBuffer buffer) { buffer.putFloat((float) value); + return Float.BYTES; } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { + public int serializeValue(Object value, DataOutputStream stream) throws IOException { stream.writeFloat((float) value); + return Float.BYTES; } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 9076ec805..9b9ee604d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -90,32 +90,37 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + int size = 0; for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); Binary binary = batchData.getBinaryByIndex(index); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); + size = Math.addExact(size, binary.getLength()); } + return size; } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { Binary binary = value.getBinary(); stream.writeInt(binary.getLength()); stream.write(binary.getValues()); + return Math.addExact(Integer.BYTES, binary.getLength()); } @Override - public void serializeValue(Object value, ByteBuffer buffer) { - ReadWriteIOUtils.write((Binary) value, buffer); + public int serializeValue(Object value, ByteBuffer buffer) { + return ReadWriteIOUtils.write((Binary) value, buffer); } @Override - public void serializeValue(Object value, DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write((Binary) value, stream); + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + return ReadWriteIOUtils.write((Binary) value, stream); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index a5c9ddf39..fad62e1b2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -132,26 +132,26 @@ default void put(BatchData batchData, long timestamp, Object value) { throw new UnSupportedDataTypeException(String.valueOf(batchData.getDataType())); } - /** Serializes all timestamp-value pairs in {@code batchData}. */ - default void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + /** Serializes all timestamp-value pairs in {@code batchData} and returns the serialized size. */ + default int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { throw new IllegalArgumentException( Messages.format("error.read.batch_data_unknown_type", batchData.getDataType())); } - /** Serializes {@code value} to {@code stream}. */ - default void serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + /** Serializes {@code value} to {@code stream} and returns the serialized size. */ + default int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { throw new IllegalArgumentException( Messages.format("error.read.batch_data_unknown_type", value.getDataType())); } - /** Serializes {@code value} to {@code buffer}. */ - default void serializeValue(Object value, ByteBuffer buffer) { + /** Serializes {@code value} to {@code buffer} and returns the serialized size. */ + default int serializeValue(Object value, ByteBuffer buffer) { throw new UnsupportedOperationException(getClass().getName()); } - /** Serializes {@code value} to {@code stream}. */ - default void serializeValue(Object value, DataOutputStream stream) throws IOException { + /** Serializes {@code value} to {@code stream} and returns the serialized size. */ + default int serializeValue(Object value, DataOutputStream stream) throws IOException { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 8d5dc065f..d72370ff1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -160,38 +160,45 @@ public void put(BatchData batchData, long timestamp, Object value) { } @Override - public void serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) throws IOException { + int size = 0; for (int i = 0; i < batchData.length(); i++) { int index = isDesc ? batchData.length() - 1 - i : i; outputStream.writeLong(batchData.getTimeByIndex(index)); TsPrimitiveType[] values = batchData.getVectorByIndex(index); outputStream.writeInt(values.length); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); for (TsPrimitiveType value : values) { + size = Math.addExact(size, Byte.BYTES); if (value == null) { outputStream.write(0); continue; } outputStream.write(1); outputStream.write(value.getDataType().serialize()); - Type.fromTsDataType(value.getDataType()).serialize(value, outputStream); + size = Math.addExact(size, Byte.BYTES); + size = + Math.addExact( + size, Type.fromTsDataType(value.getDataType()).serialize(value, outputStream)); } } + return size; } @Override - public void serialize(TsPrimitiveType value, DataOutputStream stream) { + public int serialize(TsPrimitiveType value, DataOutputStream stream) { throw new IllegalArgumentException( Messages.format("error.read.batch_data_unknown_type", value.getDataType())); } @Override - public void serializeValue(Object value, ByteBuffer buffer) { + public int serializeValue(Object value, ByteBuffer buffer) { throw new UnsupportedOperationException(getDisplayName()); } @Override - public void serializeValue(Object value, DataOutputStream stream) { + public int serializeValue(Object value, DataOutputStream stream) { throw new UnsupportedOperationException(getDisplayName()); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 80ba2a575..9f09057fd 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -249,15 +249,19 @@ public void testSerializeValue() throws IOException { Type type = Type.fromTsDataType((TSDataType) testCase[0]); Object value = testCase[1]; ByteBuffer buffer = ByteBuffer.allocate(64); - type.serializeValue(value, buffer); + int bufferSerializedSize = type.serializeValue(value, buffer); + Assert.assertEquals(buffer.position(), bufferSerializedSize); ByteArrayOutputStream output = new ByteArrayOutputStream(); - type.serialize(type.getTsPrimitiveType(value), new DataOutputStream(output)); + int serializedSize = + type.serialize(type.getTsPrimitiveType(value), new DataOutputStream(output)); byte[] expected = output.toByteArray(); + Assert.assertEquals(expected.length, serializedSize); Assert.assertArrayEquals(expected, Arrays.copyOf(buffer.array(), buffer.position())); output.reset(); - type.serializeValue(value, new DataOutputStream(output)); + int streamSerializedSize = type.serializeValue(value, new DataOutputStream(output)); + Assert.assertEquals(output.size(), streamSerializedSize); Assert.assertArrayEquals(expected, output.toByteArray()); } @@ -279,6 +283,43 @@ public void testSerializeValue() throws IOException { } } + @Test + public void testSerializeBatchDataReturnsSerializedSize() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260722}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary}, + { + TSDataType.VECTOR, + new TsPrimitiveType[] { + null, new TsPrimitiveType.TsInt(1), new TsPrimitiveType.TsBinary(binary) + } + } + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + BatchData batchData = new BatchData((TSDataType) testCase[0]); + batchData.putAnObject(1L, testCase[1]); + batchData.putAnObject(2L, testCase[1]); + + for (boolean isDesc : new boolean[] {false, true}) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + int serializedSize = type.serialize(batchData, new DataOutputStream(output), isDesc); + Assert.assertEquals(output.size(), serializedSize); + } + } + } + @Test public void testDeserializeValue() throws IOException { Binary binary = new Binary("test", StandardCharsets.UTF_8); From f685246605acc6485d22b61ac6ccf829cca14dae Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 22 Jul 2026 16:27:33 +0800 Subject: [PATCH 121/131] add TsPrimitiveType.copy --- .../apache/tsfile/utils/TsPrimitiveType.java | 142 ++++++++++++++++++ .../tsfile/utils/TsPrimitiveTypeTest.java | 63 ++++++++ 2 files changed, 205 insertions(+) diff --git a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java index 0bd41191e..4b20ffbab 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java @@ -115,6 +115,36 @@ public void setVector(TsPrimitiveType[] val) { public abstract TSDataType getDataType(); + public abstract void copy(TsPrimitiveType value); + + private static TsPrimitiveType copyValue(TsPrimitiveType value) { + if (value == null) { + return null; + } + if (value instanceof TsBoolean) { + return new TsBoolean((TsBoolean) value); + } + if (value instanceof TsInt) { + return new TsInt((TsInt) value); + } + if (value instanceof TsLong) { + return new TsLong((TsLong) value); + } + if (value instanceof TsFloat) { + return new TsFloat((TsFloat) value); + } + if (value instanceof TsDouble) { + return new TsDouble((TsDouble) value); + } + if (value instanceof TsBinary) { + return new TsBinary((TsBinary) value); + } + if (value instanceof TsVector) { + return new TsVector((TsVector) value); + } + throw new UnSupportedDataTypeException(String.valueOf(value.getDataType())); + } + @Override public String toString() { return getStringValue(); @@ -141,6 +171,10 @@ public TsBoolean(boolean value) { this.value = value; } + public TsBoolean(TsBoolean other) { + this.value = other.value; + } + @Override public boolean getBoolean() { return value; @@ -166,6 +200,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsBoolean", "Boolean")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setBoolean(value.getBoolean()); + } + @Override public void reset() { value = false; @@ -230,6 +273,11 @@ public TsInt(int value, TSDataType dataType) { this.dataType = dataType; } + public TsInt(TsInt other) { + this.value = other.value; + this.dataType = other.dataType; + } + @Override public int getInt() { return value; @@ -273,6 +321,16 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsInt", "Integer")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setInt(value.getInt()); + dataType = value.getDataType(); + } + @Override public void reset() { value = 0; @@ -326,6 +384,10 @@ public TsLong(long value) { this.value = value; } + public TsLong(TsLong other) { + this.value = other.value; + } + @Override public long getLong() { return value; @@ -356,6 +418,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsLong", "Long")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setLong(value.getLong()); + } + @Override public void reset() { value = 0; @@ -409,6 +480,10 @@ public TsFloat(float value) { this.value = value; } + public TsFloat(TsFloat other) { + this.value = other.value; + } + @Override public float getFloat() { return value; @@ -439,6 +514,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsFloat", "float")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setFloat(value.getFloat()); + } + @Override public void reset() { value = 0; @@ -492,6 +576,10 @@ public TsDouble(double value) { this.value = value; } + public TsDouble(TsDouble other) { + this.value = other.value; + } + @Override public double getDouble() { return value; @@ -517,6 +605,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsDouble", "Double")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setDouble(value.getDouble()); + } + @Override public void reset() { value = 0.0; @@ -570,6 +667,18 @@ public TsBinary(Binary value) { this.value = value; } + public TsBinary(TsBinary other) { + this.value = copyBinary(other.value); + } + + private static Binary copyBinary(Binary value) { + if (value == null) { + return null; + } + byte[] bytes = value.getValues(); + return new Binary(bytes == null ? null : Arrays.copyOf(bytes, bytes.length)); + } + @Override public Binary getBinary() { return value; @@ -590,6 +699,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsBinary", "Binary")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setBinary(copyBinary(value.getBinary())); + } + @Override public void reset() { value = null; @@ -643,6 +761,21 @@ public TsVector(TsPrimitiveType[] values) { this.values = values; } + public TsVector(TsVector other) { + this.values = copyValues(other.values); + } + + private static TsPrimitiveType[] copyValues(TsPrimitiveType[] values) { + if (values == null) { + return null; + } + TsPrimitiveType[] copiedValues = new TsPrimitiveType[values.length]; + for (int i = 0; i < values.length; i++) { + copiedValues[i] = copyValue(values[i]); + } + return copiedValues; + } + @Override public TsPrimitiveType[] getVector() { return values; @@ -664,6 +797,15 @@ public void setObject(Object val) { "error.common.tsprimitive_wrong_value_type", "TsVector", "TsPrimitiveType[]")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setVector(copyValues(value.getVector())); + } + @Override public void reset() { values = null; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java index cdfd3a79a..7c55269c7 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java @@ -29,6 +29,7 @@ import org.apache.tsfile.utils.TsPrimitiveType.TsFloat; import org.apache.tsfile.utils.TsPrimitiveType.TsInt; import org.apache.tsfile.utils.TsPrimitiveType.TsLong; +import org.apache.tsfile.utils.TsPrimitiveType.TsVector; import org.junit.Assert; import org.junit.Test; @@ -93,4 +94,66 @@ public void testCompareWithNullValue() { new TsPrimitiveType[] {new TsBoolean(true), null, new TsInt(1), null})); Assert.assertFalse(timeValuePair1.equals(timeValuePair2)); } + + @Test + public void testCopy() { + TsBoolean booleanValue = new TsBoolean(true); + TsBoolean booleanCopy = new TsBoolean(); + booleanCopy.copy(booleanValue); + booleanValue.setBoolean(false); + Assert.assertTrue(booleanCopy.getBoolean()); + + TsInt dateValue = new TsInt(20260722, TSDataType.DATE); + TsInt dateCopy = new TsInt(); + dateCopy.copy(dateValue); + dateValue.setInt(20260723); + Assert.assertEquals(20260722, dateCopy.getInt()); + Assert.assertEquals(TSDataType.DATE, dateCopy.getDataType()); + + TsLong longValue = new TsLong(1L); + TsLong longCopy = new TsLong(); + longCopy.copy(longValue); + longValue.setLong(2L); + Assert.assertEquals(1L, longCopy.getLong()); + + TsFloat floatValue = new TsFloat(1.0F); + TsFloat floatCopy = new TsFloat(); + floatCopy.copy(floatValue); + floatValue.setFloat(2.0F); + Assert.assertEquals(1.0F, floatCopy.getFloat(), 0.0F); + + TsDouble doubleValue = new TsDouble(1.0D); + TsDouble doubleCopy = new TsDouble(); + doubleCopy.copy(doubleValue); + doubleValue.setDouble(2.0D); + Assert.assertEquals(1.0D, doubleCopy.getDouble(), 0.0D); + + Binary binary = new Binary("original", TSFileConfig.STRING_CHARSET); + TsBinary binaryValue = new TsBinary(binary); + TsBinary binaryCopy = new TsBinary(); + binaryCopy.copy(binaryValue); + binary.getValues()[0] = 'O'; + Assert.assertEquals( + "original", binaryCopy.getBinary().getStringValue(TSFileConfig.STRING_CHARSET)); + + TsVector vectorValue = + new TsVector(new TsPrimitiveType[] {dateCopy, binaryCopy, null, new TsBoolean(true)}); + TsVector vectorCopy = new TsVector(); + vectorCopy.copy(vectorValue); + vectorValue.getVector()[0].setInt(0); + vectorValue.getVector()[1].getBinary().getValues()[0] = 'O'; + vectorValue.getVector()[3].setBoolean(false); + + Assert.assertNotSame(vectorValue.getVector(), vectorCopy.getVector()); + Assert.assertEquals(20260722, vectorCopy.getVector()[0].getInt()); + Assert.assertEquals(TSDataType.DATE, vectorCopy.getVector()[0].getDataType()); + Assert.assertEquals( + "original", + vectorCopy.getVector()[1].getBinary().getStringValue(TSFileConfig.STRING_CHARSET)); + Assert.assertNull(vectorCopy.getVector()[2]); + Assert.assertTrue(vectorCopy.getVector()[3].getBoolean()); + + booleanCopy.copy(null); + Assert.assertFalse(booleanCopy.getBoolean()); + } } From 50fdec3111ef5efdc582161508ca47b9c9f2574f Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 22 Jul 2026 17:16:20 +0800 Subject: [PATCH 122/131] add Type.deserializeArray --- .../tsfile/read/common/type/AbstractIntType.java | 10 ++++++++++ .../tsfile/read/common/type/AbstractLongType.java | 10 ++++++++++ .../apache/tsfile/read/common/type/AbstractType.java | 10 ++++++++++ .../tsfile/read/common/type/AbstractVarcharType.java | 6 ++++++ .../org/apache/tsfile/read/common/type/BlobType.java | 6 ++++++ .../apache/tsfile/read/common/type/BooleanType.java | 10 ++++++++++ .../org/apache/tsfile/read/common/type/DateType.java | 10 ++++++++++ .../org/apache/tsfile/read/common/type/DoubleType.java | 10 ++++++++++ .../org/apache/tsfile/read/common/type/FloatType.java | 10 ++++++++++ .../org/apache/tsfile/read/common/type/ObjectType.java | 6 ++++++ .../java/org/apache/tsfile/read/common/type/Type.java | 6 ++++++ 11 files changed, 94 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 54102b549..dbefd1451 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -415,6 +416,15 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + int[] values = new int[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readInt(stream); + } + return values; + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = ReadWriteIOUtils.readInt(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 319276bab..8b27fa17e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -414,6 +415,15 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + long[] values = new long[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readLong(stream); + } + return values; + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = ReadWriteIOUtils.readLong(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 10fe2cac6..2afe06758 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -30,6 +30,7 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.lang.reflect.Array; @@ -125,6 +126,15 @@ protected Binary[] deserializeBinaryValues(ByteBuffer buffer, int rowSize) { return values; } + protected Binary[] deserializeBinaryValues(DataInputStream stream, int rowSize) + throws IOException { + Binary[] values = new Binary[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = stream.readBoolean() ? ReadWriteIOUtils.readBinary(stream) : Binary.EMPTY_VALUE; + } + return values; + } + protected Binary[] deserializeBinaryValues( ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { Binary[] values = new Binary[rowSize]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index de5951e6e..ff69d8937 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -372,6 +373,11 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + return deserializeBinaryValues(stream, rowSize); + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = ReadWriteIOUtils.readBinary(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 901fbd961..9ddda4ec5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -388,6 +389,11 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + return deserializeBinaryValues(stream, rowSize); + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = ReadWriteIOUtils.readBinary(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 2fd080166..043a6db47 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -383,6 +384,15 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + boolean[] values = new boolean[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(stream)); + } + return values; + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index ed18c7a50..57a48be48 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -215,6 +216,15 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + LocalDate[] values = new LocalDate[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = DateUtils.parseIntToLocalDate(ReadWriteIOUtils.readInt(stream)); + } + return values; + } + @Override public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { return ((IntColumn) super.deserializeColumn(buffer, rowSize, nullIndicators)) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index e017f23c3..f414516d4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -424,6 +425,15 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + double[] values = new double[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readDouble(stream); + } + return values; + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = ReadWriteIOUtils.readDouble(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 28cf3b0dd..718084a87 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -422,6 +423,15 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return values; } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + float[] values = new float[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readFloat(stream); + } + return values; + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = ReadWriteIOUtils.readFloat(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 9b9ee604d..5e0f11e9c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read.common.type; +import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -378,6 +379,11 @@ public Object deserializeArray(ByteBuffer buffer, int rowSize) { return deserializeBinaryValues(buffer, rowSize); } + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + return deserializeBinaryValues(stream, rowSize); + } + @Override public void deserialize(Object[] array, int index, ByteBuffer buffer) { array[index] = ReadWriteIOUtils.readBinary(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index fad62e1b2..d5b1d7bb5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -43,6 +43,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; @@ -256,6 +257,11 @@ default Object deserializeArray(ByteBuffer buffer, int rowSize) { throw new UnsupportedOperationException(getClass().getName()); } + /** Deserializes an array with {@code rowSize} entries from {@code stream}. */ + default Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Deserializes one value from {@code buffer} into {@code array} at {@code index}. */ default void deserialize(Object[] array, int index, ByteBuffer buffer) { throw new UnsupportedOperationException(getClass().getName()); From 5f650864e30a9037f305979a8990c095be44d3e4 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 22 Jul 2026 17:19:03 +0800 Subject: [PATCH 123/131] spotless --- .../org/apache/tsfile/read/common/type/AbstractIntType.java | 2 +- .../org/apache/tsfile/read/common/type/AbstractLongType.java | 2 +- .../org/apache/tsfile/read/common/type/AbstractVarcharType.java | 2 +- .../main/java/org/apache/tsfile/read/common/type/BlobType.java | 2 +- .../java/org/apache/tsfile/read/common/type/BooleanType.java | 2 +- .../main/java/org/apache/tsfile/read/common/type/DateType.java | 2 +- .../java/org/apache/tsfile/read/common/type/DoubleType.java | 2 +- .../main/java/org/apache/tsfile/read/common/type/FloatType.java | 2 +- .../java/org/apache/tsfile/read/common/type/ObjectType.java | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index dbefd1451..d1c632257 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -55,6 +54,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 8b27fa17e..53f87f7c8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -54,6 +53,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index ff69d8937..d40f4fe3f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -46,6 +45,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 9ddda4ec5..c1e1bc88a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -48,6 +47,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 043a6db47..ac0044c36 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -49,6 +48,7 @@ import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; import org.apache.tsfile.write.record.datapoint.DataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 57a48be48..a90bf5793 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -39,6 +38,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index f414516d4..40a9b8e1f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -54,6 +53,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 718084a87..d16d237d9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -53,6 +52,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.FloatDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 5e0f11e9c..b30039236 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.common.type; -import java.io.DataInputStream; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnBuilderStatus; @@ -48,6 +47,7 @@ import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; From bab44db47169199bac9f30c2539ad4260a55b74e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 23 Jul 2026 11:06:17 +0800 Subject: [PATCH 124/131] add Column.writeTo --- .../apache/tsfile/block/column/Column.java | 11 ++++ .../common/block/column/BinaryColumn.java | 15 +++++ .../common/block/column/BooleanColumn.java | 11 ++++ .../common/block/column/DictionaryColumn.java | 13 +++++ .../common/block/column/DoubleColumn.java | 11 ++++ .../read/common/block/column/FloatColumn.java | 11 ++++ .../read/common/block/column/IntColumn.java | 11 ++++ .../read/common/block/column/LongColumn.java | 11 ++++ .../read/common/block/column/NullColumn.java | 11 ++++ .../block/column/RunLengthEncodedColumn.java | 13 +++++ .../read/common/block/column/TimeColumn.java | 11 ++++ .../apache/tsfile/read/common/ColumnTest.java | 58 +++++++++++++++++++ 12 files changed, 187 insertions(+) diff --git a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java index 9b7c88edd..445396f53 100644 --- a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java +++ b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java @@ -25,6 +25,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; public interface Column { @@ -110,6 +111,16 @@ default TsPrimitiveType getTsPrimitiveType(int position) { throw new UnsupportedOperationException(getClass().getName()); } + /** Serializes the value at {@code index} to {@code buffer}. */ + default void writeTo(int index, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Serializes the value at {@code index} to {@code stream}. */ + default void writeTo(int index, DataOutputStream stream) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Serializes all non-null values in this column. */ default void serializeWithoutNulls(DataOutputStream output) throws IOException { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index a11885f7a..07a2b58db 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -29,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Optional; @@ -145,6 +146,20 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBinary(getBinary(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + Binary value = values[index + arrayOffset]; + buffer.putInt(value.getLength()); + buffer.put(value.getValues()); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + Binary value = values[index + arrayOffset]; + stream.writeInt(value.getLength()); + stream.write(value.getValues()); + } + @Override public void serializeWithoutNulls(DataOutputStream output) throws IOException { for (int i = 0; i < positionCount; i++) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java index ac0c0dac1..7d30c6a00 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java @@ -29,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -127,6 +128,16 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBoolean(getBoolean(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.put(values[index + arrayOffset] ? (byte) 1 : (byte) 0); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeBoolean(values[index + arrayOffset]); + } + @Override public void serializeWithoutNulls(DataOutputStream output) throws IOException { int nonNullCount = 0; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java index e26f42c09..1e2f7067b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java @@ -27,6 +27,9 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -427,6 +430,16 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return dictionary.getTsPrimitiveType(position); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + dictionary.writeTo(getId(index), buffer); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + dictionary.writeTo(getId(index), stream); + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { return dictionary.arePositionsEqual(getId(thisPos), that, thatPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java index 366a99d42..81e19a307 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java @@ -29,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -128,6 +129,16 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsDouble(getDouble(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putDouble(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeDouble(values[index + arrayOffset]); + } + @Override public void serializeWithoutNulls(DataOutputStream output) throws IOException { for (int i = 0; i < positionCount; i++) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java index 35b110cb8..56ce7a1f8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java @@ -29,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -143,6 +144,16 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsFloat(getFloat(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putFloat(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeFloat(values[index + arrayOffset]); + } + @Override public void serializeWithoutNulls(DataOutputStream output) throws IOException { for (int i = 0; i < positionCount; i++) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index 10e6179d3..59d3a7505 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -29,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -199,6 +200,16 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsInt(getInt(position), dataType); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putInt(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeInt(values[index + arrayOffset]); + } + @Override public void serializeWithoutNulls(DataOutputStream output) throws IOException { for (int i = 0; i < positionCount; i++) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 8f089cdf2..2b6bc46fc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -29,6 +29,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -143,6 +144,16 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsLong(getLong(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putLong(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeLong(values[index + arrayOffset]); + } + @Override public void serializeWithoutNulls(DataOutputStream output) throws IOException { for (int i = 0; i < positionCount; i++) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index 06d720275..0d0145295 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -27,6 +27,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.io.DataOutputStream; +import java.nio.ByteBuffer; import static java.util.Objects.requireNonNull; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkArrayRange; @@ -89,6 +90,16 @@ public void serializeWithoutNulls(DataOutputStream output) { // There are no non-null values to serialize. } + @Override + public void writeTo(int index, ByteBuffer buffer) { + // There is no value to serialize. + } + + @Override + public void writeTo(int index, DataOutputStream stream) { + // There is no value to serialize. + } + @Override public int getPositionCount() { return positionCount; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java index 548cfab81..30c3af73e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java @@ -27,6 +27,9 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import static java.util.Objects.requireNonNull; @@ -165,6 +168,16 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return value.getTsPrimitiveType(0); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + value.writeTo(0, buffer); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + value.writeTo(0, stream); + } + @Override public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { return value.arePositionsEqual(0, that, thatPos); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java index 4156482af..c2a5e7427 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkArrayRange; @@ -92,6 +93,16 @@ public Object getObject(int position) { return getLong(position); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putLong(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeLong(values[index + arrayOffset]); + } + @Override public void serializeWithoutNulls(DataOutputStream output) throws IOException { for (int i = 0; i < positionCount; i++) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index 6d6fe91a1..5e71a89c3 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -38,17 +38,75 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Optional; public class ColumnTest { + @Test + public void testWriteTo() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Column intDictionary = + DictionaryColumn.create( + 2, new IntColumn(3, Optional.empty(), new int[] {7, 8, 9}), new int[] {2, 0}); + Column binaryRle = + new RunLengthEncodedColumn(new BinaryColumn(1, Optional.empty(), new Binary[] {binary}), 3); + + Object[][] testCases = { + {new BooleanColumn(2, Optional.empty(), new boolean[] {false, true}).getRegion(1, 1), 0}, + {new IntColumn(2, Optional.empty(), new int[] {0, 42}).getRegion(1, 1), 0}, + {new LongColumn(2, Optional.empty(), new long[] {0L, 42L}).getRegion(1, 1), 0}, + {new FloatColumn(2, Optional.empty(), new float[] {0.0F, 1.25F}).getRegion(1, 1), 0}, + {new DoubleColumn(2, Optional.empty(), new double[] {0.0D, 2.5D}).getRegion(1, 1), 0}, + { + new BinaryColumn(2, Optional.empty(), new Binary[] {Binary.EMPTY_VALUE, binary}) + .getRegion(1, 1), + 0 + }, + {new TimeColumn(2, new long[] {0L, 42L}).getRegion(1, 1), 0}, + {intDictionary, 0}, + {binaryRle, 2} + }; + + for (Object[] testCase : testCases) { + Column column = (Column) testCase[0]; + int index = (int) testCase[1]; + ByteArrayOutputStream expectedOutput = new ByteArrayOutputStream(); + Type.fromTsDataType(column.getDataType()) + .serializeValue(column.getObject(index), new DataOutputStream(expectedOutput)); + byte[] expected = expectedOutput.toByteArray(); + + ByteBuffer buffer = ByteBuffer.allocate(expected.length); + column.writeTo(index, buffer); + Assert.assertEquals(expected.length, buffer.position()); + Assert.assertArrayEquals(expected, Arrays.copyOf(buffer.array(), buffer.position())); + + ByteArrayOutputStream actualOutput = new ByteArrayOutputStream(); + column.writeTo(index, new DataOutputStream(actualOutput)); + Assert.assertArrayEquals(expected, actualOutput.toByteArray()); + } + + Column nullColumn = new NullColumn(1); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + nullColumn.writeTo(0, emptyBuffer); + ByteArrayOutputStream emptyOutput = new ByteArrayOutputStream(); + nullColumn.writeTo(0, new DataOutputStream(emptyOutput)); + Assert.assertEquals(0, emptyBuffer.position()); + Assert.assertEquals(0, emptyOutput.size()); + } + @Test public void testArePositionsEqual() { IntColumnBuilder builder = new IntColumnBuilder(null, 5); From fc7cae41c5e8e8137004d28e771d5dff60b21232 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 23 Jul 2026 15:43:52 +0800 Subject: [PATCH 125/131] add Type.write --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 5 +++ .../tsfile/read/common/type/BlobType.java | 5 +++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DateType.java | 9 ++++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 5 +++ .../tsfile/read/common/type/RowType.java | 5 +++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/VectorType.java | 5 +++ .../org/apache/tsfile/utils/TypeServices.java | 6 +++ .../tsfile/read/common/type/TypeTest.java | 44 +++++++++++++++++++ 14 files changed, 114 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index d1c632257..97c7d0f1a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -334,6 +334,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeInt(value.getInt()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((int[]) array)[index] = value.getInt(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeInt(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 53f87f7c8..25da498b5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -333,6 +333,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeLong(value.getLong()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((long[]) array)[index] = value.getLong(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeLong(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index d40f4fe3f..ebee3bdc3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -295,6 +295,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((Binary[]) array)[index] = value.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index c1e1bc88a..040bdc094 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -313,6 +313,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((Binary[]) array)[index] = value.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index ac0044c36..32fe1c037 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -317,6 +317,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBoolean(value.getBoolean()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((boolean[]) array)[index] = value.getBoolean(); + } + @Override public void writeBoolean(ColumnBuilder builder, boolean value) { builder.writeBoolean(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index a90bf5793..1334f2f38 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -68,6 +68,15 @@ public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { return new TsInt(getDateInt(array, rowIndex), TSDataType.DATE); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + if (array instanceof int[]) { + super.write(value, array, index); + return; + } + ((LocalDate[]) array)[index] = DateUtils.parseIntToLocalDate(value.getInt()); + } + @Override public DataPoint getDataPoint(String measurementId, String value) { return new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 40a9b8e1f..f9093b06c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -343,6 +343,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeDouble(value.getDouble()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((double[]) array)[index] = value.getDouble(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeDouble(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index d16d237d9..8d1457e05 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -341,6 +341,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeFloat(value.getFloat()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((float[]) array)[index] = value.getFloat(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeFloat(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index b30039236..fa625dc39 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -303,6 +303,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((Binary[]) array)[index] = value.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java index c9339afd8..5fc6faafa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java @@ -89,6 +89,11 @@ public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { return new TsPrimitiveType.TsVector(((TsPrimitiveType[][]) array)[rowIndex]); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((TsPrimitiveType[][]) array)[index] = value.getVector(); + } + public static Field field(String name, Type type) { return new Field(Optional.of(name), type); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index d5b1d7bb5..216a263d2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -161,6 +161,11 @@ default void write(ColumnBuilder builder, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } + /** Writes a TsPrimitiveType value to {@code array} at {@code index}. */ + default void write(TsPrimitiveType value, Object array, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index d72370ff1..fb783f630 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -88,6 +88,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } + @Override + public void write(TsPrimitiveType value, Object array, int index) { + ((TsPrimitiveType[][]) array)[index] = value.getVector(); + } + @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((long[]) array, newLength); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java index 18f644bba..34c5c16d6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -37,6 +37,8 @@ public final class TypeServices { + // Page value decoding services for BatchData and TsBlock outputs. + public static final TypeService READ_PAGE_VALUE_TO_BATCHDATA_SERVICE = type -> switch (type.getTypeEnum()) { @@ -286,6 +288,8 @@ public final class TypeServices { }; }; + // Value page decoding services for TsPrimitiveType and ColumnBuilder outputs. + public static final TypeService READ_PAGE_VALUE_TO_TSPRIMITIVETYPE_SERVICE = type -> @@ -426,6 +430,8 @@ public final class TypeServices { }; }; + // Chunk writing services. + public static final TypeService WRITE_EMPTY_VALUE_TO_CHUNK_SERVICE = type -> switch (type.getTypeEnum()) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 9f09057fd..aa323eb9e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -107,6 +107,50 @@ public void testWriteTsPrimitiveTypeToColumnBuilder() { } } + @Test + public void testWriteTsPrimitiveTypeToArray() { + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType[] vector = {new TsPrimitiveType.TsInt(1), new TsPrimitiveType.TsLong(2L)}; + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true), new boolean[2]}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1), new int[2]}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260723, TSDataType.DATE), new int[2]}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260723, TSDataType.DATE), new LocalDate[2]}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L), new long[2]}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L), new long[2]}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F), new float[2]}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D), new double[2]}, + {TSDataType.TEXT, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.STRING, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.BLOB, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.OBJECT, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.VECTOR, new TsPrimitiveType.TsVector(vector), new TsPrimitiveType[2][]} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + TsPrimitiveType expected = (TsPrimitiveType) testCase[1]; + Object array = testCase[2]; + type.write(expected, array, 1); + TsPrimitiveType actual = type.getValueAsTsPrimitiveType(array, 1); + Assert.assertEquals(expected, actual); + Assert.assertEquals(expected.getDataType(), actual.getDataType()); + } + + Type rowType = RowType.anonymousRow(IntType.getInstance(), LongType.getInstance()); + TsPrimitiveType rowValue = new TsPrimitiveType.TsVector(vector); + TsPrimitiveType[][] rowArray = new TsPrimitiveType[2][]; + rowType.write(rowValue, rowArray, 1); + Assert.assertEquals(rowValue, rowType.getValueAsTsPrimitiveType(rowArray, 1)); + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).write(new TsPrimitiveType.TsInt(1), new int[1], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + @Test public void testGetValueAsTsPrimitiveType() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From cc0ba0175f5f9dc7c04b63d686ac29789cf0ecfd Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 23 Jul 2026 15:51:45 +0800 Subject: [PATCH 126/131] add Type.write --- .../apache/tsfile/read/common/type/AbstractIntType.java | 4 ++-- .../apache/tsfile/read/common/type/AbstractLongType.java | 4 ++-- .../tsfile/read/common/type/AbstractVarcharType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/BlobType.java | 4 ++-- .../org/apache/tsfile/read/common/type/BooleanType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/DateType.java | 8 ++++---- .../org/apache/tsfile/read/common/type/DoubleType.java | 4 ++-- .../org/apache/tsfile/read/common/type/FloatType.java | 4 ++-- .../org/apache/tsfile/read/common/type/ObjectType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/RowType.java | 4 ++-- .../java/org/apache/tsfile/read/common/type/Type.java | 2 +- .../org/apache/tsfile/read/common/type/VectorType.java | 4 ++-- 12 files changed, 25 insertions(+), 25 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 97c7d0f1a..790337053 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -335,8 +335,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((int[]) array)[index] = value.getInt(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((int[]) toArray)[index] = from.getInt(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 25da498b5..350244782 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -334,8 +334,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((long[]) array)[index] = value.getLong(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((long[]) toArray)[index] = from.getLong(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index ebee3bdc3..26620691f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -296,8 +296,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((Binary[]) array)[index] = value.getBinary(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 040bdc094..373364602 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -314,8 +314,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((Binary[]) array)[index] = value.getBinary(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 32fe1c037..db8b29162 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -318,8 +318,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((boolean[]) array)[index] = value.getBoolean(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((boolean[]) toArray)[index] = from.getBoolean(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 1334f2f38..d823578cc 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -69,12 +69,12 @@ public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - if (array instanceof int[]) { - super.write(value, array, index); + public void write(TsPrimitiveType from, Object toArray, int index) { + if (toArray instanceof int[]) { + super.write(from, toArray, index); return; } - ((LocalDate[]) array)[index] = DateUtils.parseIntToLocalDate(value.getInt()); + ((LocalDate[]) toArray)[index] = DateUtils.parseIntToLocalDate(from.getInt()); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index f9093b06c..802e9b0a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -344,8 +344,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((double[]) array)[index] = value.getDouble(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((double[]) toArray)[index] = from.getDouble(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 8d1457e05..fce2a6a82 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -342,8 +342,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((float[]) array)[index] = value.getFloat(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((float[]) toArray)[index] = from.getFloat(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index fa625dc39..cc6fb5197 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -304,8 +304,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((Binary[]) array)[index] = value.getBinary(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java index 5fc6faafa..7b84f7a67 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java @@ -90,8 +90,8 @@ public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((TsPrimitiveType[][]) array)[index] = value.getVector(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); } public static Field field(String name, Type type) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 216a263d2..69af1231a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -162,7 +162,7 @@ default void write(ColumnBuilder builder, TsPrimitiveType value) { } /** Writes a TsPrimitiveType value to {@code array} at {@code index}. */ - default void write(TsPrimitiveType value, Object array, int index) { + default void write(TsPrimitiveType from, Object toArray, int index) { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index fb783f630..aab959c9c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -89,8 +89,8 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { } @Override - public void write(TsPrimitiveType value, Object array, int index) { - ((TsPrimitiveType[][]) array)[index] = value.getVector(); + public void write(TsPrimitiveType from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); } @Override From 1eb933719da13eb2b3fd850ad7592df2a3bd4e02 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 23 Jul 2026 16:13:33 +0800 Subject: [PATCH 127/131] add Type.write --- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DateType.java | 10 ++++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../tsfile/read/common/type/RowType.java | 6 +++ .../apache/tsfile/read/common/type/Type.java | 5 ++ .../tsfile/read/common/type/VectorType.java | 5 ++ .../tsfile/read/common/type/TypeTest.java | 48 +++++++++++++++++++ 13 files changed, 114 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 790337053..902964584 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -339,6 +339,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((int[]) toArray)[index] = from.getInt(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((int[]) toArray)[index] = from.getInt(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeInt(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 350244782..41bff77b0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -338,6 +338,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((long[]) toArray)[index] = from.getLong(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((long[]) toArray)[index] = from.getLong(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeLong(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 26620691f..fdd2f6bae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -300,6 +300,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((Binary[]) toArray)[index] = from.getBinary(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 373364602..53e87e86e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -318,6 +318,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((Binary[]) toArray)[index] = from.getBinary(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index db8b29162..e747ded74 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -322,6 +322,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((boolean[]) toArray)[index] = from.getBoolean(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((boolean[]) toArray)[index] = from.getBoolean(); + } + @Override public void writeBoolean(ColumnBuilder builder, boolean value) { builder.writeBoolean(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index d823578cc..c231a6f9b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -25,6 +25,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.statistics.DateStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; @@ -77,6 +78,15 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((LocalDate[]) toArray)[index] = DateUtils.parseIntToLocalDate(from.getInt()); } + @Override + public void write(BatchData from, Object toArray, int index) { + if (toArray instanceof int[]) { + super.write(from, toArray, index); + return; + } + ((LocalDate[]) toArray)[index] = DateUtils.parseIntToLocalDate(from.getInt()); + } + @Override public DataPoint getDataPoint(String measurementId, String value) { return new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 802e9b0a4..005a25c3f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -348,6 +348,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((double[]) toArray)[index] = from.getDouble(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((double[]) toArray)[index] = from.getDouble(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeDouble(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index fce2a6a82..b910614be 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -346,6 +346,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((float[]) toArray)[index] = from.getFloat(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((float[]) toArray)[index] = from.getFloat(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeFloat(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index cc6fb5197..94e20bc8a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -308,6 +308,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((Binary[]) toArray)[index] = from.getBinary(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java index 7b84f7a67..6123fb30b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.utils.TsPrimitiveType; import java.util.ArrayList; @@ -94,6 +95,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); + } + public static Field field(String name, Type type) { return new Field(Optional.of(name), type); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 69af1231a..a29094573 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -166,6 +166,11 @@ default void write(TsPrimitiveType from, Object toArray, int index) { throw new UnsupportedOperationException(getClass().getName()); } + /** Writes the current value from {@code from} to {@code toArray} at {@code index}. */ + default void write(BatchData from, Object toArray, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index aab959c9c..afdfca2c3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -93,6 +93,11 @@ public void write(TsPrimitiveType from, Object toArray, int index) { ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); } + @Override + public void write(BatchData from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); + } + @Override public Object arrayCopyOf(Object array, int newLength) { return Arrays.copyOf((long[]) array, newLength); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index aa323eb9e..a11d58960 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -151,6 +151,54 @@ public void testWriteTsPrimitiveTypeToArray() { } } + @Test + public void testWriteBatchDataToArray() { + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType[] vector = {new TsPrimitiveType.TsInt(1), new TsPrimitiveType.TsLong(2L)}; + Object[][] testCases = { + {TSDataType.BOOLEAN, true, new boolean[2]}, + {TSDataType.INT32, 1, new int[2]}, + {TSDataType.DATE, 20260723, new int[2]}, + {TSDataType.DATE, 20260723, new LocalDate[2]}, + {TSDataType.INT64, 2L, new long[2]}, + {TSDataType.TIMESTAMP, 3L, new long[2]}, + {TSDataType.FLOAT, 1.25F, new float[2]}, + {TSDataType.DOUBLE, 2.5D, new double[2]}, + {TSDataType.TEXT, binary, new Binary[2]}, + {TSDataType.STRING, binary, new Binary[2]}, + {TSDataType.BLOB, binary, new Binary[2]}, + {TSDataType.OBJECT, binary, new Binary[2]}, + {TSDataType.VECTOR, vector, new TsPrimitiveType[2][]} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Type type = Type.fromTsDataType(dataType); + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(1L, testCase[1]); + Object array = testCase[2]; + type.write(batchData, array, 1); + TsPrimitiveType actual = type.getValueAsTsPrimitiveType(array, 1); + Assert.assertEquals(batchData.currentTsPrimitiveType(), actual); + Assert.assertEquals(batchData.currentTsPrimitiveType().getDataType(), actual.getDataType()); + } + + Type rowType = RowType.anonymousRow(IntType.getInstance(), LongType.getInstance()); + BatchData vectorBatchData = new BatchData(TSDataType.VECTOR); + vectorBatchData.putAnObject(1L, vector); + TsPrimitiveType[][] rowArray = new TsPrimitiveType[2][]; + rowType.write(vectorBatchData, rowArray, 1); + Assert.assertEquals( + vectorBatchData.currentTsPrimitiveType(), rowType.getValueAsTsPrimitiveType(rowArray, 1)); + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).write((BatchData) null, new Object[1], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + @Test public void testGetValueAsTsPrimitiveType() { Binary binary = new Binary("test", StandardCharsets.UTF_8); From 2f4ac1389167e0a84b0ced5437461fbe617a3d25 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 23 Jul 2026 16:18:54 +0800 Subject: [PATCH 128/131] add Type.write --- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 6 +++ .../tsfile/read/common/type/VectorType.java | 6 +++ .../tsfile/read/common/type/TypeTest.java | 51 +++++++++++++++++++ 11 files changed, 103 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 902964584..8294940df 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -289,6 +289,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getInt()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getInt()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (int) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 41bff77b0..5ae887095 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -288,6 +288,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getLong()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getLong()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (long) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index fdd2f6bae..1c7f279ce 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -265,6 +265,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getBinary()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBinary()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (Binary) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 53e87e86e..480ba6854 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -276,6 +276,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getBinary()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBinary()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (Binary) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index e747ded74..d4161fde4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -280,6 +280,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getBoolean()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBoolean()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (boolean) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 005a25c3f..04c446f10 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -291,6 +291,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getDouble()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getDouble()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (double) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index b910614be..bae44ca67 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -289,6 +289,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getFloat()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getFloat()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (float) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 94e20bc8a..f43db6605 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -266,6 +266,11 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { writer.write(time, value.getBinary()); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBinary()); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { writer.write(time, (Binary) value); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index a29094573..0b8483279 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -428,6 +428,12 @@ default void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } + /** Writes the current value in {@code data} to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, BatchData data) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + /** Writes a value to a non-aligned chunk. */ default void write(ChunkWriterImpl writer, long time, Object value) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index afdfca2c3..ea99ee9b6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -146,6 +146,12 @@ public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { Messages.format("error.write.type_not_supported", getTypeEnum())); } + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + @Override public void write(ChunkWriterImpl writer, long time, Object value) { throw new UnSupportedDataTypeException( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index a11d58960..0703d6c85 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -1028,6 +1028,57 @@ public void testWriteTsPrimitiveTypeToNonAlignedValueChunk() { } } + @Test + public void testWriteBatchDataToNonAlignedValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1L, true}, + {TSDataType.INT32, 2L, 1}, + {TSDataType.DATE, 3L, 20260717}, + {TSDataType.INT64, 4L, 2L}, + {TSDataType.TIMESTAMP, 5L, 3L}, + {TSDataType.FLOAT, 6L, 1.25F}, + {TSDataType.DOUBLE, 7L, 2.5D} + }; + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + BatchData data = new BatchData(dataType); + data.putAnObject(0L, testCase[2]); + Type.fromTsDataType(dataType).write(writer, (long) testCase[1], data); + } + + Mockito.verify(writer).write(1L, true); + Mockito.verify(writer).write(2L, 1); + Mockito.verify(writer).write(3L, 20260717); + Mockito.verify(writer).write(4L, 2L); + Mockito.verify(writer).write(5L, 3L); + Mockito.verify(writer).write(6L, 1.25F); + Mockito.verify(writer).write(7L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + BatchData data = new BatchData(dataType); + data.putAnObject(0L, binary); + Type.fromTsDataType(dataType).write(writer, 8L, data); + Mockito.verify(writer).write(8L, binary); + Mockito.reset(writer); + } + + BatchData vectorData = new BatchData(TSDataType.VECTOR); + vectorData.putAnObject(0L, new TsPrimitiveType[] {new TsPrimitiveType.TsLong(1L)}); + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(writer, 9L, vectorData); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + @Test public void testToString() { Object[][] testCases = { From 6c8f2420e866517efa0542fd865c5c02d3a2419c Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 29 Jul 2026 10:33:12 +0800 Subject: [PATCH 129/131] add Type.write --- .../read/common/type/AbstractIntType.java | 5 ++ .../read/common/type/AbstractLongType.java | 5 ++ .../read/common/type/AbstractVarcharType.java | 5 ++ .../tsfile/read/common/type/BlobType.java | 5 ++ .../tsfile/read/common/type/BooleanType.java | 5 ++ .../tsfile/read/common/type/DoubleType.java | 5 ++ .../tsfile/read/common/type/FloatType.java | 5 ++ .../tsfile/read/common/type/ObjectType.java | 5 ++ .../apache/tsfile/read/common/type/Type.java | 5 ++ .../tsfile/read/common/type/VectorType.java | 5 ++ .../tsfile/read/common/type/TypeTest.java | 54 +++++++++++++++++++ 11 files changed, 104 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 8294940df..765285569 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -339,6 +339,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeInt(value.getInt()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeInt(column.getInt(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((int[]) toArray)[index] = from.getInt(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 5ae887095..48f1652f0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -338,6 +338,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeLong(value.getLong()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeLong(column.getLong(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((long[]) toArray)[index] = from.getLong(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 1c7f279ce..058686a69 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -300,6 +300,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBinary(column.getBinary(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((Binary[]) toArray)[index] = from.getBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 480ba6854..3ab2a1e0d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -318,6 +318,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBinary(column.getBinary(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((Binary[]) toArray)[index] = from.getBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index d4161fde4..17a69bc00 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -322,6 +322,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBoolean(value.getBoolean()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBoolean(column.getBoolean(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((boolean[]) toArray)[index] = from.getBoolean(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 04c446f10..0f19a9145 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -348,6 +348,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeDouble(value.getDouble()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeDouble(column.getDouble(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((double[]) toArray)[index] = from.getDouble(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index bae44ca67..636838641 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -346,6 +346,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeFloat(value.getFloat()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeFloat(column.getFloat(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((float[]) toArray)[index] = from.getFloat(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index f43db6605..40ea9b3f7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -308,6 +308,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBinary(column.getBinary(index)); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((Binary[]) toArray)[index] = from.getBinary(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 0b8483279..3fd425d08 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -161,6 +161,11 @@ default void write(ColumnBuilder builder, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } + /** Writes the value at {@code index} in {@code column} to the current entry. */ + default void write(ColumnBuilder builder, Column column, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Writes a TsPrimitiveType value to {@code array} at {@code index}. */ default void write(TsPrimitiveType from, Object toArray, int index) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index ea99ee9b6..90e84b151 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -88,6 +88,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } + @Override + public void write(ColumnBuilder builder, Column column, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + @Override public void write(TsPrimitiveType from, Object toArray, int index) { ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index 0703d6c85..e7562794e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -107,6 +107,60 @@ public void testWriteTsPrimitiveTypeToColumnBuilder() { } } + @Test + public void testWriteColumnValueToColumnBuilder() { + ColumnBuilder builder = Mockito.mock(ColumnBuilder.class); + Column column = Mockito.mock(Column.class); + Binary binary = new Binary("value", StandardCharsets.UTF_8); + + Mockito.when(column.getBoolean(0)).thenReturn(true); + Mockito.when(column.getInt(1)).thenReturn(1); + Mockito.when(column.getInt(2)).thenReturn(20260729); + Mockito.when(column.getLong(3)).thenReturn(2L); + Mockito.when(column.getLong(4)).thenReturn(3L); + Mockito.when(column.getFloat(5)).thenReturn(1.25F); + Mockito.when(column.getDouble(6)).thenReturn(2.5D); + Mockito.when(column.getBinary(7)).thenReturn(binary); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 0}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 2}, + {TSDataType.INT64, 3}, + {TSDataType.TIMESTAMP, 4}, + {TSDataType.FLOAT, 5}, + {TSDataType.DOUBLE, 6} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]).write(builder, column, (int) testCase[1]); + } + + Mockito.verify(builder).writeBoolean(true); + Mockito.verify(builder).writeInt(1); + Mockito.verify(builder).writeInt(20260729); + Mockito.verify(builder).writeLong(2L); + Mockito.verify(builder).writeLong(3L); + Mockito.verify(builder).writeFloat(1.25F); + Mockito.verify(builder).writeDouble(2.5D); + Mockito.reset(builder); + + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(builder, column, 7); + Mockito.verify(builder).writeBinary(binary); + Mockito.reset(builder); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(builder, column, 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + @Test public void testWriteTsPrimitiveTypeToArray() { Binary binary = new Binary("value", StandardCharsets.UTF_8); From 59d0ae12a1471de071be981c89840caa44b5f9f4 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 30 Jul 2026 18:49:08 +0800 Subject: [PATCH 130/131] add Type.write --- .../read/common/type/AbstractIntType.java | 5 +++ .../read/common/type/AbstractLongType.java | 5 +++ .../read/common/type/AbstractVarcharType.java | 9 ++++ .../tsfile/read/common/type/BlobType.java | 8 ++++ .../tsfile/read/common/type/BooleanType.java | 5 +++ .../tsfile/read/common/type/DoubleType.java | 5 +++ .../tsfile/read/common/type/FloatType.java | 5 +++ .../tsfile/read/common/type/ObjectType.java | 9 ++++ .../apache/tsfile/read/common/type/Type.java | 5 +++ .../tsfile/read/common/type/VectorType.java | 5 +++ .../tsfile/read/common/type/TypeTest.java | 45 +++++++++++++++++++ 11 files changed, 106 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 765285569..3d29a5b50 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -339,6 +339,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeInt(value.getInt()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeInt(BytesUtils.bytesToInt(bytes, offset)); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeInt(column.getInt(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index 48f1652f0..6555b3962 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -338,6 +338,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeLong(value.getLong()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeLong(BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset)); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeLong(column.getLong(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index 058686a69..520f16444 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -36,6 +36,7 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -300,6 +301,14 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + int length = BytesUtils.bytesToInt(bytes, offset); + builder.writeBinary( + new Binary( + Arrays.copyOfRange(bytes, offset + Integer.BYTES, offset + Integer.BYTES + length))); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeBinary(column.getBinary(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index 3ab2a1e0d..c0c1e8b81 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -318,6 +318,14 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + int length = BytesUtils.bytesToInt(bytes, offset); + builder.writeBinary( + new Binary( + Arrays.copyOfRange(bytes, offset + Integer.BYTES, offset + Integer.BYTES + length))); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeBinary(column.getBinary(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 17a69bc00..55370db6e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -322,6 +322,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBoolean(value.getBoolean()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeBoolean(BytesUtils.bytesToBool(bytes, offset)); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeBoolean(column.getBoolean(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 0f19a9145..f847fe68c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -348,6 +348,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeDouble(value.getDouble()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeDouble(BytesUtils.bytesToDouble(bytes, offset)); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeDouble(column.getDouble(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 636838641..635ba53ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -346,6 +346,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeFloat(value.getFloat()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeFloat(BytesUtils.bytesToFloat(bytes, offset)); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeFloat(column.getFloat(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index 40ea9b3f7..1025d55da 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -37,6 +37,7 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; @@ -308,6 +309,14 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { builder.writeBinary(value.getBinary()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + int length = BytesUtils.bytesToInt(bytes, offset); + builder.writeBinary( + new Binary( + Arrays.copyOfRange(bytes, offset + Integer.BYTES, offset + Integer.BYTES + length))); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { builder.writeBinary(column.getBinary(index)); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 3fd425d08..d7be3dbab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -161,6 +161,11 @@ default void write(ColumnBuilder builder, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } + /** Writes the value serialized in {@code bytes} at {@code offset} to the current entry. */ + default void write(ColumnBuilder builder, byte[] bytes, int offset) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Writes the value at {@code index} in {@code column} to the current entry. */ default void write(ColumnBuilder builder, Column column, int index) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java index 90e84b151..680900af4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -88,6 +88,11 @@ public void write(ColumnBuilder builder, TsPrimitiveType value) { throw new UnsupportedOperationException(getClass().getName()); } + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + throw new UnsupportedOperationException(getClass().getName()); + } + @Override public void write(ColumnBuilder builder, Column column, int index) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java index e7562794e..6f36ec3a9 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -107,6 +107,51 @@ public void testWriteTsPrimitiveTypeToColumnBuilder() { } } + @Test + public void testWriteBytesToColumnBuilder() { + ColumnBuilder builder = Mockito.mock(ColumnBuilder.class); + Binary binary = new Binary("value", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260721, TSDataType.DATE)}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L)}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L)}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D)}, + {TSDataType.TEXT, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.STRING, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.BLOB, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.OBJECT, new TsPrimitiveType.TsBinary(binary)} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + TsPrimitiveType value = (TsPrimitiveType) testCase[1]; + byte[] bytes = new byte[OFFSET + type.calcTypeSize(value)]; + type.toBytes(value, bytes, OFFSET); + type.write(builder, bytes, OFFSET); + } + + Mockito.verify(builder).writeBoolean(true); + Mockito.verify(builder).writeInt(1); + Mockito.verify(builder).writeInt(20260721); + Mockito.verify(builder).writeLong(2L); + Mockito.verify(builder).writeLong(3L); + Mockito.verify(builder).writeFloat(1.25F); + Mockito.verify(builder).writeDouble(2.5D); + Mockito.verify(builder, Mockito.times(4)).writeBinary(binary); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(builder, new byte[Long.BYTES], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + @Test public void testWriteColumnValueToColumnBuilder() { ColumnBuilder builder = Mockito.mock(ColumnBuilder.class); From 355fdf1898ec2544a184a0ccedcdc237174984e0 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 12 Aug 2026 15:34:42 +0800 Subject: [PATCH 131/131] add Column.convertTo --- .../apache/tsfile/block/column/Column.java | 3 + .../common/block/column/BinaryColumn.java | 20 +++ .../common/block/column/BooleanColumn.java | 20 +++ .../read/common/block/column/ColumnUtil.java | 9 ++ .../common/block/column/DictionaryColumn.java | 18 +++ .../common/block/column/DoubleColumn.java | 20 +++ .../read/common/block/column/FloatColumn.java | 20 +++ .../read/common/block/column/IntColumn.java | 24 +++ .../read/common/block/column/LongColumn.java | 20 +++ .../read/common/block/column/NullColumn.java | 5 + .../block/column/RunLengthEncodedColumn.java | 11 ++ .../read/common/block/column/TimeColumn.java | 16 ++ .../apache/tsfile/read/common/ColumnTest.java | 152 ++++++++++++++++++ 13 files changed, 338 insertions(+) diff --git a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java index 445396f53..7cf721bed 100644 --- a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java +++ b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java @@ -36,6 +36,9 @@ public interface Column { /** Get the encoding for this column. */ ColumnEncoding getEncoding(); + /** Converts this column to the specified data type. */ + Column convertTo(TSDataType type); + /** Gets a boolean at {@code position}. */ default boolean getBoolean(int position) { throw new UnsupportedOperationException(getClass().getName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index 07a2b58db..6b67f87fb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -20,9 +20,11 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -126,6 +128,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.BINARY_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.TEXT) { + return this; + } + ColumnUtil.checkConversion(TSDataType.TEXT, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeBinary(getBinary(position)); + } + } + return builder.build(); + } + @Override public Binary getBinary(int position) { return values[position + arrayOffset]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java index 7d30c6a00..38c6a8720 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java @@ -20,9 +20,11 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -94,6 +96,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.BYTE_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.BOOLEAN) { + return this; + } + ColumnUtil.checkConversion(TSDataType.BOOLEAN, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeBoolean(getBoolean(position)); + } + } + return builder.build(); + } + @Override public boolean getBoolean(int position) { return values[position + arrayOffset]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java index 3c741db46..77b5746f6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; import java.util.Arrays; @@ -37,6 +38,14 @@ public class ColumnUtil { private ColumnUtil() {} + static void checkConversion(TSDataType sourceType, TSDataType targetType) { + requireNonNull(targetType, "type is null"); + if (!targetType.isCompatible(sourceType)) { + throw new ClassCastException( + Messages.format("error.common.unsupported_cast", sourceType, targetType)); + } + } + static void checkArrayRange(int[] array, int offset, int length) { requireNonNull(array, "array is null"); if (offset < 0 || length < 0 || offset + length > array.length) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java index 1e2f7067b..7ce598483 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java @@ -252,6 +252,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.DICTIONARY; } + @Override + public Column convertTo(TSDataType type) { + TSDataType sourceType = getDataType(); + if (type == sourceType) { + return this; + } + ColumnUtil.checkConversion(sourceType, type); + // Transform dictionary values only, preserving the ids and dictionary encoding. + return new DictionaryColumn( + idsOffset, + positionCount, + dictionary.convertTo(type), + ids, + isCompact(), + isSequentialIds, + randomDictionaryId()); + } + @Override public boolean mayHaveNull() { return mayHaveNull && dictionary.mayHaveNull(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java index 81e19a307..953f13730 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java @@ -20,9 +20,11 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -95,6 +97,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT64_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.DOUBLE) { + return this; + } + ColumnUtil.checkConversion(TSDataType.DOUBLE, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeDouble(getDouble(position)); + } + } + return builder.build(); + } + @Override public double getDouble(int position) { return values[position + arrayOffset]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java index 56ce7a1f8..77326c91d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java @@ -20,9 +20,11 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -96,6 +98,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT32_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.FLOAT) { + return this; + } + ColumnUtil.checkConversion(TSDataType.FLOAT, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeFloat(getFloat(position)); + } + } + return builder.build(); + } + @Override public float getFloat(int position) { return values[position + arrayOffset]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index 59d3a7505..810450605 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -20,9 +20,11 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -124,6 +126,28 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT32_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == dataType) { + return this; + } + ColumnUtil.checkConversion(dataType, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else if (dataType == TSDataType.DATE) { + // DATE has an int physical representation but a formatted text cast. + builder.writeBinary( + new Binary(TSDataType.getDateStringValue(getInt(position)), StandardCharsets.UTF_8)); + } else { + builder.writeInt(getInt(position)); + } + } + return builder.build(); + } + @Override public int getInt(int position) { return values[position + arrayOffset]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 2b6bc46fc..7abedf523 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -20,9 +20,11 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -96,6 +98,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT64_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.INT64) { + return this; + } + ColumnUtil.checkConversion(TSDataType.INT64, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeLong(getLong(position)); + } + } + return builder.build(); + } + @Override public long getLong(int position) { return values[position + arrayOffset]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index 0d0145295..75fea0ed5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -65,6 +65,11 @@ public ColumnEncoding getEncoding() { throw new UnsupportedOperationException(getClass().getName()); } + @Override + public Column convertTo(TSDataType type) { + return create(type, positionCount); + } + @Override public boolean mayHaveNull() { return true; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java index 30c3af73e..8c39c9549 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java @@ -79,6 +79,17 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.RLE; } + @Override + public Column convertTo(TSDataType type) { + TSDataType sourceType = getDataType(); + if (type == sourceType) { + return this; + } + ColumnUtil.checkConversion(sourceType, type); + // Transform the single stored value and keep the run-length representation. + return new RunLengthEncodedColumn(value.convertTo(type), positionCount); + } + @Override public boolean getBoolean(int position) { return value.getBoolean(0); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java index c2a5e7427..d0dc19377 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java @@ -20,9 +20,11 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; import java.io.DataOutputStream; @@ -83,6 +85,20 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT64_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.INT64) { + return this; + } + ColumnUtil.checkConversion(TSDataType.INT64, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + builder.writeLong(getLong(position)); + } + return builder.build(); + } + @Override public long getLong(int position) { return values[position + arrayOffset]; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index 5e71a89c3..51203e0db 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.BooleanColumn; @@ -55,6 +56,157 @@ public class ColumnTest { + @Test + public void testConvertTo() { + Column source = + new IntColumn( + 4, Optional.of(new boolean[] {false, true, false, false}), new int[] {0, 1, 2, 3}) + .getRegion(1, 3); + + Column converted = source.convertTo(TSDataType.DOUBLE); + Assert.assertEquals(TSDataType.DOUBLE, converted.getDataType()); + Assert.assertEquals(3, converted.getPositionCount()); + Assert.assertTrue(converted.isNull(0)); + Assert.assertEquals(2.0D, converted.getDouble(1), 0); + Assert.assertEquals(3.0D, converted.getDouble(2), 0); + + Assert.assertSame(source, source.convertTo(TSDataType.INT32)); + Assert.assertThrows(ClassCastException.class, () -> source.convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows(NullPointerException.class, () -> source.convertTo(null)); + } + + @Test + public void testConvertToText() { + Column dateColumn = + new IntColumn( + 3, + Optional.of(new boolean[] {false, true, false}), + new int[] {20260714, 20260715, 20260716}, + TSDataType.DATE); + + Column converted = dateColumn.convertTo(TSDataType.TEXT); + Assert.assertEquals(TSDataType.TEXT, converted.getDataType()); + Assert.assertEquals(new Binary("2026-07-14", StandardCharsets.UTF_8), converted.getBinary(0)); + Assert.assertTrue(converted.isNull(1)); + Assert.assertEquals(new Binary("2026-07-16", StandardCharsets.UTF_8), converted.getBinary(2)); + } + + @Test + public void testConvertEncodedColumns() { + Column dictionary = + DictionaryColumn.create( + 3, new IntColumn(3, Optional.empty(), new int[] {1, 2, 3}), new int[] {2, 0, 2}); + Column convertedDictionary = dictionary.convertTo(TSDataType.DOUBLE); + Assert.assertTrue(convertedDictionary instanceof DictionaryColumn); + Assert.assertEquals(3.0D, convertedDictionary.getDouble(0), 0); + Assert.assertEquals(1.0D, convertedDictionary.getDouble(1), 0); + Assert.assertEquals(3.0D, convertedDictionary.getDouble(2), 0); + + Column runLengthEncoded = + new RunLengthEncodedColumn(new FloatColumn(1, Optional.empty(), new float[] {1.25F}), 4); + Column convertedRle = runLengthEncoded.convertTo(TSDataType.DOUBLE); + Assert.assertTrue(convertedRle instanceof RunLengthEncodedColumn); + Assert.assertEquals(4, convertedRle.getPositionCount()); + Assert.assertEquals(1.25D, convertedRle.getDouble(3), 0); + + Column nullColumn = new NullColumn(3).convertTo(TSDataType.DOUBLE); + Assert.assertEquals(TSDataType.DOUBLE, nullColumn.getDataType()); + Assert.assertEquals(3, nullColumn.getPositionCount()); + Assert.assertTrue(nullColumn.isNull(0)); + Assert.assertTrue(nullColumn.isNull(2)); + } + + @Test + public void testConvertToRejectsIncompatibleEmptyColumns() { + Column emptyDictionary = + DictionaryColumn.create( + 2, new IntColumn(2, Optional.empty(), new int[] {1, 2}), new int[] {0, 1}) + .getRegion(0, 0); + Column emptyRle = + new RunLengthEncodedColumn( + new IntColumn(1, Optional.of(new boolean[] {true}), new int[1]), 0); + + Assert.assertThrows( + ClassCastException.class, + () -> new IntColumn(0, Optional.empty(), new int[0]).convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows( + ClassCastException.class, + () -> + new IntColumn(2, Optional.of(new boolean[] {true, true}), new int[2]) + .convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows( + ClassCastException.class, () -> emptyDictionary.convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows(ClassCastException.class, () -> emptyRle.convertTo(TSDataType.BOOLEAN)); + } + + @Test + public void testConvertToDoesNotReadBoxedValues() { + Column booleanColumn = + new BooleanColumn(1, Optional.empty(), new boolean[] {true}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column intColumn = + new IntColumn(1, Optional.empty(), new int[] {1}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column longColumn = + new LongColumn(1, Optional.empty(), new long[] {2L}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column floatColumn = + new FloatColumn(1, Optional.empty(), new float[] {3.0F}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column doubleColumn = + new DoubleColumn(1, Optional.empty(), new double[] {4.0D}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column binaryColumn = + new BinaryColumn( + 1, Optional.empty(), new Binary[] {new Binary("5", StandardCharsets.UTF_8)}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column timeColumn = + new TimeColumn(1, new long[] {6L}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + + Assert.assertEquals( + new Binary("true", StandardCharsets.UTF_8), + booleanColumn.convertTo(TSDataType.TEXT).getBinary(0)); + Assert.assertEquals(1.0D, intColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + Assert.assertEquals(2.0D, longColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + Assert.assertEquals(3.0D, floatColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + Assert.assertEquals( + new Binary("4.0", StandardCharsets.UTF_8), + doubleColumn.convertTo(TSDataType.TEXT).getBinary(0)); + Assert.assertEquals( + new Binary("5", StandardCharsets.UTF_8), + binaryColumn.convertTo(TSDataType.BLOB).getBinary(0)); + Assert.assertEquals(6.0D, timeColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + } + @Test public void testWriteTo() throws IOException { Binary binary = new Binary("test", StandardCharsets.UTF_8);