diff --git a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java index c9e30d200..7cf721bed 100644 --- a/java/common/src/main/java/org/apache/tsfile/block/column/Column.java +++ b/java/common/src/main/java/org/apache/tsfile/block/column/Column.java @@ -23,6 +23,9 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; public interface Column { @@ -33,6 +36,9 @@ public interface Column { /** Get the encoding for this column. */ ColumnEncoding getEncoding(); + /** Converts this column to the specified data type. */ + Column convertTo(TSDataType type); + /** Gets a boolean at {@code position}. */ default boolean getBoolean(int position) { throw new UnsupportedOperationException(getClass().getName()); @@ -108,6 +114,33 @@ default TsPrimitiveType getTsPrimitiveType(int position) { throw new UnsupportedOperationException(getClass().getName()); } + /** Serializes the value at {@code index} to {@code buffer}. */ + default void writeTo(int index, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Serializes the value at {@code index} to {@code stream}. */ + default void writeTo(int index, DataOutputStream stream) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Serializes all non-null values in this column. */ + default void serializeWithoutNulls(DataOutputStream output) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns whether the values at the two positions are equal. */ + default boolean arePositionsEqual(int pos1, int pos2) { + return arePositionsEqual(pos1, this, pos2); + } + + /** + * Returns whether this value and the value in another column are equal. + * + *

For non-null values, this column and {@code that} must provide compatible value accessors. + */ + boolean arePositionsEqual(int thisPos, Column that, int thatPos); + /** * Is it possible the column may have a null value? If false, the column cannot contain a null, * but if true, the column may or may not have a null. diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index f97291046..eaa09be73 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -39,45 +39,95 @@ public enum TSDataType { /** BOOLEAN. */ - BOOLEAN((byte) 0), + BOOLEAN( + (byte) 0, + 1, + false, + true, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray), /** INT32. */ - INT32((byte) 1), + INT32( + (byte) 1, + 4, + true, + true, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray), /** INT64. */ - INT64((byte) 2), + INT64((byte) 2, 8, true, true, TSDataType::castToLongSingleValue, TSDataType::castToLongArray), /** FLOAT. */ - FLOAT((byte) 3), + FLOAT((byte) 3, 4, true, true, TSDataType::castToFloatSingleValue, TSDataType::castToFloatArray), /** DOUBLE. */ - DOUBLE((byte) 4), + DOUBLE( + (byte) 4, 8, true, true, TSDataType::castToDoubleSingleValue, TSDataType::castToDoubleArray), /** TEXT. */ - TEXT((byte) 5), + TEXT((byte) 5, 8, false, true, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** VECTOR. */ - VECTOR((byte) 6), + VECTOR( + (byte) 6, + 8, + false, + false, + TSDataType::unsupportedSingleValueCast, + TSDataType::unsupportedArrayCast), /** UNKNOWN. */ - UNKNOWN((byte) 7), + UNKNOWN( + (byte) 7, + TSDataType.UNSUPPORTED_DATA_TYPE_SIZE, + false, + false, + TSDataType::unsupportedSingleValueCast, + TSDataType::unsupportedArrayCast), /** TIMESTAMP. */ - TIMESTAMP((byte) 8), + TIMESTAMP( + (byte) 8, + 8, + false, + true, + TSDataType::castToTimestampSingleValue, + TSDataType::castToTimestampArray), /** DATE. */ - DATE((byte) 9), + DATE( + (byte) 9, + 4, + false, + true, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray), /** BLOB. */ - BLOB((byte) 10), + BLOB((byte) 10, 8, false, false, TSDataType::castToBlobSingleValue, TSDataType::castToBlobArray), /** STRING */ - STRING((byte) 11), + STRING((byte) 11, 8, false, true, TSDataType::castToTextSingleValue, TSDataType::castToTextArray), /** OBJECT */ - OBJECT((byte) 12); - + OBJECT( + (byte) 12, + 8, + false, + false, + TSDataType::castIdenticalSingleValue, + TSDataType::castIdenticalArray); + + private static final int UNSUPPORTED_DATA_TYPE_SIZE = -1; + private static final Object UNSUPPORTED_CAST = new Object(); private final byte type; + private final int dataTypeSize; + private final boolean numeric; + private final boolean comparable; + private final SingleValueCaster singleValueCaster; + private final ArrayCaster arrayCaster; private static final Map> compatibleTypes; static { @@ -147,8 +197,19 @@ public enum TSDataType { compatibleTypes.put(OBJECT, Collections.emptySet()); } - TSDataType(byte type) { + TSDataType( + byte type, + int dataTypeSize, + boolean numeric, + boolean comparable, + SingleValueCaster singleValueCaster, + ArrayCaster arrayCaster) { this.type = type; + this.dataTypeSize = dataTypeSize; + this.numeric = numeric; + this.comparable = comparable; + this.singleValueCaster = singleValueCaster; + this.arrayCaster = arrayCaster; } /** @@ -211,271 +272,230 @@ public Object castFromSingleValue(TSDataType sourceType, Object value) { if (Objects.isNull(value)) { return null; } - switch (this) { - case BOOLEAN: - if (sourceType == TSDataType.BOOLEAN) { - return value; - } else { - break; - } - case INT32: - if (sourceType == TSDataType.INT32) { - return value; - } else { - break; - } - case INT64: - if (sourceType == TSDataType.INT64) { - return value; - } else if (sourceType == INT32) { - return (long) ((int) value); - } else if (sourceType == TIMESTAMP) { - return value; - } else { - break; - } - case FLOAT: - if (sourceType == TSDataType.FLOAT) { - return value; - } else if (sourceType == INT32) { - return (float) ((int) value); - } else { - break; - } - case DOUBLE: - if (sourceType == TSDataType.DOUBLE) { - return value; - } else if (sourceType == INT32) { - return (double) ((int) value); - } else if (sourceType == INT64) { - return (double) ((long) value); - } else if (sourceType == FLOAT) { - return (double) ((float) value); - } else if (sourceType == TIMESTAMP) { - return (double) ((long) value); - } else { - break; - } - case TEXT: - if (sourceType == TSDataType.TEXT || sourceType == TSDataType.STRING) { - return value; - } else if (sourceType == TSDataType.INT32 - || sourceType == TSDataType.INT64 - || sourceType == TSDataType.FLOAT - || sourceType == TSDataType.DOUBLE - || sourceType == TSDataType.BOOLEAN - || sourceType == TSDataType.TIMESTAMP) { - return new Binary(String.valueOf(value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.DATE) { - return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.BLOB) { - return new Binary(value.toString(), StandardCharsets.UTF_8); - } else { - break; - } - case TIMESTAMP: - if (sourceType == TSDataType.TIMESTAMP) { - return value; - } else if (sourceType == INT32) { - return (long) ((int) value); - } else if (sourceType == INT64) { - return value; - } else { - break; - } - case DATE: - if (sourceType == TSDataType.DATE) { - return value; - } else { - break; - } - case BLOB: - if (sourceType == TSDataType.BLOB - || sourceType == TSDataType.STRING - || sourceType == TSDataType.TEXT) { - return value; - } else { - break; - } - case STRING: - if (sourceType == TSDataType.STRING || sourceType == TSDataType.TEXT) { - return value; - } else if (sourceType == TSDataType.INT32 - || sourceType == TSDataType.INT64 - || sourceType == TSDataType.FLOAT - || sourceType == TSDataType.DOUBLE - || sourceType == TSDataType.BOOLEAN - || sourceType == TSDataType.TIMESTAMP) { - return new Binary(String.valueOf(value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.DATE) { - return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8); - } else if (sourceType == TSDataType.BLOB) { - return new Binary(value.toString(), StandardCharsets.UTF_8); - } else { - break; - } - case OBJECT: - if (sourceType == TSDataType.OBJECT) { - return value; - } else { - break; - } - case VECTOR: - case UNKNOWN: - default: - break; + Object result = singleValueCaster.cast(this, sourceType, value); + if (result != UNSUPPORTED_CAST) { + return result; } throw new ClassCastException( Messages.format("error.common.unsupported_cast", sourceType, this)); } - @SuppressWarnings({"java:S3012", "java:S3776", "java:S6541"}) + private static Object castIdenticalSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + return sourceType == targetType ? value : UNSUPPORTED_CAST; + } + + private static Object castToLongSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return value; + } + return sourceType == INT32 ? (long) ((int) value) : UNSUPPORTED_CAST; + } + + private static Object castToFloatSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == FLOAT) { + return value; + } + return sourceType == INT32 ? (float) ((int) value) : UNSUPPORTED_CAST; + } + + private static Object castToDoubleSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == DOUBLE) { + return value; + } + if (sourceType == INT32) { + return (double) ((int) value); + } + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return (double) ((long) value); + } + return sourceType == FLOAT ? (double) ((float) value) : UNSUPPORTED_CAST; + } + + private static Object castToTextSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == TEXT || sourceType == STRING) { + return value; + } + if (sourceType == INT32 + || sourceType == INT64 + || sourceType == FLOAT + || sourceType == DOUBLE + || sourceType == BOOLEAN + || sourceType == TIMESTAMP) { + return new Binary(String.valueOf(value), StandardCharsets.UTF_8); + } + if (sourceType == DATE) { + return new Binary(getDateStringValue((int) value), StandardCharsets.UTF_8); + } + return sourceType == BLOB + ? new Binary(value.toString(), StandardCharsets.UTF_8) + : UNSUPPORTED_CAST; + } + + private static Object castToTimestampSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + if (sourceType == TIMESTAMP || sourceType == INT64) { + return value; + } + return sourceType == INT32 ? (long) ((int) value) : UNSUPPORTED_CAST; + } + + private static Object castToBlobSingleValue( + TSDataType targetType, TSDataType sourceType, Object value) { + return sourceType == BLOB || sourceType == STRING || sourceType == TEXT + ? value + : UNSUPPORTED_CAST; + } + + private static Object unsupportedSingleValueCast( + TSDataType targetType, TSDataType sourceType, Object value) { + return UNSUPPORTED_CAST; + } + public Object castFromArray(TSDataType sourceType, Object array) { - switch (this) { - case BOOLEAN: - if (sourceType == TSDataType.BOOLEAN) { - return array; - } else { - break; - } - case INT32: - if (sourceType == TSDataType.INT32) { - return array; - } else { - break; - } - case INT64: - if (sourceType == TSDataType.INT64) { - return array; - } else if (sourceType == INT32) { - return Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray(); - } else if (sourceType == TIMESTAMP) { - return array; - } else { - break; - } - case FLOAT: - if (sourceType == TSDataType.FLOAT) { - return array; - } else if (sourceType == INT32) { - int[] tmp = (int[]) array; - float[] result = new float[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = tmp[i]; - } - return result; - } else { - break; - } - case DOUBLE: - if (sourceType == TSDataType.DOUBLE) { - return array; - } else if (sourceType == INT32) { - return Arrays.stream((int[]) array).mapToDouble(Double::valueOf).toArray(); - } else if (sourceType == INT64) { - return Arrays.stream((long[]) array).mapToDouble(Double::valueOf).toArray(); - } else if (sourceType == FLOAT) { - float[] tmp = (float[]) array; - double[] result = new double[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = tmp[i]; - } - return result; - } else if (sourceType == TIMESTAMP) { - return Arrays.stream((long[]) array).mapToDouble(Double::valueOf).toArray(); - } else { - break; - } - case TIMESTAMP: - if (sourceType == TSDataType.TIMESTAMP) { - return array; - } else if (sourceType == INT32) { - return Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray(); - } else if (sourceType == INT64) { - return array; - } else { - break; - } - case DATE: - if (sourceType == TSDataType.DATE) { - return array; - } else { - break; - } - case BLOB: - if (sourceType == TSDataType.BLOB - || sourceType == TSDataType.STRING - || sourceType == TSDataType.TEXT) { - return array; - } else { - break; - } - case TEXT: - case STRING: - if (sourceType == TSDataType.STRING - || sourceType == TSDataType.TEXT - || sourceType == TSDataType.BLOB) { - return array; - } else if (sourceType == TSDataType.INT32) { - int[] tmp = (int[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.DATE) { - int[] tmp = (int[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(TSDataType.getDateStringValue(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.INT64 || sourceType == TSDataType.TIMESTAMP) { - long[] tmp = (long[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.FLOAT) { - float[] tmp = (float[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.DOUBLE) { - double[] tmp = (double[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else if (sourceType == TSDataType.BOOLEAN) { - boolean[] tmp = (boolean[]) array; - Binary[] result = new Binary[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = new Binary(String.valueOf(tmp[i]), StandardCharsets.UTF_8); - } - return result; - } else { - break; - } - case OBJECT: - if (sourceType == TSDataType.OBJECT) { - return array; - } else { - break; - } - case VECTOR: - case UNKNOWN: - default: - break; + Object result = arrayCaster.cast(this, sourceType, array); + if (result != UNSUPPORTED_CAST) { + return result; } throw new ClassCastException( Messages.format("error.common.unsupported_cast", sourceType, this)); } + private static Object castIdenticalArray( + TSDataType targetType, TSDataType sourceType, Object array) { + return sourceType == targetType ? array : UNSUPPORTED_CAST; + } + + private static Object castToLongArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return array; + } + return sourceType == INT32 + ? Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray() + : UNSUPPORTED_CAST; + } + + private static Object castToFloatArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == FLOAT) { + return array; + } + if (sourceType != INT32) { + return UNSUPPORTED_CAST; + } + int[] values = (int[]) array; + float[] result = new float[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = values[i]; + } + return result; + } + + private static Object castToDoubleArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == DOUBLE) { + return array; + } + if (sourceType == INT32) { + return Arrays.stream((int[]) array).mapToDouble(Double::valueOf).toArray(); + } + if (sourceType == INT64 || sourceType == TIMESTAMP) { + return Arrays.stream((long[]) array).mapToDouble(Double::valueOf).toArray(); + } + if (sourceType != FLOAT) { + return UNSUPPORTED_CAST; + } + float[] values = (float[]) array; + double[] result = new double[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = values[i]; + } + return result; + } + + private static Object castToTimestampArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == TIMESTAMP || sourceType == INT64) { + return array; + } + return sourceType == INT32 + ? Arrays.stream((int[]) array).mapToLong(Long::valueOf).toArray() + : UNSUPPORTED_CAST; + } + + private static Object castToBlobArray( + TSDataType targetType, TSDataType sourceType, Object array) { + return sourceType == BLOB || sourceType == STRING || sourceType == TEXT + ? array + : UNSUPPORTED_CAST; + } + + private static Object castToTextArray( + TSDataType targetType, TSDataType sourceType, Object array) { + if (sourceType == STRING || sourceType == TEXT || sourceType == BLOB) { + return array; + } + if (sourceType == INT32) { + int[] values = (int[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == DATE) { + int[] values = (int[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(getDateStringValue(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == INT64 || sourceType == TIMESTAMP) { + long[] values = (long[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == FLOAT) { + float[] values = (float[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == DOUBLE) { + double[] values = (double[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + if (sourceType == BOOLEAN) { + boolean[] values = (boolean[]) array; + Binary[] result = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return result; + } + return UNSUPPORTED_CAST; + } + + private static Object unsupportedArrayCast( + TSDataType targetType, TSDataType sourceType, Object array) { + return UNSUPPORTED_CAST; + } + public static TSDataType deserializeFrom(ByteBuffer buffer) { return deserialize(buffer.get()); } @@ -501,26 +521,10 @@ public void serializeTo(FileOutputStream outputStream) throws IOException { } public int getDataTypeSize() { - switch (this) { - case BOOLEAN: - return 1; - case INT32: - case FLOAT: - case DATE: - return 4; - // For text: return the size of reference here - case TEXT: - case INT64: - case DOUBLE: - case VECTOR: - case BLOB: - case OBJECT: - case STRING: - case TIMESTAMP: - return 8; - default: - throw new UnSupportedDataTypeException(this.toString()); + if (dataTypeSize == UNSUPPORTED_DATA_TYPE_SIZE) { + throw new UnSupportedDataTypeException(this.toString()); } + return dataTypeSize; } /** @@ -539,25 +543,10 @@ public byte serialize() { * @throws UnSupportedDataTypeException when meets unSupported DataType */ public boolean isNumeric() { - switch (this) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - return true; - // For text: return the size of reference here - case BLOB: - case TIMESTAMP: - case DATE: - case STRING: - case BOOLEAN: - case TEXT: - case VECTOR: - case OBJECT: - return false; - default: - throw new UnSupportedDataTypeException(this.toString()); + if (this == UNKNOWN) { + throw new UnSupportedDataTypeException(this.toString()); } + return numeric; } /** @@ -567,24 +556,10 @@ public boolean isNumeric() { * @throws UnSupportedDataTypeException when meets unSupported DataType */ public boolean isComparable() { - switch (this) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case TEXT: - case BOOLEAN: - case TIMESTAMP: - case DATE: - case STRING: - return true; - case VECTOR: - case BLOB: - case OBJECT: - return false; - default: - throw new UnSupportedDataTypeException(this.toString()); + if (this == UNKNOWN) { + throw new UnSupportedDataTypeException(this.toString()); } + return comparable; } public boolean isBinary() { @@ -603,4 +578,16 @@ public boolean hasNoValueInStatistics() { public static String getDateStringValue(int value) { return String.format("%04d-%02d-%02d", value / 10000, (value % 10000) / 100, value % 100); } + + @FunctionalInterface + private interface SingleValueCaster { + + Object cast(TSDataType targetType, TSDataType sourceType, Object value); + } + + @FunctionalInterface + private interface ArrayCaster { + + Object cast(TSDataType targetType, TSDataType sourceType, Object array); + } } diff --git a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java index fa88e68c6..4b20ffbab 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java @@ -28,73 +28,6 @@ public abstract class TsPrimitiveType implements Serializable { - /** - * get tsPrimitiveType by resultDataType. - * - * @param dataType given TsDataType - */ - public static TsPrimitiveType getByType(TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return new TsPrimitiveType.TsBoolean(); - case INT32: - return new TsPrimitiveType.TsInt(); - case DATE: - return new TsPrimitiveType.TsInt(TSDataType.DATE); - case INT64: - case TIMESTAMP: - return new TsPrimitiveType.TsLong(); - case FLOAT: - return new TsPrimitiveType.TsFloat(); - case DOUBLE: - return new TsPrimitiveType.TsDouble(); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new TsPrimitiveType.TsBinary(); - case VECTOR: - return new TsPrimitiveType.TsVector(); - default: - throw new UnSupportedDataTypeException( - Messages.format("error.common.unsupported_data_type", dataType)); - } - } - - /** - * get tsPrimitiveType by resultDataType and initial value. - * - * @param dataType given TsDataType - * @param v initial value - */ - public static TsPrimitiveType getByType(TSDataType dataType, Object v) { - switch (dataType) { - case BOOLEAN: - return new TsPrimitiveType.TsBoolean((boolean) v); - case INT32: - return new TsPrimitiveType.TsInt((int) v); - case DATE: - return new TsPrimitiveType.TsInt((int) v, TSDataType.DATE); - case INT64: - case TIMESTAMP: - return new TsPrimitiveType.TsLong((long) v); - case FLOAT: - return new TsPrimitiveType.TsFloat((float) v); - case DOUBLE: - return new TsPrimitiveType.TsDouble((double) v); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new TsPrimitiveType.TsBinary((Binary) v); - case VECTOR: - return new TsPrimitiveType.TsVector((TsPrimitiveType[]) v); - default: - throw new UnSupportedDataTypeException( - Messages.format("error.common.unsupported_data_type", dataType)); - } - } - public boolean getBoolean() { throw new UnsupportedOperationException( Messages.format("error.common.subclass_op_not_supported", "getBoolean()")); @@ -182,6 +115,36 @@ public void setVector(TsPrimitiveType[] val) { public abstract TSDataType getDataType(); + public abstract void copy(TsPrimitiveType value); + + private static TsPrimitiveType copyValue(TsPrimitiveType value) { + if (value == null) { + return null; + } + if (value instanceof TsBoolean) { + return new TsBoolean((TsBoolean) value); + } + if (value instanceof TsInt) { + return new TsInt((TsInt) value); + } + if (value instanceof TsLong) { + return new TsLong((TsLong) value); + } + if (value instanceof TsFloat) { + return new TsFloat((TsFloat) value); + } + if (value instanceof TsDouble) { + return new TsDouble((TsDouble) value); + } + if (value instanceof TsBinary) { + return new TsBinary((TsBinary) value); + } + if (value instanceof TsVector) { + return new TsVector((TsVector) value); + } + throw new UnSupportedDataTypeException(String.valueOf(value.getDataType())); + } + @Override public String toString() { return getStringValue(); @@ -208,6 +171,10 @@ public TsBoolean(boolean value) { this.value = value; } + public TsBoolean(TsBoolean other) { + this.value = other.value; + } + @Override public boolean getBoolean() { return value; @@ -233,6 +200,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsBoolean", "Boolean")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setBoolean(value.getBoolean()); + } + @Override public void reset() { value = false; @@ -297,6 +273,11 @@ public TsInt(int value, TSDataType dataType) { this.dataType = dataType; } + public TsInt(TsInt other) { + this.value = other.value; + this.dataType = other.dataType; + } + @Override public int getInt() { return value; @@ -304,12 +285,12 @@ public int getInt() { @Override public double getDouble() { - return (double) value; + return value; } @Override public long getLong() { - return (long) value; + return value; } @Override @@ -319,6 +300,9 @@ public float getFloat() { @Override public Binary getBinary() { + if (dataType == TSDataType.DATE) { + return new Binary(TSDataType.getDateStringValue(value), StandardCharsets.UTF_8); + } return new Binary(String.valueOf(this.value), StandardCharsets.UTF_8); } @@ -337,6 +321,16 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsInt", "Integer")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setInt(value.getInt()); + dataType = value.getDataType(); + } + @Override public void reset() { value = 0; @@ -390,6 +384,10 @@ public TsLong(long value) { this.value = value; } + public TsLong(TsLong other) { + this.value = other.value; + } + @Override public long getLong() { return value; @@ -420,6 +418,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsLong", "Long")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setLong(value.getLong()); + } + @Override public void reset() { value = 0; @@ -473,6 +480,10 @@ public TsFloat(float value) { this.value = value; } + public TsFloat(TsFloat other) { + this.value = other.value; + } + @Override public float getFloat() { return value; @@ -503,6 +514,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsFloat", "float")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setFloat(value.getFloat()); + } + @Override public void reset() { value = 0; @@ -556,6 +576,10 @@ public TsDouble(double value) { this.value = value; } + public TsDouble(TsDouble other) { + this.value = other.value; + } + @Override public double getDouble() { return value; @@ -581,6 +605,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsDouble", "Double")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setDouble(value.getDouble()); + } + @Override public void reset() { value = 0.0; @@ -634,6 +667,18 @@ public TsBinary(Binary value) { this.value = value; } + public TsBinary(TsBinary other) { + this.value = copyBinary(other.value); + } + + private static Binary copyBinary(Binary value) { + if (value == null) { + return null; + } + byte[] bytes = value.getValues(); + return new Binary(bytes == null ? null : Arrays.copyOf(bytes, bytes.length)); + } + @Override public Binary getBinary() { return value; @@ -654,6 +699,15 @@ public void setObject(Object val) { Messages.format("error.common.tsprimitive_wrong_value_type", "TsBinary", "Binary")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setBinary(copyBinary(value.getBinary())); + } + @Override public void reset() { value = null; @@ -707,6 +761,21 @@ public TsVector(TsPrimitiveType[] values) { this.values = values; } + public TsVector(TsVector other) { + this.values = copyValues(other.values); + } + + private static TsPrimitiveType[] copyValues(TsPrimitiveType[] values) { + if (values == null) { + return null; + } + TsPrimitiveType[] copiedValues = new TsPrimitiveType[values.length]; + for (int i = 0; i < values.length; i++) { + copiedValues[i] = copyValue(values[i]); + } + return copiedValues; + } + @Override public TsPrimitiveType[] getVector() { return values; @@ -728,6 +797,15 @@ public void setObject(Object val) { "error.common.tsprimitive_wrong_value_type", "TsVector", "TsPrimitiveType[]")); } + @Override + public void copy(TsPrimitiveType value) { + if (value == null) { + reset(); + return; + } + setVector(copyValues(value.getVector())); + } + @Override public void reset() { values = null; diff --git a/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java b/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java index a7ab7dac1..2ec35339c 100644 --- a/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java +++ b/java/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java @@ -21,6 +21,10 @@ public class UnSupportedDataTypeException extends RuntimeException { + // true if the exception is raised by a datatype that is known to be unsupported, + // otherwise, it is caused by an unknown/uncovered datatype + private boolean isChecked = false; + public UnSupportedDataTypeException(String message) { super("Unsupported dataType: " + message); } @@ -28,4 +32,13 @@ public UnSupportedDataTypeException(String message) { public UnSupportedDataTypeException(String message, Throwable e) { super(message + e.getMessage()); } + + public UnSupportedDataTypeException setChecked(boolean checked) { + isChecked = checked; + return this; + } + + public boolean isChecked() { + return isChecked; + } } diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties index 98909f7a6..59679f6fd 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties @@ -1320,6 +1320,18 @@ log.utils.bytes_int_to_bytes_error = tsfile-common BytesUtils: cannot convert an # BytesUtils.longToBytes — cannot convert long to byte array (3 args: srcNum, pos, width) log.utils.bytes_long_to_bytes_error = tsfile-common BytesUtils: cannot convert a long {} to a byte array, pos {}, width {} +# RecordUtils.parseSimpleTupleRecord — invalid timestamp +log.utils.record_timestamp_illegal = Given timestamp is illegal: {} + +# RecordUtils.parseSimpleTupleRecord — measurement type not found +log.utils.record_measurement_type_not_found = Type not found for measurement ID {}; skipping + +# RecordUtils.parseSimpleTupleRecord — failed to parse measurement +log.utils.record_parse_measurement_error = Failed to parse measurement; omitting it + +# RecordUtils.parseSimpleTupleRecord — unsupported data type +log.utils.record_unsupported_data_type = Unsupported data type: {} + # TsFileSketchTool — cannot load file (filename arg) error.utils.sketch_tool_cannot_load = Cannot load file %1$s because the file has crashed. diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties index a1a437cfb..c2406b634 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties @@ -1320,6 +1320,18 @@ log.utils.bytes_int_to_bytes_error = tsfile-common BytesUtils: 无法将 integer # BytesUtils.longToBytes — cannot convert long to byte array (3 args: srcNum, pos, width) log.utils.bytes_long_to_bytes_error = tsfile-common BytesUtils: 无法将 long {} 转换为 byte 数组，pos {}，width {} +# RecordUtils.parseSimpleTupleRecord — 时间戳非法 +log.utils.record_timestamp_illegal = 给定的时间戳非法：{} + +# RecordUtils.parseSimpleTupleRecord — 未找到测点类型 +log.utils.record_measurement_type_not_found = 未找到 measurement ID {} 的类型，跳过该测点 + +# RecordUtils.parseSimpleTupleRecord — 测点解析失败 +log.utils.record_parse_measurement_error = 解析测点时出错，已忽略 + +# RecordUtils.parseSimpleTupleRecord — 不支持的数据类型 +log.utils.record_unsupported_data_type = 不支持的数据类型：{} + # TsFileSketchTool — cannot load file (filename arg) error.utils.sketch_tool_cannot_load = 无法加载文件 %1$s，因为文件已损坏。 diff --git a/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java b/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java index 159e2ea65..d33f822c2 100644 --- a/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java +++ b/java/examples/src/main/java/org/apache/tsfile/DataGenerator.java @@ -21,32 +21,38 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import java.time.LocalDate; public class DataGenerator { + + private static final TypeService GENERATE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> index -> index % 2 == 0; + case INT32 -> index -> index; + case INT64, TIMESTAMP -> index -> (long) index; + case FLOAT -> index -> (float) index; + case DOUBLE -> index -> (double) index; + case DATE -> index -> LocalDate.of(2024, 1, index % 30 + 1); + case TEXT, STRING, BLOB -> + index -> new Binary(String.valueOf(index), TSFileConfig.STRING_CHARSET); + case OBJECT, ROW, UNKNOWN, VECTOR -> index -> null; + }; + + static { + GENERATE_VALUE_SERVICE.check(); + } + public static Object generate(TSDataType type, int index) { - switch (type) { - case INT32: - return index; - case INT64: - case TIMESTAMP: - return (long) index; - case FLOAT: - return (float) index; - case DOUBLE: - return (double) index; - case BOOLEAN: - return index % 2 == 0; - case DATE: - return LocalDate.of(2024, 1, index % 30 + 1); - case TEXT: - case STRING: - case BLOB: - return new Binary(String.valueOf(index), TSFileConfig.STRING_CHARSET); - default: - return null; - } + return GENERATE_VALUE_SERVICE.call(Type.fromTsDataType(type)).generate(index); + } + + @FunctionalInterface + private interface ValueGenerator { + Object generate(int index); } } diff --git a/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java b/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java index ecd3fdd27..5873784e7 100644 --- a/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java +++ b/java/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java @@ -34,6 +34,8 @@ import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.reader.page.PageReader; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -51,6 +53,25 @@ public class TsFileSequenceRead { public static final String POINT_IN_PAGE = "\t\tpoints in the page: "; private static int MASK = 0x80; + private static final TypeService PAGE_VALUE_READER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> Decoder::readBoolean; + case INT32, DATE -> Decoder::readInt; + case INT64, TIMESTAMP -> Decoder::readLong; + case FLOAT -> Decoder::readFloat; + case DOUBLE -> Decoder::readDouble; + case TEXT, STRING, BLOB -> Decoder::readBinary; + case OBJECT, ROW, UNKNOWN, VECTOR -> + (decoder, buffer) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())); + }; + }; + + static { + PAGE_VALUE_READER_SERVICE.check(); + } + @SuppressWarnings({ "squid:S3776", "squid:S106" @@ -133,6 +154,8 @@ public static void main(String[] args) throws IOException { } } else if ((header.getChunkType() & TsFileConstant.VALUE_COLUMN_MASK) == TsFileConstant.VALUE_COLUMN_MASK) { // Value Chunk + PageValueReader valueReader = + PAGE_VALUE_READER_SERVICE.call(Type.fromTsDataType(header.getDataType())); int pointNum = 0; byte[] bitmap = null; if (pageData.hasRemaining()) { @@ -149,33 +172,7 @@ public static void main(String[] args) throws IOException { } continue; } - Object value; - switch (header.getDataType()) { - case BOOLEAN: - value = valueDecoder.readBoolean(pageData); - break; - case INT32: - case DATE: - value = valueDecoder.readInt(pageData); - break; - case INT64: - case TIMESTAMP: - value = valueDecoder.readLong(pageData); - break; - case FLOAT: - value = valueDecoder.readFloat(pageData); - break; - case DOUBLE: - value = valueDecoder.readDouble(pageData); - break; - case TEXT: - case STRING: - case BLOB: - value = valueDecoder.readBinary(pageData); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(header.getDataType())); - } + Object value = valueReader.read(valueDecoder, pageData); if (printDetail) { System.out.println("\t\t\tvalue: " + value); } @@ -240,4 +237,9 @@ public static void main(String[] args) throws IOException { } } } + + @FunctionalInterface + private interface PageValueReader { + Object read(Decoder decoder, ByteBuffer buffer); + } } diff --git a/java/pom.xml b/java/pom.xml index 3106d99bb..00a18d68d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -188,7 +188,7 @@ org.apache.tsfile,,javax,java,\# - + UNIX diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java b/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java index a45de52f9..75145f5e0 100644 --- a/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java +++ b/java/tools/src/main/java/org/apache/tsfile/tools/ValueConverter.java @@ -19,6 +19,8 @@ package org.apache.tsfile.tools; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import java.nio.charset.StandardCharsets; @@ -30,6 +32,102 @@ public class ValueConverter { + private static final TypeService FROM_STRING_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (value, isMeasurement, timePrecision) -> Boolean.valueOf(value); + case INT32 -> (value, isMeasurement, timePrecision) -> Integer.valueOf(value); + case INT64 -> (value, isMeasurement, timePrecision) -> Long.valueOf(value); + case FLOAT -> (value, isMeasurement, timePrecision) -> Float.valueOf(value); + case DOUBLE -> (value, isMeasurement, timePrecision) -> Double.valueOf(value); + case TEXT, STRING -> + (value, isMeasurement, timePrecision) -> + isMeasurement ? new Binary(value, StandardCharsets.UTF_8) : value; + case BLOB -> + (value, isMeasurement, timePrecision) -> + new Binary(value.getBytes(StandardCharsets.UTF_8)); + case DATE -> (value, isMeasurement, timePrecision) -> parseDate(value); + case TIMESTAMP -> + (value, isMeasurement, timePrecision) -> parseTimestamp(value, timePrecision); + case OBJECT, ROW, UNKNOWN, VECTOR -> (value, isMeasurement, timePrecision) -> value; + }; + + private static final TypeService FROM_OBJECT_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (value, isMeasurement, timePrecision) -> + value instanceof Boolean ? value : Boolean.valueOf(value.toString()); + case INT32 -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Integer) { + return value; + } + if (value instanceof Number) { + return ((Number) value).intValue(); + } + return Integer.valueOf(value.toString()); + }; + case INT64 -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Long) { + return value; + } + if (value instanceof Number) { + return ((Number) value).longValue(); + } + return Long.valueOf(value.toString()); + }; + case FLOAT -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Float) { + return value; + } + if (value instanceof Number) { + return ((Number) value).floatValue(); + } + return Float.valueOf(value.toString()); + }; + case DOUBLE -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Double) { + return value; + } + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + return Double.valueOf(value.toString()); + }; + case TEXT, STRING -> + (value, isMeasurement, timePrecision) -> { + if (isMeasurement) { + return value instanceof Binary + ? value + : new Binary(value.toString(), StandardCharsets.UTF_8); + } + return value instanceof String ? value : value.toString(); + }; + case BLOB -> + (value, isMeasurement, timePrecision) -> { + if (value instanceof Binary) { + return value; + } + if (value instanceof byte[]) { + return new Binary((byte[]) value); + } + return new Binary(value.toString().getBytes(StandardCharsets.UTF_8)); + }; + case DATE -> (value, isMeasurement, timePrecision) -> toLocalDate(value); + case TIMESTAMP -> + (value, isMeasurement, timePrecision) -> toEpochLong(value, timePrecision); + case OBJECT, ROW, UNKNOWN, VECTOR -> (value, isMeasurement, timePrecision) -> value; + }; + + static { + FROM_STRING_SERVICE.check(); + FROM_OBJECT_SERVICE.check(); + } + public static Object convert(Object value, TSDataType targetType, boolean isMeasurement) { return convert(value, targetType, isMeasurement, "ms"); } @@ -47,101 +145,16 @@ public static Object convert( private static Object fromString( String value, TSDataType targetType, boolean isMeasurement, String timePrecision) { - switch (targetType) { - case BOOLEAN: - return Boolean.valueOf(value); - case INT32: - return Integer.valueOf(value); - case INT64: - return Long.valueOf(value); - case FLOAT: - return Float.valueOf(value); - case DOUBLE: - return Double.valueOf(value); - case TEXT: - case STRING: - if (isMeasurement) { - return new Binary(value, StandardCharsets.UTF_8); - } - return value; - case BLOB: - return new Binary(value.getBytes(StandardCharsets.UTF_8)); - case DATE: - return parseDate(value); - case TIMESTAMP: - return parseTimestamp(value, timePrecision); - default: - return value; - } + return FROM_STRING_SERVICE + .call(Type.fromTsDataType(targetType)) + .convert(value, isMeasurement, timePrecision); } private static Object fromObject( Object value, TSDataType targetType, boolean isMeasurement, String timePrecision) { - switch (targetType) { - case BOOLEAN: - if (value instanceof Boolean) { - return value; - } - return Boolean.valueOf(value.toString()); - case INT32: - if (value instanceof Integer) { - return value; - } - if (value instanceof Number) { - return ((Number) value).intValue(); - } - return Integer.valueOf(value.toString()); - case INT64: - if (value instanceof Long) { - return value; - } - if (value instanceof Number) { - return ((Number) value).longValue(); - } - return Long.valueOf(value.toString()); - case FLOAT: - if (value instanceof Float) { - return value; - } - if (value instanceof Number) { - return ((Number) value).floatValue(); - } - return Float.valueOf(value.toString()); - case DOUBLE: - if (value instanceof Double) { - return value; - } - if (value instanceof Number) { - return ((Number) value).doubleValue(); - } - return Double.valueOf(value.toString()); - case TEXT: - case STRING: - if (isMeasurement) { - if (value instanceof Binary) { - return value; - } - return new Binary(value.toString(), StandardCharsets.UTF_8); - } - if (value instanceof String) { - return value; - } - return value.toString(); - case BLOB: - if (value instanceof Binary) { - return value; - } - if (value instanceof byte[]) { - return new Binary((byte[]) value); - } - return new Binary(value.toString().getBytes(StandardCharsets.UTF_8)); - case DATE: - return toLocalDate(value); - case TIMESTAMP: - return toEpochLong(value, timePrecision); - default: - return value; - } + return FROM_OBJECT_SERVICE + .call(Type.fromTsDataType(targetType)) + .convert(value, isMeasurement, timePrecision); } private static LocalDate parseDate(String value) { @@ -220,4 +233,14 @@ private static long toEpochLong(Object value, String timePrecision) { } return parseTimestamp(value.toString(), timePrecision); } + + @FunctionalInterface + private interface StringConverter { + Object convert(String value, boolean isMeasurement, String timePrecision); + } + + @FunctionalInterface + private interface ObjectConverter { + Object convert(Object value, boolean isMeasurement, String timePrecision); + } } diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml index ec327381c..768d23f95 100644 --- a/java/tsfile/pom.xml +++ b/java/tsfile/pom.xml @@ -145,10 +145,10 @@ shade - + false - + @@ -185,7 +185,7 @@ org.apache.tsfile.* common;inline=true false - + <_removeheaders>Bnd-LastModified,Built-By org.apache.tsfile diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index 764eda5bd..6c79d6567 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@ -24,6 +24,7 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.FSUtils; import java.io.Serializable; @@ -370,62 +371,11 @@ public String getValueEncoder() { } public TSEncoding getValueEncoder(TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return booleanEncoding; - case INT32: - case DATE: - return int32Encoding; - case INT64: - case TIMESTAMP: - return int64Encoding; - case FLOAT: - return floatEncoding; - case DOUBLE: - return doubleEncoding; - case STRING: - case BLOB: - case TEXT: - case OBJECT: - default: - return textEncoding; - } + return Type.fromTsDataType(dataType).getDefaultEncoding(this); } public CompressionType getCompressor(TSDataType dataType) { - CompressionType compressionType; - switch (dataType) { - case BOOLEAN: - compressionType = booleanCompression; - break; - case INT32: - case DATE: - compressionType = int32Compression; - break; - case INT64: - case TIMESTAMP: - compressionType = int64Compression; - break; - case FLOAT: - compressionType = floatCompression; - break; - case DOUBLE: - compressionType = doubleCompression; - break; - case STRING: - case BLOB: - case TEXT: - case OBJECT: - compressionType = textCompression; - break; - default: - compressionType = null; - } - - if (compressionType == null) { - compressionType = compressor; - } - return compressionType; + return Type.fromTsDataType(dataType).getDefaultCompressor(this); } public void setValueEncoder(String valueEncoder) { @@ -532,6 +482,34 @@ public CompressionType getCompressor() { return compressor; } + public CompressionType getBooleanCompressor() { + return getCompressorOrDefault(booleanCompression); + } + + public CompressionType getInt32Compressor() { + return getCompressorOrDefault(int32Compression); + } + + public CompressionType getInt64Compressor() { + return getCompressorOrDefault(int64Compression); + } + + public CompressionType getFloatCompressor() { + return getCompressorOrDefault(floatCompression); + } + + public CompressionType getDoubleCompressor() { + return getCompressorOrDefault(doubleCompression); + } + + public CompressionType getTextCompressor() { + return getCompressorOrDefault(textCompression); + } + + private CompressionType getCompressorOrDefault(CompressionType compressionType) { + return compressionType == null ? compressor : compressionType; + } + public void setCompressor(String compressor) { this.compressor = CompressionType.valueOf(compressor); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java index 340316bb1..bfa1218ab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java @@ -23,6 +23,7 @@ import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import java.io.IOException; @@ -31,8 +32,6 @@ public abstract class Decoder { - private static final String ERROR_MSG_KEY = "error.encoding.decoder_not_found"; - private TSEncoding type; public Decoder(TSEncoding type) { @@ -48,146 +47,7 @@ public TSEncoding getType() { } public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) { - switch (encoding) { - case PLAIN: - return new PlainDecoder(); - case RLE: - switch (dataType) { - case BOOLEAN: - case INT32: - case DATE: - return new IntRleDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new LongRleDecoder(); - case FLOAT: - case DOUBLE: - return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case TS_2DIFF: - switch (dataType) { - case INT32: - case DATE: - return new DeltaBinaryDecoder.IntDeltaDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new DeltaBinaryDecoder.LongDeltaDecoder(); - case FLOAT: - case DOUBLE: - return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case GORILLA_V1: - switch (dataType) { - case FLOAT: - return new SinglePrecisionDecoderV1(); - case DOUBLE: - return new DoublePrecisionDecoderV1(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case REGULAR: - switch (dataType) { - case INT32: - case DATE: - return new RegularDataDecoder.IntRegularDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new RegularDataDecoder.LongRegularDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case GORILLA: - switch (dataType) { - case FLOAT: - return new SinglePrecisionDecoderV2(); - case DOUBLE: - return new DoublePrecisionDecoderV2(); - case INT32: - case DATE: - return new IntGorillaDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new LongGorillaDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case DICTIONARY: - return new DictionaryDecoder(); - case ZIGZAG: - switch (dataType) { - case INT32: - case DATE: - return new IntZigzagDecoder(); - case INT64: - case TIMESTAMP: - return new LongZigzagDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case CHIMP: - switch (dataType) { - case FLOAT: - return new SinglePrecisionChimpDecoder(); - case DOUBLE: - return new DoublePrecisionChimpDecoder(); - case INT32: - case DATE: - return new IntChimpDecoder(); - case INT64: - case VECTOR: - case TIMESTAMP: - return new LongChimpDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case SPRINTZ: - switch (dataType) { - case INT32: - case DATE: - return new IntSprintzDecoder(); - case INT64: - case TIMESTAMP: - return new LongSprintzDecoder(); - case FLOAT: - return new FloatSprintzDecoder(); - case DOUBLE: - return new DoubleSprintzDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case RLBE: - switch (dataType) { - case INT32: - case DATE: - return new IntRLBEDecoder(); - case INT64: - case TIMESTAMP: - return new LongRLBEDecoder(); - case FLOAT: - return new FloatRLBEDecoder(); - case DOUBLE: - return new DoubleRLBEDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - case CAMEL: - switch (dataType) { - case DOUBLE: - return new CamelDecoder(); - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } - default: - throw new TsFileDecodingException(Messages.format(ERROR_MSG_KEY, encoding, dataType)); - } + return Type.fromTsDataType(dataType).getDecoder(encoding); } public int readInt(ByteBuffer buffer) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java index 0f9fd8206..f2b518c61 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java @@ -19,11 +19,11 @@ package org.apache.tsfile.encoding.encoder; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.encoding.TsFileEncodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -112,28 +112,7 @@ public void flush(ByteArrayOutputStream out) { @Override public int getOneItemMaxSize() { - switch (dataType) { - case BOOLEAN: - return 1; - case INT32: - case DATE: - return 4; - case INT64: - case TIMESTAMP: - return 8; - case FLOAT: - return 4; - case DOUBLE: - return 8; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - // refer to encode(Binary,ByteArrayOutputStream) - return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength; - default: - throw new UnsupportedOperationException(dataType.toString()); - } + return Type.fromTsDataType(dataType).getOneItemMaxSize(maxStringLength); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index 77d6568ee..0db1cc211 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -25,6 +25,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.EncodingTypeServices; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.slf4j.Logger; @@ -143,21 +145,9 @@ public static class Rle extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - case BOOLEAN: - return new IntRleEncoder(); - case INT64: - case TIMESTAMP: - return new LongRleEncoder(); - case FLOAT: - case DOUBLE: - return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.RLE, type)); - } + return EncodingTypeServices.RLE_ENCODER_SERVICE + .call(Type.fromTsDataType(type)) + .build(maxPointNumber); } /** @@ -198,20 +188,9 @@ public static class Ts2Diff extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new DeltaBinaryEncoder.IntDeltaEncoder(); - case INT64: - case TIMESTAMP: - return new DeltaBinaryEncoder.LongDeltaEncoder(); - case FLOAT: - case DOUBLE: - return new FloatEncoder(TSEncoding.TS_2DIFF, type, maxPointNumber); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.TS_2DIFF, type)); - } + return EncodingTypeServices.TS_2DIFF_ENCODER_SERVICE + .call(Type.fromTsDataType(type)) + .build(maxPointNumber); } /** @@ -250,15 +229,7 @@ public static class GorillaV1 extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case FLOAT: - return new SinglePrecisionEncoderV1(); - case DOUBLE: - return new DoublePrecisionEncoderV1(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.GORILLA_V1, type)); - } + return EncodingTypeServices.GORILLA_V1_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -290,17 +261,7 @@ public static class Regular extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new RegularDataEncoder.IntRegularEncoder(); - case INT64: - case TIMESTAMP: - return new RegularDataEncoder.LongRegularEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.REGULAR, type)); - } + return EncodingTypeServices.REGULAR_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -314,21 +275,7 @@ public static class GorillaV2 extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case FLOAT: - return new SinglePrecisionEncoderV2(); - case DOUBLE: - return new DoublePrecisionEncoderV2(); - case INT32: - case DATE: - return new IntGorillaEncoder(); - case INT64: - case TIMESTAMP: - return new LongGorillaEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.GORILLA, type)); - } + return EncodingTypeServices.GORILLA_V2_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -340,21 +287,7 @@ public void initFromProps(Map props) { public static class Sprintz extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new IntSprintzEncoder(); - case INT64: - case TIMESTAMP: - return new LongSprintzEncoder(); - case FLOAT: - return new FloatSprintzEncoder(); - case DOUBLE: - return new DoubleSprintzEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.SPRINTZ, type)); - } + return EncodingTypeServices.SPRINTZ_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -367,21 +300,7 @@ public static class RLBE extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new IntRLBE(); - case INT64: - case TIMESTAMP: - return new LongRLBE(); - case FLOAT: - return new FloatRLBE(); - case DOUBLE: - return new DoubleRLBE(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.RLBE, type)); - } + return EncodingTypeServices.RLBE_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -411,17 +330,7 @@ public static class Zigzag extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case INT32: - case DATE: - return new IntZigzagEncoder(); - case INT64: - case TIMESTAMP: - return new LongZigzagEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.ZIGZAG, type)); - } + return EncodingTypeServices.ZIGZAG_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override @@ -435,21 +344,7 @@ public static class Chimp extends TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - switch (type) { - case FLOAT: - return new SinglePrecisionChimpEncoder(); - case DOUBLE: - return new DoublePrecisionChimpEncoder(); - case INT32: - case DATE: - return new IntChimpEncoder(); - case INT64: - case TIMESTAMP: - return new LongChimpEncoder(); - default: - throw new UnSupportedDataTypeException( - Messages.format(ERROR_MSG_KEY, TSEncoding.CHIMP, type)); - } + return EncodingTypeServices.CHIMP_ENCODER_SERVICE.call(Type.fromTsDataType(type)); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java index 323115aee..2b9ff6f29 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java @@ -23,6 +23,7 @@ import org.apache.tsfile.exception.filter.StatisticsClassException; import org.apache.tsfile.exception.write.UnknownColumnTypeException; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; @@ -104,65 +105,11 @@ public abstract class Statistics { * @throws UnknownColumnTypeException if the type is unknown */ public static Statistics getStatsByType(TSDataType type) { - switch (type) { - case INT32: - return new IntegerStatistics(); - case INT64: - return new LongStatistics(); - case TEXT: - return new BinaryStatistics(); - case BOOLEAN: - return new BooleanStatistics(); - case DOUBLE: - return new DoubleStatistics(); - case FLOAT: - return new FloatStatistics(); - case VECTOR: - return new TimeStatistics(); - case DATE: - return new DateStatistics(); - case TIMESTAMP: - return new TimestampStatistics(); - case STRING: - return new StringStatistics(); - case BLOB: - return new BlobStatistics(); - case OBJECT: - return new ObjectStatistics(); - default: - throw new UnknownColumnTypeException(type.toString()); - } + return Type.fromTsDataType(type).createStatistics(); } public static long getSizeByType(TSDataType type) { - switch (type) { - case INT32: - return IntegerStatistics.INSTANCE_SIZE; - case INT64: - return LongStatistics.INSTANCE_SIZE; - case TEXT: - return BinaryStatistics.INSTANCE_SIZE; - case BOOLEAN: - return BooleanStatistics.INSTANCE_SIZE; - case DOUBLE: - return DoubleStatistics.INSTANCE_SIZE; - case FLOAT: - return FloatStatistics.INSTANCE_SIZE; - case VECTOR: - return TimeStatistics.INSTANCE_SIZE; - case DATE: - return DateStatistics.INSTANCE_SIZE; - case TIMESTAMP: - return TimestampStatistics.INSTANCE_SIZE; - case STRING: - return StringStatistics.INSTANCE_SIZE; - case BLOB: - return BlobStatistics.INSTANCE_SIZE; - case OBJECT: - return ObjectStatistics.INSTANCE_SIZE; - default: - throw new UnknownColumnTypeException(type.toString()); - } + return Type.fromTsDataType(type).getStatisticsInstanceSize(); } public abstract TSDataType getType(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 97dcddc0d..396a75576 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -66,6 +66,7 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.tsfile.read.reader.TsFileInput; @@ -2457,6 +2458,7 @@ public long selfCheck( chunkHeader.getCompressionType()); measurementSchemaList.add(measurementSchema); dataType = chunkHeader.getDataType(); + Type valueType = Type.fromTsDataType(dataType); Statistics chunkStatistics = Statistics.getStatsByType(dataType); @@ -2534,34 +2536,7 @@ public long selfCheck( continue; } long timeStamp = timeBatch.get(timeBatchIndex)[i]; - switch (dataType) { - case INT32: - case DATE: - chunkStatistics.update(timeStamp, value.getInt()); - break; - case INT64: - case TIMESTAMP: - chunkStatistics.update(timeStamp, value.getLong()); - break; - case FLOAT: - chunkStatistics.update(timeStamp, value.getFloat()); - break; - case DOUBLE: - chunkStatistics.update(timeStamp, value.getDouble()); - break; - case BOOLEAN: - chunkStatistics.update(timeStamp, value.getBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkStatistics.update(timeStamp, value.getBinary()); - break; - default: - throw new IOException( - Messages.format("error.read.unexpected_type", dataType)); - } + valueType.update(chunkStatistics, timeStamp, value); } } @@ -2575,34 +2550,7 @@ public long selfCheck( timeDecoder); BatchData batchData = reader.getAllSatisfiedPageData(); while (batchData.hasCurrent()) { - switch (dataType) { - case INT32: - case DATE: - chunkStatistics.update(batchData.currentTime(), batchData.getInt()); - break; - case INT64: - case TIMESTAMP: - chunkStatistics.update(batchData.currentTime(), batchData.getLong()); - break; - case FLOAT: - chunkStatistics.update(batchData.currentTime(), batchData.getFloat()); - break; - case DOUBLE: - chunkStatistics.update(batchData.currentTime(), batchData.getDouble()); - break; - case BOOLEAN: - chunkStatistics.update(batchData.currentTime(), batchData.getBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkStatistics.update(batchData.currentTime(), batchData.getBinary()); - break; - default: - throw new IOException( - Messages.format("error.read.unexpected_type", dataType)); - } + valueType.update(chunkStatistics, batchData); batchData.next(); } } @@ -2781,6 +2729,7 @@ public long checkChunkAndPagesStatistics(IChunkMetadata chunkMetadata) throws IO byte marker = this.readMarker(); ChunkHeader chunkHeader = this.readChunkHeader(marker); TSDataType dataType = chunkHeader.getDataType(); + Type valueType = Type.fromTsDataType(dataType); Statistics chunkStatistics = Statistics.getStatsByType(dataType); int dataSize = chunkHeader.getDataSize(); if (((byte) (chunkHeader.getChunkType() & 0x3F)) == MetaMarker.CHUNK_HEADER) { @@ -2808,33 +2757,7 @@ public long checkChunkAndPagesStatistics(IChunkMetadata chunkMetadata) throws IO pageHeader, pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder); BatchData batchData = reader.getAllSatisfiedPageData(); while (batchData.hasCurrent()) { - switch (dataType) { - case INT32: - case DATE: - chunkStatistics.update(batchData.currentTime(), batchData.getInt()); - break; - case INT64: - case TIMESTAMP: - chunkStatistics.update(batchData.currentTime(), batchData.getLong()); - break; - case FLOAT: - chunkStatistics.update(batchData.currentTime(), batchData.getFloat()); - break; - case DOUBLE: - chunkStatistics.update(batchData.currentTime(), batchData.getDouble()); - break; - case BOOLEAN: - chunkStatistics.update(batchData.currentTime(), batchData.getBoolean()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkStatistics.update(batchData.currentTime(), batchData.getBinary()); - break; - default: - throw new IOException(Messages.format("error.read.unexpected_type", dataType)); - } + valueType.update(chunkStatistics, batchData); batchData.next(); } chunkHeader.increasePageNums(1); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index 9fbda97f1..7633a62fd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -23,17 +23,10 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.utils.TsPrimitiveType.TsBinary; -import org.apache.tsfile.utils.TsPrimitiveType.TsBoolean; -import org.apache.tsfile.utils.TsPrimitiveType.TsDouble; -import org.apache.tsfile.utils.TsPrimitiveType.TsFloat; -import org.apache.tsfile.utils.TsPrimitiveType.TsInt; -import org.apache.tsfile.utils.TsPrimitiveType.TsLong; -import org.apache.tsfile.utils.TsPrimitiveType.TsVector; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.DataOutputStream; import java.io.IOException; @@ -129,55 +122,12 @@ public long currentTime() { * @return current value */ public Object currentValue() { - switch (dataType) { - case INT32: - case DATE: - return getInt(); - case INT64: - case TIMESTAMP: - return getLong(); - case FLOAT: - return getFloat(); - case DOUBLE: - return getDouble(); - case BOOLEAN: - return getBoolean(); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return getBinary(); - case VECTOR: - return getVector(); - default: - return null; - } + return Type.fromTsDataType(dataType).getCurrentValue(this); } public TsPrimitiveType currentTsPrimitiveType() { - switch (dataType) { - case INT32: - case DATE: - return new TsInt(getInt()); - case INT64: - case TIMESTAMP: - return new TsLong(getLong()); - case FLOAT: - return new TsFloat(getFloat()); - case DOUBLE: - return new TsDouble(getDouble()); - case BOOLEAN: - return new TsBoolean(getBoolean()); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new TsBinary(getBinary()); - case VECTOR: - return new TsVector(getVector()); - default: - return null; - } + Type type = Type.fromTsDataType(dataType); + return type.getTsPrimitiveType(type.getCurrentValue(this)); } public TSDataType getDataType() { @@ -208,43 +158,42 @@ public void init(TSDataType type) { timeRet.add(new long[capacity]); count = 0; - switch (dataType) { - case BOOLEAN: - booleanRet = new ArrayList<>(); - booleanRet.add(new boolean[capacity]); - break; - case INT32: - case DATE: - intRet = new ArrayList<>(); - intRet.add(new int[capacity]); - break; - case INT64: - case TIMESTAMP: - longRet = new ArrayList<>(); - longRet.add(new long[capacity]); - break; - case FLOAT: - floatRet = new ArrayList<>(); - floatRet.add(new float[capacity]); - break; - case DOUBLE: - doubleRet = new ArrayList<>(); - doubleRet.add(new double[capacity]); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - binaryRet = new ArrayList<>(); - binaryRet.add(new Binary[capacity]); - break; - case VECTOR: - vectorRet = new ArrayList<>(); - vectorRet.add(new TsPrimitiveType[capacity][]); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).init(this); + } + + public void initBooleanValues() { + booleanRet = new ArrayList<>(); + booleanRet.add(new boolean[capacity]); + } + + public void initIntValues() { + intRet = new ArrayList<>(); + intRet.add(new int[capacity]); + } + + public void initLongValues() { + longRet = new ArrayList<>(); + longRet.add(new long[capacity]); + } + + public void initFloatValues() { + floatRet = new ArrayList<>(); + floatRet.add(new float[capacity]); + } + + public void initDoubleValues() { + doubleRet = new ArrayList<>(); + doubleRet.add(new double[capacity]); + } + + public void initBinaryValues() { + binaryRet = new ArrayList<>(); + binaryRet.add(new Binary[capacity]); + } + + public void initVectorValues() { + vectorRet = new ArrayList<>(); + vectorRet.add(new TsPrimitiveType[capacity][]); } /** @@ -558,36 +507,7 @@ public void setTime(long v) { * @param v object */ public void putAnObject(long t, Object v) { - switch (dataType) { - case BOOLEAN: - putBoolean(t, (boolean) v); - break; - case INT32: - case DATE: - putInt(t, (int) v); - break; - case INT64: - case TIMESTAMP: - putLong(t, (long) v); - break; - case FLOAT: - putFloat(t, (float) v); - break; - case DOUBLE: - putDouble(t, (double) v); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - putBinary(t, (Binary) v); - break; - case VECTOR: - putVector(t, (TsPrimitiveType[]) v); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).put(this, t, v); } public int length() { @@ -682,99 +602,7 @@ public IBatchDataIterator getBatchDataIterator(int subIndex) { * writing, otherwise after deserializing the sequence will be reversed */ public void serializeData(DataOutputStream outputStream) throws IOException { - switch (dataType) { - case BOOLEAN: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeBoolean(getBooleanByIndex(i)); - } - break; - case DOUBLE: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeDouble(getDoubleByIndex(i)); - } - break; - case FLOAT: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeFloat(getFloatByIndex(i)); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - Binary binary = getBinaryByIndex(i); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - } - break; - case INT64: - case TIMESTAMP: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeLong(getLongByIndex(i)); - } - break; - case INT32: - case DATE: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeInt(getIntByIndex(i)); - } - break; - case VECTOR: - for (int i = 0; i < length(); i++) { - outputStream.writeLong(getTimeByIndex(i)); - TsPrimitiveType[] values = getVectorByIndex(i); - outputStream.writeInt(values.length); - for (TsPrimitiveType value : values) { - if (value == null) { - outputStream.write(0); - } else { - outputStream.write(1); - outputStream.write(value.getDataType().serialize()); - switch (value.getDataType()) { - case BOOLEAN: - outputStream.writeBoolean(value.getBoolean()); - break; - case DOUBLE: - outputStream.writeDouble(value.getDouble()); - break; - case FLOAT: - outputStream.writeFloat(value.getFloat()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary binary = value.getBinary(); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - break; - case INT64: - case TIMESTAMP: - outputStream.writeLong(value.getLong()); - break; - case INT32: - case DATE: - outputStream.writeInt(value.getInt()); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.batch_data_unknown_type", dataType)); - } - } - } - } - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.batch_data_unknown_type", dataType)); - } + Type.fromTsDataType(dataType).serialize(this, outputStream, false); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java index bf83a4539..d41ad14a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java @@ -29,11 +29,11 @@ import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.read.reader.chunk.TableChunkReader; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -257,51 +257,8 @@ public Chunk rewrite(TSDataType newType, Chunk timeChunk) throws IOException { chunkHeader.getDataType(), point.getValue().getVector()[0].getValue()); } long timestamp = point.getTimestamp(); - switch (newType) { - case BOOLEAN: - chunkWriter.write( - timestamp, - convertedValue == null ? true : (boolean) convertedValue, - convertedValue == null); - break; - case DATE: - case INT32: - chunkWriter.write( - timestamp, - convertedValue == null ? Integer.MAX_VALUE : (int) convertedValue, - convertedValue == null); - break; - case TIMESTAMP: - case INT64: - chunkWriter.write( - timestamp, - convertedValue == null ? (long) Integer.MAX_VALUE : (long) convertedValue, - convertedValue == null); - break; - case FLOAT: - chunkWriter.write( - timestamp, - convertedValue == null ? (float) Integer.MAX_VALUE : (float) convertedValue, - convertedValue == null); - break; - case DOUBLE: - chunkWriter.write( - timestamp, - convertedValue == null ? (double) Integer.MAX_VALUE : (double) convertedValue, - convertedValue == null); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - chunkWriter.write( - timestamp, - convertedValue == null ? Binary.EMPTY_VALUE : (Binary) convertedValue, - convertedValue == null); - break; - default: - throw new IOException(Messages.format("error.read.chunk_unsupported_type", newType)); - } + Type.fromTsDataType(newType) + .write(chunkWriter, timestamp, convertedValue, convertedValue == null); } chunkWriter.sealCurrentPage(); } @@ -346,33 +303,7 @@ public Chunk rewrite(TSDataType newType) throws IOException { if (convertedValue == null) { throw new IOException(Messages.format("error.read.chunk_non_aligned_null", timestamp)); } - switch (newType) { - case BOOLEAN: - chunkWriter.write(timestamp, (boolean) convertedValue); - break; - case DATE: - case INT32: - chunkWriter.write(timestamp, (int) convertedValue); - break; - case TIMESTAMP: - case INT64: - chunkWriter.write(timestamp, (long) convertedValue); - break; - case FLOAT: - chunkWriter.write(timestamp, (float) convertedValue); - break; - case DOUBLE: - chunkWriter.write(timestamp, (double) convertedValue); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - chunkWriter.write(timestamp, (Binary) convertedValue); - break; - default: - throw new IOException(Messages.format("error.read.chunk_unsupported_type", newType)); - } + Type.fromTsDataType(newType).write(chunkWriter, timestamp, convertedValue); } chunkWriter.sealCurrentPage(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java index b667c537d..78ec4ed3c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java @@ -20,9 +20,9 @@ package org.apache.tsfile.read.common; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.DataOutputStream; import java.io.IOException; @@ -49,43 +49,7 @@ public DescReadWriteBatchData(TSDataType dataType) { timeRet.add(new long[capacity]); count = 0; - switch (dataType) { - case BOOLEAN: - booleanRet = new LinkedList<>(); - booleanRet.add(new boolean[capacity]); - break; - case INT32: - case DATE: - intRet = new LinkedList<>(); - intRet.add(new int[capacity]); - break; - case INT64: - case TIMESTAMP: - longRet = new LinkedList<>(); - longRet.add(new long[capacity]); - break; - case FLOAT: - floatRet = new LinkedList<>(); - floatRet.add(new float[capacity]); - break; - case DOUBLE: - doubleRet = new LinkedList<>(); - doubleRet.add(new double[capacity]); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - binaryRet = new LinkedList<>(); - binaryRet.add(new Binary[capacity]); - break; - case VECTOR: - vectorRet = new LinkedList<>(); - vectorRet.add(new TsPrimitiveType[capacity][]); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).init(this); } /** @@ -419,97 +383,7 @@ public TsPrimitiveType[] getVectorByIndex(int idx) { @Override public void serializeData(DataOutputStream outputStream) throws IOException { - switch (dataType) { - case BOOLEAN: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeBoolean(getBooleanByIndex(i)); - } - break; - case DOUBLE: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeDouble(getDoubleByIndex(i)); - } - break; - case FLOAT: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeFloat(getFloatByIndex(i)); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - Binary binary = getBinaryByIndex(i); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - } - break; - case INT64: - case TIMESTAMP: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeLong(getLongByIndex(i)); - } - break; - case INT32: - case DATE: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - outputStream.writeInt(getIntByIndex(i)); - } - break; - case VECTOR: - for (int i = length() - 1; i >= 0; i--) { - outputStream.writeLong(getTimeByIndex(i)); - TsPrimitiveType[] values = getVectorByIndex(i); - outputStream.writeInt(values.length); - for (TsPrimitiveType value : values) { - if (value == null) { - outputStream.write(0); - } else { - outputStream.write(1); - outputStream.write(value.getDataType().serialize()); - switch (value.getDataType()) { - case BOOLEAN: - outputStream.writeBoolean(value.getBoolean()); - break; - case DOUBLE: - outputStream.writeDouble(value.getDouble()); - break; - case FLOAT: - outputStream.writeFloat(value.getFloat()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary binary = value.getBinary(); - outputStream.writeInt(binary.getLength()); - outputStream.write(binary.getValues()); - break; - case INT64: - case TIMESTAMP: - outputStream.writeLong(value.getLong()); - break; - case INT32: - case DATE: - outputStream.writeInt(value.getInt()); - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } - } - } - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + Type.fromTsDataType(dataType).serialize(this, outputStream, true); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java index 1e8a3afcc..21e192a07 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java @@ -21,17 +21,13 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NullFieldException; -import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.time.LocalDate; -import static org.apache.tsfile.utils.BytesUtils.parseObjectByteArrayToString; - /** * Field is component of one {@code RowRecord} which stores a value in specific data type. The value * type of Field is primitive(int long, float, double, binary, boolean). @@ -53,33 +49,7 @@ public Field(TSDataType dataType) { public static Field copy(Field field) { Field out = new Field(field.dataType); if (out.dataType != null) { - switch (out.dataType) { - case DOUBLE: - out.setDoubleV(field.getDoubleV()); - break; - case FLOAT: - out.setFloatV(field.getFloatV()); - break; - case INT64: - case TIMESTAMP: - out.setLongV(field.getLongV()); - break; - case INT32: - case DATE: - out.setIntV(field.getIntV()); - break; - case BOOLEAN: - out.setBoolV(field.getBoolV()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - out.setBinaryV(field.getBinaryV()); - break; - default: - throw new UnSupportedDataTypeException(out.dataType.toString()); - } + Type.fromTsDataType(out.dataType).setTo(field, out); } return out; @@ -147,9 +117,6 @@ public void setDoubleV(double doubleV) { public Binary getBinaryV() { if (dataType == null) { throw new NullFieldException(); - } else if (dataType == TSDataType.OBJECT) { - throw new UnsupportedOperationException( - Messages.get("error.read.field_object_type_no_binary")); } return binaryV; } @@ -174,29 +141,7 @@ public String getStringValue() { if (dataType == null) { return "null"; } - switch (dataType) { - case BOOLEAN: - return String.valueOf(boolV); - case INT32: - case DATE: - return String.valueOf(intV); - case INT64: - case TIMESTAMP: - return String.valueOf(longV); - case FLOAT: - return String.valueOf(floatV); - case DOUBLE: - return String.valueOf(doubleV); - case TEXT: - case STRING: - return binaryV.toString(); - case OBJECT: - return parseObjectByteArrayToString(binaryV.getValues()); - case BLOB: - return BytesUtils.parseBlobByteArrayToString(binaryV.getValues()); - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + return Type.fromTsDataType(dataType).toString(this); } @Override @@ -208,94 +153,17 @@ public Object getObjectValue(TSDataType dataType) { if (this.dataType == null) { return null; } - switch (dataType) { - case DOUBLE: - return getDoubleV(); - case FLOAT: - return getFloatV(); - case INT64: - case TIMESTAMP: - return getLongV(); - case INT32: - return getIntV(); - case DATE: - return getDateV(); - case BOOLEAN: - return getBoolV(); - case TEXT: - case BLOB: - case STRING: - return getBinaryV(); - case OBJECT: - return getStringValue(); - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } + return Type.fromTsDataType(dataType).getValue(this); } public static Field getField(Object value, TSDataType dataType) { if (value == null) { return null; } - Field field = new Field(dataType); - switch (dataType) { - case INT32: - case DATE: - field.setIntV((int) value); - break; - case INT64: - case TIMESTAMP: - field.setLongV((long) value); - break; - case FLOAT: - field.setFloatV((float) value); - break; - case DOUBLE: - field.setDoubleV((double) value); - break; - case BOOLEAN: - field.setBoolV((boolean) value); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - field.setBinaryV((Binary) value); - break; - default: - throw new UnSupportedDataTypeException(dataType.toString()); - } - return field; + return Type.fromTsDataType(dataType).getField(value); } public static void setTsPrimitiveValue(TsPrimitiveType value, Field field) { - switch (value.getDataType()) { - case BOOLEAN: - field.setBoolV(value.getBoolean()); - break; - case INT32: - case DATE: - field.setIntV(value.getInt()); - break; - case INT64: - case TIMESTAMP: - field.setLongV(value.getLong()); - break; - case FLOAT: - field.setFloatV(value.getFloat()); - break; - case DOUBLE: - field.setDoubleV(value.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - field.setBinaryV(value.getBinary()); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.common.unsupported_data_type", value.getDataType())); - } + Type.fromTsDataType(value.getDataType()).setTo(value, field); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java index 658024c14..de6158409 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java @@ -26,10 +26,10 @@ import org.apache.tsfile.read.common.IBatchDataIterator; import org.apache.tsfile.read.common.block.column.ColumnFactory; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.IMeasurementSchema; import java.util.Arrays; @@ -560,47 +560,9 @@ public void updateWithoutTimeColumn(int updateIdx, TsBlock sourceTsBlock, int so valueColumns[i].isNull()[updateIdx] = true; continue; } - switch (valueColumns[i].getDataType()) { - case BOOLEAN: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getBooleans()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getBoolean(sourceIndex); - break; - case INT32: - case DATE: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getInts()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getInt(sourceIndex); - break; - case INT64: - case TIMESTAMP: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getLongs()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getLong(sourceIndex); - break; - case FLOAT: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getFloats()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getFloat(sourceIndex); - break; - case DOUBLE: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getDoubles()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getDouble(sourceIndex); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueColumns[i].isNull()[updateIdx] = false; - valueColumns[i].getBinaries()[updateIdx] = - sourceTsBlock.getValueColumns()[i].getBinary(sourceIndex); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format( - "error.read.tsblock_unknown_datatype", valueColumns[i].getDataType())); - } + valueColumns[i].isNull()[updateIdx] = false; + Type.fromTsDataType(valueColumns[i].getDataType()) + .setTo(sourceTsBlock.getValueColumns()[i], sourceIndex, valueColumns[i], updateIdx); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java index 456167304..e8868d04b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java @@ -26,14 +26,9 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; -import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; -import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; -import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; -import org.apache.tsfile.read.common.block.column.IntColumnBuilder; -import org.apache.tsfile.read.common.block.column.LongColumnBuilder; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import java.util.List; @@ -104,54 +99,10 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List types) { valueColumnBuilders = new ColumnBuilder[types.size()]; int initialExpectedEntries = timeColumnBuilder.getPositionCount(); for (int i = 0; i < valueColumnBuilders.length; i++) { - switch (types.get(i)) { - case BOOLEAN: - valueColumnBuilders[i] = - new BooleanColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case INT32: - valueColumnBuilders[i] = - new IntColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), - initialExpectedEntries, - TSDataType.INT32); - break; - case DATE: - valueColumnBuilders[i] = - new IntColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), - initialExpectedEntries, - TSDataType.DATE); - break; - case INT64: - case TIMESTAMP: - valueColumnBuilders[i] = - new LongColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case FLOAT: - valueColumnBuilders[i] = - new FloatColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case DOUBLE: - valueColumnBuilders[i] = - new DoubleColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueColumnBuilders[i] = - new BinaryColumnBuilder( + valueColumnBuilders[i] = + Type.fromTsDataType(types.get(i)) + .createColumnBuilder( tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.tsblock_builder_unknown_type", types.get(i))); - } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java index 88f010381..6b67f87fb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BinaryColumn.java @@ -20,13 +20,18 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Optional; @@ -60,7 +65,7 @@ public BinaryColumn(int positionCount, Optional valueIsNull, Binary[] this(0, positionCount, valueIsNull.orElse(null), values); } - BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) { + public BinaryColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, Binary[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } @@ -123,6 +128,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.BINARY_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.TEXT) { + return this; + } + ColumnUtil.checkConversion(TSDataType.TEXT, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeBinary(getBinary(position)); + } + } + return builder.build(); + } + @Override public Binary getBinary(int position) { return values[position + arrayOffset]; @@ -143,6 +166,40 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBinary(getBinary(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + Binary value = values[index + arrayOffset]; + buffer.putInt(value.getLength()); + buffer.put(value.getValues()); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + Binary value = values[index + arrayOffset]; + stream.writeInt(value.getLength()); + stream.write(value.getValues()); + } + + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + Binary value = values[i + arrayOffset]; + output.writeInt(value.getLength()); + output.write(value.getValues()); + } + } + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); + } + return !that.isNull(thatPos) && getBinary(thisPos).equals(that.getBinary(thatPos)); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java index 318c450f7..38c6a8720 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java @@ -20,13 +20,18 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -91,6 +96,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.BYTE_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.BOOLEAN) { + return this; + } + ColumnUtil.checkConversion(TSDataType.BOOLEAN, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeBoolean(getBoolean(position)); + } + } + return builder.build(); + } + @Override public boolean getBoolean(int position) { return values[position + arrayOffset]; @@ -125,6 +148,48 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsBoolean(getBoolean(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.put(values[index + arrayOffset] ? (byte) 1 : (byte) 0); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeBoolean(values[index + arrayOffset]); + } + + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + int nonNullCount = 0; + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + nonNullCount++; + } + } + + byte[] packedValues = new byte[(nonNullCount + 7) / 8]; + int nonNullIndex = 0; + for (int i = 0; i < positionCount; i++) { + if (isNull(i)) { + continue; + } + if (values[i + arrayOffset]) { + packedValues[nonNullIndex / 8] |= (byte) (0b1000_0000 >>> (nonNullIndex % 8)); + } + nonNullIndex++; + } + output.write(packedValues); + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); + } + return !that.isNull(thatPos) && getBoolean(thisPos) == that.getBoolean(thatPos); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java index ed5b853ff..0f73c9b0e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java @@ -21,32 +21,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; public class ColumnFactory { public static Column create(TSDataType dataType, int initialCapacity) { - switch (dataType) { - case INT64: - case TIMESTAMP: - return new LongColumn(initialCapacity); - case DOUBLE: - return new DoubleColumn(initialCapacity); - case FLOAT: - return new FloatColumn(initialCapacity); - case TEXT: - case STRING: - case BLOB: - case OBJECT: - return new BinaryColumn(initialCapacity); - case INT32: - return new IntColumn(initialCapacity, TSDataType.INT32); - case DATE: - return new IntColumn(initialCapacity, TSDataType.DATE); - case BOOLEAN: - return new BooleanColumn(initialCapacity); - default: - throw new IllegalArgumentException( - Messages.format("error.read.column_factory_unsupported_type", dataType)); - } + Type type = Type.fromTsDataType(dataType); + return type.createColumnWithZeroPosition(initialCapacity); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java index 3c741db46..77b5746f6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnUtil.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; import java.util.Arrays; @@ -37,6 +38,14 @@ public class ColumnUtil { private ColumnUtil() {} + static void checkConversion(TSDataType sourceType, TSDataType targetType) { + requireNonNull(targetType, "type is null"); + if (!targetType.isCompatible(sourceType)) { + throw new ClassCastException( + Messages.format("error.common.unsupported_cast", sourceType, targetType)); + } + } + static void checkArrayRange(int[] array, int offset, int length) { requireNonNull(array, "array is null"); if (offset < 0 || length < 0 || offset + length > array.length) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java index af7f22bef..7ce598483 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DictionaryColumn.java @@ -27,6 +27,9 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -249,6 +252,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.DICTIONARY; } + @Override + public Column convertTo(TSDataType type) { + TSDataType sourceType = getDataType(); + if (type == sourceType) { + return this; + } + ColumnUtil.checkConversion(sourceType, type); + // Transform dictionary values only, preserving the ids and dictionary encoding. + return new DictionaryColumn( + idsOffset, + positionCount, + dictionary.convertTo(type), + ids, + isCompact(), + isSequentialIds, + randomDictionaryId()); + } + @Override public boolean mayHaveNull() { return mayHaveNull && dictionary.mayHaveNull(); @@ -427,6 +448,21 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return dictionary.getTsPrimitiveType(position); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + dictionary.writeTo(getId(index), buffer); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + dictionary.writeTo(getId(index), stream); + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return dictionary.arePositionsEqual(getId(thisPos), that, thatPos); + } + @Override public void reset() { throw new UnsupportedOperationException(getClass().getSimpleName()); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java index 4ac8d08ae..953f13730 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java @@ -20,13 +20,18 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -92,6 +97,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT64_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.DOUBLE) { + return this; + } + ColumnUtil.checkConversion(TSDataType.DOUBLE, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeDouble(getDouble(position)); + } + } + return builder.build(); + } + @Override public double getDouble(int position) { return values[position + arrayOffset]; @@ -126,6 +149,34 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsDouble(getDouble(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putDouble(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeDouble(values[index + arrayOffset]); + } + + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeLong(Double.doubleToLongBits(values[i + arrayOffset])); + } + } + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); + } + return !that.isNull(thatPos) && getDouble(thisPos) == that.getDouble(thatPos); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java index d8a31f110..77326c91d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java @@ -20,13 +20,18 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -93,6 +98,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT32_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.FLOAT) { + return this; + } + ColumnUtil.checkConversion(TSDataType.FLOAT, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeFloat(getFloat(position)); + } + } + return builder.build(); + } + @Override public float getFloat(int position) { return values[position + arrayOffset]; @@ -141,6 +164,34 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsFloat(getFloat(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putFloat(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeFloat(values[index + arrayOffset]); + } + + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeInt(Float.floatToIntBits(values[i + arrayOffset])); + } + } + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); + } + return !that.isNull(thatPos) && getFloat(thisPos) == that.getFloat(thatPos); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java index 3c6e9b1af..4e49502cd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import java.io.DataOutputStream; import java.io.IOException; @@ -39,42 +40,16 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun // | byte | list[byte] | list[int32] | // +---------------+-----------------+-------------+ - boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); - - switch (dataType) { - case INT32: - case DATE: - int[] intValues = new int[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - intValues[i] = input.getInt(); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - intValues[i] = input.getInt(); - } - } - } - return new IntColumn(0, positionCount, nullIndicators, intValues); - case FLOAT: - float[] floatValues = new float[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - floatValues[i] = Float.intBitsToFloat(input.getInt()); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - floatValues[i] = Float.intBitsToFloat(input.getInt()); - } - } - } - return new FloatColumn(0, positionCount, nullIndicators, floatValues); - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT32 + && dataType != TSDataType.DATE + && dataType != TSDataType.FLOAT) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + + boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); + return (Column) + Type.fromTsDataType(dataType).deserializeColumn(input, positionCount, nullIndicators); } @Override @@ -83,38 +58,12 @@ public void writeColumn(DataOutputStream output, Column column) throws IOExcepti ColumnEncoder.serializeNullIndicators(output, column); TSDataType dataType = column.getDataType(); - int positionCount = column.getPositionCount(); - switch (dataType) { - case INT32: - case DATE: - if (column.mayHaveNull()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeInt(column.getInt(i)); - } - } - } else { - for (int i = 0; i < positionCount; i++) { - output.writeInt(column.getInt(i)); - } - } - break; - case FLOAT: - if (column.mayHaveNull()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeInt(Float.floatToIntBits(column.getFloat(i))); - } - } - } else { - for (int i = 0; i < positionCount; i++) { - output.writeInt(Float.floatToIntBits(column.getFloat(i))); - } - } - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT32 + && dataType != TSDataType.DATE + && dataType != TSDataType.FLOAT) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + column.serializeWithoutNulls(output); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java index 0076c88f0..e7e3272c1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import java.io.DataOutputStream; import java.io.IOException; @@ -39,41 +40,16 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun // | byte | list[byte] | list[int64] | // +---------------+-----------------+-------------+ - boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); - switch (dataType) { - case INT64: - case TIMESTAMP: - long[] values = new long[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - values[i] = input.getLong(); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - values[i] = input.getLong(); - } - } - } - return new LongColumn(0, positionCount, nullIndicators, values); - case DOUBLE: - double[] doubleValues = new double[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - doubleValues[i] = Double.longBitsToDouble(input.getLong()); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - doubleValues[i] = Double.longBitsToDouble(input.getLong()); - } - } - } - return new DoubleColumn(0, positionCount, nullIndicators, doubleValues); - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT64 + && dataType != TSDataType.TIMESTAMP + && dataType != TSDataType.DOUBLE) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + + boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); + return (Column) + Type.fromTsDataType(dataType).deserializeColumn(input, positionCount, nullIndicators); } @Override @@ -82,26 +58,12 @@ public void writeColumn(DataOutputStream output, Column column) throws IOExcepti ColumnEncoder.serializeNullIndicators(output, column); TSDataType dataType = column.getDataType(); - int positionCount = column.getPositionCount(); - switch (dataType) { - case INT64: - case TIMESTAMP: - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeLong(column.getLong(i)); - } - } - break; - case DOUBLE: - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeLong(Double.doubleToLongBits(column.getDouble(i))); - } - } - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.col_encoder_invalid_type", dataType)); + if (dataType != TSDataType.INT64 + && dataType != TSDataType.TIMESTAMP + && dataType != TSDataType.DOUBLE) { + throw new IllegalArgumentException( + Messages.format("error.read.col_encoder_invalid_type", dataType)); } + column.serializeWithoutNulls(output); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index fbea95799..810450605 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -20,13 +20,18 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -72,7 +77,7 @@ public IntColumn( this.dataType = dataType; } - IntColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, int[] values) { + public IntColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, int[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } @@ -111,8 +116,9 @@ public TSDataType getDataType() { return dataType; } - public void modifyDataType(TSDataType dataType) { + public IntColumn modifyDataType(TSDataType dataType) { this.dataType = dataType; + return this; } @Override @@ -120,6 +126,28 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT32_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == dataType) { + return this; + } + ColumnUtil.checkConversion(dataType, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else if (dataType == TSDataType.DATE) { + // DATE has an int physical representation but a formatted text cast. + builder.writeBinary( + new Binary(TSDataType.getDateStringValue(getInt(position)), StandardCharsets.UTF_8)); + } else { + builder.writeInt(getInt(position)); + } + } + return builder.build(); + } + @Override public int getInt(int position) { return values[position + arrayOffset]; @@ -196,6 +224,34 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsInt(getInt(position), dataType); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putInt(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeInt(values[index + arrayOffset]); + } + + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeInt(values[i + arrayOffset]); + } + } + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); + } + return !that.isNull(thatPos) && getInt(thisPos) == that.getInt(thatPos); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 5bf01c752..7abedf523 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -20,13 +20,18 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -59,7 +64,7 @@ public LongColumn(int positionCount, Optional valueIsNull, long[] val this(0, positionCount, valueIsNull.orElse(null), values); } - LongColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, long[] values) { + public LongColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, long[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException(Messages.get("error.read.col_array_offset_negative")); } @@ -93,6 +98,24 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT64_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.INT64) { + return this; + } + ColumnUtil.checkConversion(TSDataType.INT64, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + if (isNull(position)) { + builder.appendNull(); + } else { + builder.writeLong(getLong(position)); + } + } + return builder.build(); + } + @Override public long getLong(int position) { return values[position + arrayOffset]; @@ -141,6 +164,34 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return new TsPrimitiveType.TsLong(getLong(position)); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putLong(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeLong(values[index + arrayOffset]); + } + + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + if (!isNull(i)) { + output.writeLong(values[i + arrayOffset]); + } + } + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + boolean thisIsNull = isNull(thisPos); + if (thisIsNull) { + return that.isNull(thatPos); + } + return !that.isNull(thatPos) && getLong(thisPos) == that.getLong(thatPos); + } + @Override public boolean mayHaveNull() { return valueIsNull != null; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index 75c32ad61..75fea0ed5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -23,8 +23,12 @@ import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; + import static java.util.Objects.requireNonNull; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkArrayRange; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkReadablePosition; @@ -61,6 +65,11 @@ public ColumnEncoding getEncoding() { throw new UnsupportedOperationException(getClass().getName()); } + @Override + public Column convertTo(TSDataType type) { + return create(type, positionCount); + } + @Override public boolean mayHaveNull() { return true; @@ -71,11 +80,31 @@ public boolean isNull(int position) { return true; } + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return that.isNull(thatPos); + } + @Override public boolean[] isNull() { throw new UnsupportedOperationException(getClass().getName()); } + @Override + public void serializeWithoutNulls(DataOutputStream output) { + // There are no non-null values to serialize. + } + + @Override + public void writeTo(int index, ByteBuffer buffer) { + // There is no value to serialize. + } + + @Override + public void writeTo(int index, DataOutputStream stream) { + // There is no value to serialize. + } + @Override public int getPositionCount() { return positionCount; @@ -139,28 +168,7 @@ public void reverse() { public static Column create(TSDataType dataType, int positionCount) { requireNonNull(dataType, "dataType is null"); - switch (dataType) { - case BOOLEAN: - return new RunLengthEncodedColumn(BooleanColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case INT32: - case DATE: - return new RunLengthEncodedColumn(IntColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case INT64: - case TIMESTAMP: - return new RunLengthEncodedColumn(LongColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case FLOAT: - return new RunLengthEncodedColumn(FloatColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case DOUBLE: - return new RunLengthEncodedColumn(DoubleColumnBuilder.NULL_VALUE_BLOCK, positionCount); - case TEXT: - case BLOB: - case STRING: - case OBJECT: - return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); - default: - throw new IllegalArgumentException( - Messages.format("error.read.null_col_unknown_type", dataType)); - } + return Type.fromTsDataType(dataType).createNullColumn(positionCount); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java index 4414e24de..8c39c9549 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/RunLengthEncodedColumn.java @@ -27,6 +27,9 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import static java.util.Objects.requireNonNull; @@ -76,6 +79,17 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.RLE; } + @Override + public Column convertTo(TSDataType type) { + TSDataType sourceType = getDataType(); + if (type == sourceType) { + return this; + } + ColumnUtil.checkConversion(sourceType, type); + // Transform the single stored value and keep the run-length representation. + return new RunLengthEncodedColumn(value.convertTo(type), positionCount); + } + @Override public boolean getBoolean(int position) { return value.getBoolean(0); @@ -165,6 +179,21 @@ public TsPrimitiveType getTsPrimitiveType(int position) { return value.getTsPrimitiveType(0); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + value.writeTo(0, buffer); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + value.writeTo(0, stream); + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return value.arePositionsEqual(0, that, thatPos); + } + @Override public boolean mayHaveNull() { return value.mayHaveNull(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java index 45053b971..d0dc19377 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/TimeColumn.java @@ -20,11 +20,16 @@ package org.apache.tsfile.read.common.block.column; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.RamUsageEstimator; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkArrayRange; @@ -80,6 +85,20 @@ public ColumnEncoding getEncoding() { return ColumnEncoding.INT64_ARRAY; } + @Override + public Column convertTo(TSDataType type) { + if (type == TSDataType.INT64) { + return this; + } + ColumnUtil.checkConversion(TSDataType.INT64, type); + + ColumnBuilder builder = Type.fromTsDataType(type).createColumnBuilder(positionCount); + for (int position = 0; position < positionCount; position++) { + builder.writeLong(getLong(position)); + } + return builder.build(); + } + @Override public long getLong(int position) { return values[position + arrayOffset]; @@ -90,6 +109,28 @@ public Object getObject(int position) { return getLong(position); } + @Override + public void writeTo(int index, ByteBuffer buffer) { + buffer.putLong(values[index + arrayOffset]); + } + + @Override + public void writeTo(int index, DataOutputStream stream) throws IOException { + stream.writeLong(values[index + arrayOffset]); + } + + @Override + public void serializeWithoutNulls(DataOutputStream output) throws IOException { + for (int i = 0; i < positionCount; i++) { + output.writeLong(values[i + arrayOffset]); + } + } + + @Override + public boolean arePositionsEqual(int thisPos, Column that, int thatPos) { + return !that.isNull(thatPos) && getLong(thisPos) == that.getLong(thatPos); + } + public boolean mayHaveNull() { return false; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java index 8425a8793..3d29a5b50 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractIntType.java @@ -21,13 +21,299 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; +import org.apache.tsfile.encoding.decoder.IntChimpDecoder; +import org.apache.tsfile.encoding.decoder.IntGorillaDecoder; +import org.apache.tsfile.encoding.decoder.IntRLBEDecoder; +import org.apache.tsfile.encoding.decoder.IntRleDecoder; +import org.apache.tsfile.encoding.decoder.IntSprintzDecoder; +import org.apache.tsfile.encoding.decoder.IntZigzagDecoder; +import org.apache.tsfile.encoding.decoder.RegularDataDecoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.IntegerStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public abstract class AbstractIntType extends AbstractType { + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.intToBytes(value.getInt(), valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return Integer.BYTES; + } + + @Override + public int calcTypeSize(Object value) { + return Integer.BYTES; + } + + @Override + public void init(BatchData batchData) { + batchData.initIntValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putInt(timestamp, (int) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeInt(batchData.getIntByIndex(index)); + } + return Math.multiplyExact(Long.BYTES + Integer.BYTES, batchData.length()); + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeInt(value.getInt()); + return Integer.BYTES; + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + buffer.putInt((int) value); + return Integer.BYTES; + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeInt((int) value); + return Integer.BYTES; + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getInt32Encoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getInt32Compressor(); + } + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new IntRleDecoder(); + case TS_2DIFF -> new DeltaBinaryDecoder.IntDeltaDecoder(); + case REGULAR -> new RegularDataDecoder.IntRegularDecoder(); + case GORILLA -> new IntGorillaDecoder(); + case ZIGZAG -> new IntZigzagDecoder(); + case CHIMP -> new IntChimpDecoder(); + case SPRINTZ -> new IntSprintzDecoder(); + case RLBE -> new IntRLBEDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES; + } + + @Override + public Statistics createStatistics() { + return new IntegerStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return IntegerStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getInt()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getInt()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getInt(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getInt(); + } + + @Override + public String toString(Field field) { + return String.valueOf(field.getIntV()); + } + + @Override + public Object getValue(Field field) { + return field.getIntV(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setIntV(from.getInt()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getInts()[index] = from.getInt(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getInts(), startIndex, endIndex, (int) value); + } + + @Override + public void setTo(Field from, Field to) { + to.setIntV(from.getIntV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setIntV(from.getInt()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getInts()[index] = from.getInt(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getInts()[toIndex] = from.getInt(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((int[]) toArray)[toIndex] = from.getInt(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getIntV()); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getInt(columnName)); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new IntDataPoint(measurementId, Integer.parseInt(value)); + } + + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new IntDataPoint(measurementId, (int) value); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? 0 : (int) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getInt() : Integer.MAX_VALUE, value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((int[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getInt(rowIndex), isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getInts(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((int[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getInt()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getInt()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (int) value); + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsInt(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsInt((int) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsInt(((int[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return c.getInt(position); @@ -48,6 +334,31 @@ public double getDouble(Column c, int position) { return c.getInt(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeInt(value.getInt()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeInt(BytesUtils.bytesToInt(bytes, offset)); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeInt(column.getInt(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((int[]) toArray)[index] = from.getInt(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((int[]) toArray)[index] = from.getInt(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeInt(value); @@ -68,11 +379,145 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeInt((int) value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + checkValueType(value, Integer.class, "Integer"); + ((int[]) array)[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((int[]) target)[targetIndex] = ((int[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((int[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new int[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfIntArray(size); + } + + @Override + public long estimateValueSize() { + return Integer.BYTES; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return Math.multiplyExact(Integer.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + int[] values = (int[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + int[] values = (int[]) array; + for (int i = 0; i < length; i++) { + buffer.putInt(values[i]); + } + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + int[] values = new int[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readInt(buffer); + } + return values; + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + int[] values = new int[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readInt(stream); + } + return values; + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readInt(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readInt(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + int[] intValues = new int[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + intValues[i] = buffer.getInt(); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + intValues[i] = buffer.getInt(); + } + } + } + return new IntColumn(0, rowSize, nullIndicators, intValues); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((int[]) left, 0, rowSize, (int[]) right, 0, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((int[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new IntColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.INT32); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new IntColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(IntColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new IntColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new IntColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public boolean isComparable() { return true; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java index e8e1ecbc3..6555b3962 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractLongType.java @@ -21,13 +21,298 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; +import org.apache.tsfile.encoding.decoder.LongChimpDecoder; +import org.apache.tsfile.encoding.decoder.LongGorillaDecoder; +import org.apache.tsfile.encoding.decoder.LongRLBEDecoder; +import org.apache.tsfile.encoding.decoder.LongRleDecoder; +import org.apache.tsfile.encoding.decoder.LongSprintzDecoder; +import org.apache.tsfile.encoding.decoder.LongZigzagDecoder; +import org.apache.tsfile.encoding.decoder.RegularDataDecoder; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.LongStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public abstract class AbstractLongType extends AbstractType { + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.longToBytes(value.getLong(), valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return Long.BYTES; + } + + @Override + public int calcTypeSize(Object value) { + return Long.BYTES; + } + + @Override + public void init(BatchData batchData) { + batchData.initLongValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putLong(timestamp, (long) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeLong(batchData.getLongByIndex(index)); + } + return Math.multiplyExact(Long.BYTES + Long.BYTES, batchData.length()); + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeLong(value.getLong()); + return Long.BYTES; + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + buffer.putLong((long) value); + return Long.BYTES; + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeLong((long) value); + return Long.BYTES; + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getInt64Encoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getInt64Compressor(); + } + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new LongRleDecoder(); + case TS_2DIFF -> new DeltaBinaryDecoder.LongDeltaDecoder(); + case REGULAR -> new RegularDataDecoder.LongRegularDecoder(); + case GORILLA -> new LongGorillaDecoder(); + case ZIGZAG -> new LongZigzagDecoder(); + case CHIMP -> new LongChimpDecoder(); + case SPRINTZ -> new LongSprintzDecoder(); + case RLBE -> new LongRLBEDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Long.BYTES; + } + + @Override + public Statistics createStatistics() { + return new LongStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return LongStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getLong()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getLong()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getLong(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getLong(); + } + + @Override + public String toString(Field field) { + return String.valueOf(field.getLongV()); + } + + @Override + public Object getValue(Field field) { + return field.getLongV(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setLongV(from.getLong()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getLongs()[index] = from.getLong(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getLongs(), startIndex, endIndex, (long) value); + } + + @Override + public void setTo(Field from, Field to) { + to.setLongV(from.getLongV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setLongV(from.getLong()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getLongs()[index] = from.getLong(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getLongs()[toIndex] = from.getLong(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((long[]) toArray)[toIndex] = from.getLong(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getLongV()); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getLong(columnName)); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new LongDataPoint(measurementId, Long.parseLong(value)); + } + + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new LongDataPoint(measurementId, value); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? 0L : (long) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getLong() : Long.MAX_VALUE, value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((long[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getLong(rowIndex), isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getLongs(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((long[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getLong()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getLong()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (long) value); + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsLong(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsLong((long) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsLong(((long[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return (int) c.getLong(position); @@ -48,6 +333,31 @@ public double getDouble(Column c, int position) { return c.getLong(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeLong(value.getLong()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeLong(BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset)); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeLong(column.getLong(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((long[]) toArray)[index] = from.getLong(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((long[]) toArray)[index] = from.getLong(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeLong(value); @@ -68,11 +378,142 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeLong((long) value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + checkValueType(value, Long.class, "Long"); + ((long[]) array)[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((long[]) target)[targetIndex] = ((long[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((long[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new long[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfLongArray(size); + } + + @Override + public long estimateValueSize() { + return Long.BYTES; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return Math.multiplyExact(Long.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + long[] values = (long[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + long[] values = (long[]) array; + for (int i = 0; i < length; i++) { + buffer.putLong(values[i]); + } + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + long[] values = new long[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readLong(buffer); + } + return values; + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + long[] values = new long[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readLong(stream); + } + return values; + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readLong(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readLong(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + long[] intValues = new long[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + intValues[i] = buffer.getLong(); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + intValues[i] = buffer.getLong(); + } + } + } + return new LongColumn(0, rowSize, nullIndicators, intValues); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((long[]) left, 0, rowSize, (long[]) right, 0, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((long[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new LongColumn( + positionCount, Optional.of(new boolean[positionCount]), new long[positionCount]); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new LongColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(LongColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new LongColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new LongColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public boolean isComparable() { return true; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java index 937b7fdcc..2afe06758 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractType.java @@ -19,8 +19,162 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DictionaryDecoder; +import org.apache.tsfile.encoding.decoder.PlainDecoder; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Array; +import java.nio.ByteBuffer; +import java.util.Objects; + public abstract class AbstractType implements Type { + private static final String DECODER_NOT_FOUND_ERROR_KEY = "error.encoding.decoder_not_found"; + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN -> new PlainDecoder(); + case DICTIONARY -> new DictionaryDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + protected TsFileDecodingException decoderNotFound(TSEncoding encoding) { + return new TsFileDecodingException( + Messages.format(DECODER_NOT_FOUND_ERROR_KEY, encoding, getTypeEnum())); + } + + protected void binaryToBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + Binary binary = value.getBinary(); + BytesUtils.intToBytes(binary.getLength(), valueBytes, offset); + System.arraycopy(binary.getValues(), 0, valueBytes, offset + Integer.BYTES, binary.getLength()); + } + + protected int binaryTypeSize(TsPrimitiveType value) { + return Integer.BYTES + value.getBinary().getLength(); + } + + protected int binaryTypeSize(Object value) { + return Integer.BYTES + ((Binary) value).getLength(); + } + + protected void checkValueType(Object value, Class expectedClass, String expectedType) { + if (value != null && !expectedClass.isInstance(value)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + expectedType, + getDisplayName(), + value.getClass().getName())); + } + } + + protected int serializedSizeOfBinaryValues(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, 0, rowSize); + } + + protected int serializedSizeOfBinaryValues(Object array, int startRow, int endRow) { + Binary[] binaryValues = (Binary[]) array; + int size = 0; + for (int i = startRow; i < endRow; i++) { + size = Math.addExact(size, Byte.BYTES); + if (binaryValues[i] != null) { + size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(binaryValues[i])); + } + } + return size; + } + + protected void serializeBinaryValues(Object array, int rowSize, DataOutputStream stream) + throws IOException { + Binary[] binaryValues = (Binary[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(binaryValues[i] != null), stream); + if (binaryValues[i] != null) { + ReadWriteIOUtils.write(binaryValues[i], stream); + } + } + } + + protected void serializeBinaryValues(Object array, int length, ByteBuffer buffer) { + Binary[] binaryValues = (Binary[]) array; + for (int i = 0; i < length; i++) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(binaryValues[i] != null), buffer); + if (binaryValues[i] != null) { + ReadWriteIOUtils.write(binaryValues[i], buffer); + } + } + } + + protected Binary[] deserializeBinaryValues(ByteBuffer buffer, int rowSize) { + Binary[] values = new Binary[rowSize]; + for (int i = 0; i < rowSize; i++) { + boolean isNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); + values[i] = isNotNull ? ReadWriteIOUtils.readBinary(buffer) : Binary.EMPTY_VALUE; + } + return values; + } + + protected Binary[] deserializeBinaryValues(DataInputStream stream, int rowSize) + throws IOException { + Binary[] values = new Binary[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = stream.readBoolean() ? ReadWriteIOUtils.readBinary(stream) : Binary.EMPTY_VALUE; + } + return values; + } + + protected Binary[] deserializeBinaryValues( + ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] values = new Binary[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readBinary(buffer); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = ReadWriteIOUtils.readBinary(buffer); + } + } + } + return values; + } + + protected boolean hasEnoughLength(Object left, Object right, int rowSize) { + return Array.getLength(left) >= rowSize && Array.getLength(right) >= rowSize; + } + + protected boolean binaryArrayEquals(Object left, Object right, int rowSize) { + if (!hasEnoughLength(left, right, rowSize)) { + return false; + } + Binary[] leftValues = (Binary[]) left; + Binary[] rightValues = (Binary[]) right; + for (int i = 0; i < rowSize; i++) { + if (!Objects.equals(leftValues[i], rightValues[i])) { + return false; + } + } + return true; + } + + @Override + public Object getValue(Object array, int rowIndex) { + return Array.get(array, rowIndex); + } + @Override public String toString() { return getDisplayName(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java index d09ed07c6..520f16444 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/AbstractVarcharType.java @@ -21,28 +21,445 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.BinaryStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public abstract class AbstractVarcharType extends AbstractType { + + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + binaryToBytes(value, valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return binaryTypeSize(value); + } + + @Override + public int calcTypeSize(Object value) { + return binaryTypeSize(value); + } + + @Override + public void init(BatchData batchData) { + batchData.initBinaryValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBinary(timestamp, (Binary) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + int size = 0; + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + Binary binary = batchData.getBinaryByIndex(index); + outputStream.writeInt(binary.getLength()); + outputStream.write(binary.getValues()); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); + size = Math.addExact(size, binary.getLength()); + } + return size; + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + Binary binary = value.getBinary(); + stream.writeInt(binary.getLength()); + stream.write(binary.getValues()); + return Math.addExact(Integer.BYTES, binary.getLength()); + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + return ReadWriteIOUtils.write((Binary) value, buffer); + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + return ReadWriteIOUtils.write((Binary) value, stream); + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getTextEncoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getTextCompressor(); + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; + } + + @Override + public Statistics createStatistics() { + return new BinaryStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return BinaryStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBinary()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBinary()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBinary(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBinary(); + } + + @Override + public String toString(Field field) { + return field.getBinaryV().toString(); + } + + @Override + public Object getValue(Field field) { + return field.getBinaryV(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBinaryV(from.getBinary()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill( + to.getBinaries(), startIndex, endIndex, value == null ? null : toBinaryValue(value)); + } + + @Override + public void setTo(Field from, Field to) { + to.setBinaryV(from.getBinaryV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setBinaryV(from.getBinary()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBinaries()[toIndex] = from.getBinary(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getStringValue()); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getString(columnName)); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? Binary.EMPTY_VALUE : (Binary) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getBinary() : Binary.EMPTY_VALUE, value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? null : column.getBinary(rowIndex), isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((Binary[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBinary()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBinary()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (Binary) value); + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsBinary(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsBinary((Binary) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsBinary(((Binary[]) array)[rowIndex]); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBinary(value.getBinary()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + int length = BytesUtils.bytesToInt(bytes, offset); + builder.writeBinary( + new Binary( + Arrays.copyOfRange(bytes, offset + Integer.BYTES, offset + Integer.BYTES + length))); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBinary(column.getBinary(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + ((Binary[]) array)[rowIndex] = toBinaryValue(value); + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((Binary[]) array, newLength); + } + + protected Binary toBinaryValue(Object value) { + if (value != null && !(value instanceof Binary) && !(value instanceof String)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + "Binary or String", + getDisplayName(), + value.getClass().getName())); + } + if (value instanceof Binary) { + return (Binary) value; + } + return value != null + ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) + : Binary.EMPTY_VALUE; + } + + @Override + public Object createArray(int capacity) { + return new Binary[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, rowSize); + } + + @Override + public int serializedSize(Object array, int startRow, int endRow) { + return serializedSizeOfBinaryValues(array, startRow, endRow); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + serializeBinaryValues(array, rowSize, stream); + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + serializeBinaryValues(array, length, buffer); + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + return deserializeBinaryValues(buffer, rowSize); + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + return deserializeBinaryValues(stream, rowSize); + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readBinary(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBinary(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] values = deserializeBinaryValues(buffer, rowSize, nullIndicators); + return new BinaryColumn(0, rowSize, nullIndicators, values); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return binaryArrayEquals(left, right, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((Binary[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new BinaryColumn( + positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BinaryColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new BinaryColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public boolean isComparable() { return true; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java index 68c874998..ece9ba52b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java @@ -19,11 +19,22 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.utils.Binary; + public class BinaryType extends AbstractVarcharType { public static final BinaryType TEXT = new BinaryType(); private BinaryType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.TEXT); + field.setBinaryV((Binary) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.TEXT; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java index d02ca7168..c0c1e8b81 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BlobType.java @@ -21,11 +21,41 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.BlobStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public class BlobType extends AbstractType { @@ -33,21 +63,418 @@ public class BlobType extends AbstractType { private BlobType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + binaryToBytes(value, valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return binaryTypeSize(value); + } + + @Override + public int calcTypeSize(Object value) { + return binaryTypeSize(value); + } + + @Override + public void init(BatchData batchData) { + batchData.initBinaryValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBinary(timestamp, (Binary) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + int size = 0; + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + Binary binary = batchData.getBinaryByIndex(index); + outputStream.writeInt(binary.getLength()); + outputStream.write(binary.getValues()); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); + size = Math.addExact(size, binary.getLength()); + } + return size; + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + Binary binary = value.getBinary(); + stream.writeInt(binary.getLength()); + stream.write(binary.getValues()); + return Math.addExact(Integer.BYTES, binary.getLength()); + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + return ReadWriteIOUtils.write((Binary) value, buffer); + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + return ReadWriteIOUtils.write((Binary) value, stream); + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getTextEncoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getTextCompressor(); + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; + } + + @Override + public Statistics createStatistics() { + return new BlobStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return BlobStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBinary()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBinary()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBinary(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBinary(); + } + + @Override + public String toString(Field field) { + return BytesUtils.parseBlobByteArrayToString(field.getBinaryV().getValues()); + } + + @Override + public Object getValue(Field field) { + return field.getBinaryV(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBinaryV(from.getBinary()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + if (value == null) { + Arrays.fill(to.getBinaries(), startIndex, endIndex, null); + return; + } + Binary[] values = new Binary[1]; + addValue(0, value, values); + Arrays.fill(to.getBinaries(), startIndex, endIndex, values[0]); + } + + @Override + public void setTo(Field from, Field to) { + to.setBinaryV(from.getBinaryV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setBinaryV(from.getBinary()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBinaries()[toIndex] = from.getBinary(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getBinaryV().getValues()); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getBinary(columnName)); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? Binary.EMPTY_VALUE : (Binary) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getBinary() : Binary.EMPTY_VALUE, value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? null : column.getBinary(rowIndex), isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((Binary[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBinary()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBinary()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (Binary) value); + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.BLOB); + field.setBinaryV((Binary) value); + return field; + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsBinary(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsBinary((Binary) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsBinary(((Binary[]) array)[rowIndex]); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBinary(value.getBinary()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + int length = BytesUtils.bytesToInt(bytes, offset); + builder.writeBinary( + new Binary( + Arrays.copyOfRange(bytes, offset + Integer.BYTES, offset + Integer.BYTES + length))); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBinary(column.getBinary(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + if (value != null && !(value instanceof Binary) && !(value instanceof String)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + "Binary or String", + getDisplayName(), + value.getClass().getName())); + } + if (value instanceof Binary) { + ((Binary[]) array)[rowIndex] = (Binary) value; + } else { + ((Binary[]) array)[rowIndex] = + value != null + ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) + : Binary.EMPTY_VALUE; + } + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((Binary[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new Binary[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, rowSize); + } + + @Override + public int serializedSize(Object array, int startRow, int endRow) { + return serializedSizeOfBinaryValues(array, startRow, endRow); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + serializeBinaryValues(array, rowSize, stream); + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + serializeBinaryValues(array, length, buffer); + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + return deserializeBinaryValues(buffer, rowSize); + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + return deserializeBinaryValues(stream, rowSize); + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readBinary(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBinary(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] binaries = deserializeBinaryValues(buffer, rowSize, nullIndicators); + return new BinaryColumn(0, rowSize, nullIndicators, binaries); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return binaryArrayEquals(left, right, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((Binary[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new BinaryColumn( + positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BinaryColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new BinaryColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.BLOB; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index e00d7ecb8..55370db6e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -21,10 +21,42 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.IntRleDecoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.BooleanStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import org.apache.tsfile.read.common.block.column.ColumnEncoder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public class BooleanType extends AbstractType { @@ -32,21 +64,433 @@ public class BooleanType extends AbstractType { private BooleanType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.boolToBytes(value.getBoolean(), valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return Byte.BYTES; + } + + @Override + public int calcTypeSize(Object value) { + return Byte.BYTES; + } + + @Override + public void init(BatchData batchData) { + batchData.initBooleanValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBoolean(timestamp, (boolean) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeBoolean(batchData.getBooleanByIndex(index)); + } + return Math.multiplyExact(Long.BYTES + Byte.BYTES, batchData.length()); + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeBoolean(value.getBoolean()); + return Byte.BYTES; + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + buffer.put((boolean) value ? (byte) 1 : (byte) 0); + return Byte.BYTES; + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeBoolean((boolean) value); + return Byte.BYTES; + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getBooleanEncoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getBooleanCompressor(); + } + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new IntRleDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Byte.BYTES; + } + + @Override + public Statistics createStatistics() { + return new BooleanStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return BooleanStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBoolean()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBoolean()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBoolean(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBoolean(); + } + + @Override + public String toString(Field field) { + return String.valueOf(field.getBoolV()); + } + + @Override + public Object getValue(Field field) { + return field.getBoolV(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBoolV(from.getBoolean()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBooleans()[index] = from.getBoolean(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getBooleans(), startIndex, endIndex, (boolean) value); + } + + @Override + public void setTo(Field from, Field to) { + to.setBoolV(from.getBoolV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setBoolV(from.getBoolean()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBooleans()[index] = from.getBoolean(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBooleans()[toIndex] = from.getBoolean(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((boolean[]) toArray)[toIndex] = from.getBoolean(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getBoolV()); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getBoolean(columnName)); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new BooleanDataPoint(measurementId, Boolean.parseBoolean(value)); + } + + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new BooleanDataPoint(measurementId, true); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, !isNull && (boolean) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null && value.getBoolean(), value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((boolean[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? false : column.getBoolean(rowIndex), isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBooleans(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((boolean[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBoolean()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBoolean()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (boolean) value); + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.BOOLEAN); + field.setBoolV((boolean) value); + return field; + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsBoolean(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsBoolean((boolean) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsBoolean(((boolean[]) array)[rowIndex]); + } + @Override public boolean getBoolean(Column c, int position) { return c.getBoolean(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBoolean(value.getBoolean()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeBoolean(BytesUtils.bytesToBool(bytes, offset)); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBoolean(column.getBoolean(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((boolean[]) toArray)[index] = from.getBoolean(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((boolean[]) toArray)[index] = from.getBoolean(); + } + @Override public void writeBoolean(ColumnBuilder builder, boolean value) { builder.writeBoolean(value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + checkValueType(value, Boolean.class, "Boolean"); + ((boolean[]) array)[rowIndex] = value != null && (boolean) value; + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((boolean[]) target)[targetIndex] = ((boolean[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((boolean[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new boolean[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfBooleanArray(size); + } + + @Override + public long estimateValueSize() { + return Byte.BYTES; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return rowSize; + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + boolean[] values = (boolean[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(BytesUtils.boolToByte(values[i]), stream); + } + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + boolean[] values = (boolean[]) array; + for (int i = 0; i < length; i++) { + buffer.put(BytesUtils.boolToByte(values[i])); + } + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + boolean[] values = new boolean[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); + } + return values; + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + boolean[] values = new boolean[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(stream)); + } + return values; + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(buffer)); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBoolean(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + if (nullIndicators == null) { + boolean[] values = ColumnEncoder.deserializeBooleanArray(buffer, rowSize); + return new BooleanColumn(rowSize, Optional.empty(), values); + } + + int nonNullCount = 0; + for (boolean isNull : nullIndicators) { + if (!isNull) { + nonNullCount++; + } + } + + boolean[] nonNullValues = ColumnEncoder.deserializeBooleanArray(buffer, nonNullCount); + boolean[] values = new boolean[rowSize]; + int nonNullIndex = 0; + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = nonNullValues[nonNullIndex++]; + } + } + return new BooleanColumn(rowSize, Optional.of(nullIndicators), values); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((boolean[]) left, 0, rowSize, (boolean[]) right, 0, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((boolean[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new BooleanColumn( + positionCount, Optional.of(new boolean[positionCount]), new boolean[positionCount]); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BooleanColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BooleanColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new BooleanColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.BOOLEAN; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java index 88d17e242..c231a6f9b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DateType.java @@ -19,12 +19,283 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.DateStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.utils.TsPrimitiveType.TsInt; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Optional; + public class DateType extends AbstractIntType { public static final DateType DATE = new DateType(); + private static final LocalDate EMPTY_DATE = LocalDate.of(1000, 1, 1); private DateType() {} + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsInt(TSDataType.DATE); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsInt((int) value, TSDataType.DATE); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsInt(getDateInt(array, rowIndex), TSDataType.DATE); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + if (toArray instanceof int[]) { + super.write(from, toArray, index); + return; + } + ((LocalDate[]) toArray)[index] = DateUtils.parseIntToLocalDate(from.getInt()); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + if (toArray instanceof int[]) { + super.write(from, toArray, index); + return; + } + ((LocalDate[]) toArray)[index] = DateUtils.parseIntToLocalDate(from.getInt()); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); + } + + @Override + public Statistics createStatistics() { + return new DateStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return DateStatistics.INSTANCE_SIZE; + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + if (array instanceof int[]) { + super.write(writer, time, array, rowIndex, isNull); + return; + } + writer.write( + time, + isNull ? 0 : DateUtils.parseDateExpressionToInt(((LocalDate[]) array)[rowIndex]), + isNull); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + if (column instanceof int[]) { + super.write(writer, time, column, rowIndex); + return; + } + writer.write(time, DateUtils.parseDateExpressionToInt(((LocalDate[]) column)[rowIndex])); + } + + @Override + public void addValue(int rowIndex, Object value, Object array) { + if (array instanceof int[]) { + if (value instanceof Integer) { + ((int[]) array)[rowIndex] = (int) value; + } else { + checkValueType(value, LocalDate.class, "LocalDate"); + ((int[]) array)[rowIndex] = + value != null + ? DateUtils.parseDateExpressionToInt((LocalDate) value) + : DateUtils.EMPTY_DATE_INT; + } + return; + } + checkValueType(value, LocalDate.class, "LocalDate"); + ((LocalDate[]) array)[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + if (toArray instanceof int[]) { + super.setTo(from, fromIndex, toArray, toIndex); + return; + } + ((LocalDate[]) toArray)[toIndex] = DateUtils.parseIntToLocalDate(from.getInt(fromIndex)); + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + if (target instanceof int[]) { + ((int[]) target)[targetIndex] = getDateInt(source, sourceIndex); + return; + } + ((LocalDate[]) target)[targetIndex] = + source instanceof int[] + ? DateUtils.parseIntToLocalDate(((int[]) source)[sourceIndex]) + : ((LocalDate[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return array instanceof int[] + ? Arrays.copyOf((int[]) array, newLength) + : Arrays.copyOf((LocalDate[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new LocalDate[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + + @Override + public Object getValue(Field field) { + return field.getDateV(); + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.DATE); + field.setIntV((int) value); + return field; + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + if (array instanceof int[]) { + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(((int[]) array)[i], stream); + } + return; + } + LocalDate[] values = (LocalDate[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write( + values[i] == null + ? DateUtils.EMPTY_DATE_INT + : DateUtils.parseDateExpressionToInt(values[i]), + stream); + } + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + if (array instanceof int[]) { + super.serializeArray(array, length, buffer); + return; + } + LocalDate[] values = (LocalDate[]) array; + for (int i = 0; i < length; i++) { + buffer.putInt( + values[i] == null + ? DateUtils.EMPTY_DATE_INT + : DateUtils.parseDateExpressionToInt(values[i])); + } + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + LocalDate[] values = new LocalDate[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = DateUtils.parseIntToLocalDate(ReadWriteIOUtils.readInt(buffer)); + } + return values; + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + LocalDate[] values = new LocalDate[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = DateUtils.parseIntToLocalDate(ReadWriteIOUtils.readInt(stream)); + } + return values; + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + return ((IntColumn) super.deserializeColumn(buffer, rowSize, nullIndicators)) + .modifyDataType(TSDataType.DATE); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + if (!hasEnoughLength(left, right, rowSize)) { + return false; + } + if (left instanceof int[] && right instanceof int[]) { + return Arrays.equals((int[]) left, 0, rowSize, (int[]) right, 0, rowSize); + } + if (left instanceof LocalDate[] && right instanceof LocalDate[]) { + return Arrays.equals((LocalDate[]) left, 0, rowSize, (LocalDate[]) right, 0, rowSize); + } + for (int i = 0; i < rowSize; i++) { + if (getDateInt(left, i) != getDateInt(right, i)) { + return false; + } + } + return true; + } + + @Override + public long estimateArraySize(Object array) { + return array instanceof int[] + ? RamUsageEstimator.sizeOf((int[]) array) + : RamUsageEstimator.sizeOf((LocalDate[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new IntColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.DATE); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new IntColumn(positionCount, TSDataType.DATE); + } + + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new IntColumnBuilder(columnBuilderStatus, expectedEntries, TSDataType.DATE); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DATE; @@ -35,6 +306,14 @@ public String getDisplayName() { return "DATE"; } + private int getDateInt(Object array, int index) { + if (array instanceof int[]) { + return ((int[]) array)[index]; + } + LocalDate value = ((LocalDate[]) array)[index]; + return value != null ? DateUtils.parseDateExpressionToInt(value) : DateUtils.EMPTY_DATE_INT; + } + public static DateType getInstance() { return DATE; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 6e325c066..f847fe68c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -21,10 +21,47 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.CamelDecoder; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DoublePrecisionChimpDecoder; +import org.apache.tsfile.encoding.decoder.DoublePrecisionDecoderV1; +import org.apache.tsfile.encoding.decoder.DoublePrecisionDecoderV2; +import org.apache.tsfile.encoding.decoder.DoubleRLBEDecoder; +import org.apache.tsfile.encoding.decoder.DoubleSprintzDecoder; +import org.apache.tsfile.encoding.decoder.FloatDecoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.DoubleStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public class DoubleType extends AbstractType { @@ -32,6 +69,260 @@ public class DoubleType extends AbstractType { private DoubleType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.doubleToBytes(value.getDouble(), valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return Double.BYTES; + } + + @Override + public int calcTypeSize(Object value) { + return Double.BYTES; + } + + @Override + public void init(BatchData batchData) { + batchData.initDoubleValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putDouble(timestamp, (double) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeDouble(batchData.getDoubleByIndex(index)); + } + return Math.multiplyExact(Long.BYTES + Double.BYTES, batchData.length()); + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeDouble(value.getDouble()); + return Double.BYTES; + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + buffer.putDouble((double) value); + return Double.BYTES; + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeDouble((double) value); + return Double.BYTES; + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getDoubleEncoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getDoubleCompressor(); + } + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE, TS_2DIFF -> new FloatDecoder(encoding, TSDataType.DOUBLE); + case GORILLA_V1 -> new DoublePrecisionDecoderV1(); + case GORILLA -> new DoublePrecisionDecoderV2(); + case CHIMP -> new DoublePrecisionChimpDecoder(); + case SPRINTZ -> new DoubleSprintzDecoder(); + case RLBE -> new DoubleRLBEDecoder(); + case CAMEL -> new CamelDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Double.BYTES; + } + + @Override + public Statistics createStatistics() { + return new DoubleStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return DoubleStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getDouble()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getDouble()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getDouble(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getDouble(); + } + + @Override + public String toString(Field field) { + return String.valueOf(field.getDoubleV()); + } + + @Override + public Object getValue(Field field) { + return field.getDoubleV(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setDoubleV(from.getDouble()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getDoubles()[index] = from.getDouble(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getDoubles(), startIndex, endIndex, (double) value); + } + + @Override + public void setTo(Field from, Field to) { + to.setDoubleV(from.getDoubleV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setDoubleV(from.getDouble()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getDoubles()[index] = from.getDouble(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getDoubles()[toIndex] = from.getDouble(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((double[]) toArray)[toIndex] = from.getDouble(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getDoubleV()); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getDouble(columnName)); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new DoubleDataPoint(measurementId, Double.parseDouble(value)); + } + + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new DoubleDataPoint(measurementId, (double) value); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? 0.0D : (double) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getDouble() : Double.MAX_VALUE, value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((double[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getDouble(rowIndex), isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getDoubles(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((double[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getDouble()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getDouble()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (double) value); + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.DOUBLE); + field.setDoubleV((double) value); + return field; + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsDouble(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsDouble((double) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsDouble(((double[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return (int) c.getDouble(position); @@ -52,6 +343,31 @@ public double getDouble(Column c, int position) { return c.getDouble(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeDouble(value.getDouble()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeDouble(BytesUtils.bytesToDouble(bytes, offset)); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeDouble(column.getDouble(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((double[]) toArray)[index] = from.getDouble(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((double[]) toArray)[index] = from.getDouble(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeDouble(value); @@ -72,11 +388,142 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeDouble(value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + checkValueType(value, Double.class, "Double"); + ((double[]) array)[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((double[]) target)[targetIndex] = ((double[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((double[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new double[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfDoubleArray(size); + } + + @Override + public long estimateValueSize() { + return Double.BYTES; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return Math.multiplyExact(Double.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + double[] values = (double[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + double[] values = (double[]) array; + for (int i = 0; i < length; i++) { + buffer.putDouble(values[i]); + } + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + double[] values = new double[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readDouble(buffer); + } + return values; + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + double[] values = new double[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readDouble(stream); + } + return values; + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readDouble(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readDouble(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + double[] values = new double[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + values[i] = Double.longBitsToDouble(buffer.getLong()); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = Double.longBitsToDouble(buffer.getLong()); + } + } + } + return new DoubleColumn(rowSize, Optional.ofNullable(nullIndicators), values); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((double[]) left, 0, rowSize, (double[]) right, 0, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((double[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new DoubleColumn( + positionCount, Optional.of(new boolean[positionCount]), new double[positionCount]); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new DoubleColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(DoubleColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new DoubleColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new DoubleColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.DOUBLE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 6417f0f52..635ba53ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -21,10 +21,46 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.FloatDecoder; +import org.apache.tsfile.encoding.decoder.FloatRLBEDecoder; +import org.apache.tsfile.encoding.decoder.FloatSprintzDecoder; +import org.apache.tsfile.encoding.decoder.SinglePrecisionChimpDecoder; +import org.apache.tsfile.encoding.decoder.SinglePrecisionDecoderV1; +import org.apache.tsfile.encoding.decoder.SinglePrecisionDecoderV2; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.FloatStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.FloatColumn; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.FloatDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; public class FloatType extends AbstractType { @@ -32,6 +68,259 @@ public class FloatType extends AbstractType { private FloatType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + BytesUtils.floatToBytes(value.getFloat(), valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return Float.BYTES; + } + + @Override + public int calcTypeSize(Object value) { + return Float.BYTES; + } + + @Override + public void init(BatchData batchData) { + batchData.initFloatValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putFloat(timestamp, (float) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + outputStream.writeFloat(batchData.getFloatByIndex(index)); + } + return Math.multiplyExact(Long.BYTES + Float.BYTES, batchData.length()); + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + stream.writeFloat(value.getFloat()); + return Float.BYTES; + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + buffer.putFloat((float) value); + return Float.BYTES; + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + stream.writeFloat((float) value); + return Float.BYTES; + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getFloatEncoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getFloatCompressor(); + } + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE, TS_2DIFF -> new FloatDecoder(encoding, TSDataType.FLOAT); + case GORILLA_V1 -> new SinglePrecisionDecoderV1(); + case GORILLA -> new SinglePrecisionDecoderV2(); + case CHIMP -> new SinglePrecisionChimpDecoder(); + case SPRINTZ -> new FloatSprintzDecoder(); + case RLBE -> new FloatRLBEDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Float.BYTES; + } + + @Override + public Statistics createStatistics() { + return new FloatStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return FloatStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getFloat()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getFloat()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getFloat(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getFloat(); + } + + @Override + public String toString(Field field) { + return String.valueOf(field.getFloatV()); + } + + @Override + public Object getValue(Field field) { + return field.getFloatV(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setFloatV(from.getFloat()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getFloats()[index] = from.getFloat(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getFloats(), startIndex, endIndex, (float) value); + } + + @Override + public void setTo(Field from, Field to) { + to.setFloatV(from.getFloatV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setFloatV(from.getFloat()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getFloats()[index] = from.getFloat(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getFloats()[toIndex] = from.getFloat(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((float[]) toArray)[toIndex] = from.getFloat(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getFloatV()); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getFloat(columnName)); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new FloatDataPoint(measurementId, Float.parseFloat(value)); + } + + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new FloatDataPoint(measurementId, (float) value); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? 0.0F : (float) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getFloat() : Float.MAX_VALUE, value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((float[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + writer.write(time, isNull ? 0 : column.getFloat(rowIndex), isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getFloats(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((float[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getFloat()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getFloat()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (float) value); + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.FLOAT); + field.setFloatV((float) value); + return field; + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsFloat(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsFloat((float) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsFloat(((float[]) array)[rowIndex]); + } + @Override public int getInt(Column c, int position) { return (int) c.getFloat(position); @@ -52,6 +341,31 @@ public double getDouble(Column c, int position) { return c.getFloat(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeFloat(value.getFloat()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + builder.writeFloat(BytesUtils.bytesToFloat(bytes, offset)); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeFloat(column.getFloat(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((float[]) toArray)[index] = from.getFloat(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((float[]) toArray)[index] = from.getFloat(); + } + @Override public void writeInt(ColumnBuilder builder, int value) { builder.writeFloat(value); @@ -72,11 +386,142 @@ public void writeDouble(ColumnBuilder builder, double value) { builder.writeFloat((float) value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + checkValueType(value, Float.class, "Float"); + ((float[]) array)[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((float[]) target)[targetIndex] = ((float[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((float[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new float[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfFloatArray(size); + } + + @Override + public long estimateValueSize() { + return Float.BYTES; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return Math.multiplyExact(Float.BYTES, rowSize); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + float[] values = (float[]) array; + for (int i = 0; i < rowSize; i++) { + ReadWriteIOUtils.write(values[i], stream); + } + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + float[] values = (float[]) array; + for (int i = 0; i < length; i++) { + buffer.putFloat(values[i]); + } + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + float[] values = new float[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readFloat(buffer); + } + return values; + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + float[] values = new float[rowSize]; + for (int i = 0; i < rowSize; i++) { + values[i] = ReadWriteIOUtils.readFloat(stream); + } + return values; + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readFloat(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readFloat(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + float[] values = new float[rowSize]; + if (nullIndicators == null) { + for (int i = 0; i < rowSize; i++) { + values[i] = Float.intBitsToFloat(buffer.getInt()); + } + } else { + for (int i = 0; i < rowSize; i++) { + if (!nullIndicators[i]) { + values[i] = Float.intBitsToFloat(buffer.getInt()); + } + } + } + return new FloatColumn(rowSize, Optional.ofNullable(nullIndicators), values); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return hasEnoughLength(left, right, rowSize) + && Arrays.equals((float[]) left, 0, rowSize, (float[]) right, 0, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((float[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new FloatColumn( + positionCount, Optional.of(new boolean[positionCount]), new float[positionCount]); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new FloatColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(FloatColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new FloatColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new FloatColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.FLOAT; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java index e4d36c2a3..32e958193 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java @@ -19,12 +19,22 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; + public class IntType extends AbstractIntType { public static final IntType INT32 = new IntType(); private IntType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.INT32); + field.setIntV((int) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.INT32; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java index a1ed2a1c9..77c3d0520 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java @@ -19,12 +19,22 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; + public class LongType extends AbstractLongType { public static final LongType INT64 = new LongType(); private LongType() {} + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.INT64); + field.setLongV((long) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.INT64; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java index b30b621f1..1025d55da 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java @@ -21,11 +21,44 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.ObjectStatistics; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.utils.TsPrimitiveType.TsBinary; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; + +import static org.apache.tsfile.utils.BytesUtils.parseObjectByteArrayToString; public class ObjectType extends AbstractType { @@ -33,21 +66,406 @@ public class ObjectType extends AbstractType { private ObjectType() {} + @Override + public void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + binaryToBytes(value, valueBytes, offset); + } + + @Override + public int calcTypeSize(TsPrimitiveType value) { + return binaryTypeSize(value); + } + + @Override + public int calcTypeSize(Object value) { + return binaryTypeSize(value); + } + + @Override + public void init(BatchData batchData) { + batchData.initBinaryValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putBinary(timestamp, (Binary) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + int size = 0; + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + Binary binary = batchData.getBinaryByIndex(index); + outputStream.writeInt(binary.getLength()); + outputStream.write(binary.getValues()); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); + size = Math.addExact(size, binary.getLength()); + } + return size; + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + Binary binary = value.getBinary(); + stream.writeInt(binary.getLength()); + stream.write(binary.getValues()); + return Math.addExact(Integer.BYTES, binary.getLength()); + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + return ReadWriteIOUtils.write((Binary) value, buffer); + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) throws IOException { + return ReadWriteIOUtils.write((Binary) value, stream); + } + + @Override + public TSEncoding getDefaultEncoding(TSFileConfig config) { + return TSEncoding.valueOf(config.getTextEncoding()); + } + + @Override + public CompressionType getDefaultCompressor(TSFileConfig config) { + return config.getTextCompressor(); + } + + @Override + public int getOneItemMaxSize(int valveLength) { + return Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * valveLength; + } + + @Override + public Statistics createStatistics() { + return new ObjectStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return ObjectStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + stats.update(timestamp, value.getBinary()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + stats.update(batchData.currentTime(), batchData.getBinary()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + stats.update(block.getTimeByIndex(rowIndex), block.getColumn(columnIndex).getBinary(rowIndex)); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getBinary(); + } + + @Override + public String toString(Field field) { + return parseObjectByteArrayToString(field.getBinaryV().getValues()); + } + + @Override + public Object getValue(Field field) { + return field.getStringValue(); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + to.setBinaryV(from.getBinary()); + } + + @Override + public void setTo(TsPrimitiveType from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + + @Override + public void setTo(Object value, Column to, int startIndex, int endIndex) { + Arrays.fill(to.getBinaries(), startIndex, endIndex, value); + } + + @Override + public void setTo(Field from, Field to) { + to.setBinaryV(from.getBinaryV()); + } + + @Override + public void setTo(BatchData from, Field to) { + to.setBinaryV(from.getBinary()); + } + + @Override + public void setTo(BatchData from, Column to, int index) { + to.getBinaries()[index] = from.getBinary(); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + to.getBinaries()[toIndex] = from.getBinary(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((Binary[]) toArray)[toIndex] = from.getBinary(fromIndex); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + record.addPoint(columnName, resultSet.getBinary(columnName)); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + record.addPoint(columnName, field.getBinaryV().getValues()); + } + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + return new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + writer.write(time, isNull ? Binary.EMPTY_VALUE : (Binary) value, isNull); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + writer.write(time, value != null ? value.getBinary() : Binary.EMPTY_VALUE, value == null); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + writer.write(time, ((Binary[]) array)[rowIndex], isNull); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + writer.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + writer.write(time, ((Binary[]) column)[rowIndex]); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + writer.write(time, value.getBinary()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + writer.write(time, data.getBinary()); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + writer.write(time, (Binary) value); + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.OBJECT); + field.setBinaryV((Binary) value); + return field; + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsBinary(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsBinary((Binary) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsBinary(((Binary[]) array)[rowIndex]); + } + @Override public Binary getBinary(Column c, int position) { return c.getBinary(position); } + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + builder.writeBinary(value.getBinary()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + int length = BytesUtils.bytesToInt(bytes, offset); + builder.writeBinary( + new Binary( + Arrays.copyOfRange(bytes, offset + Integer.BYTES, offset + Integer.BYTES + length))); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + builder.writeBinary(column.getBinary(index)); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((Binary[]) toArray)[index] = from.getBinary(); + } + @Override public void writeBinary(ColumnBuilder builder, Binary value) { builder.writeBinary(value); } + @Override + public void addValue(int rowIndex, Object value, Object array) { + if (value != null && !(value instanceof Binary) && !(value instanceof String)) { + throw new IllegalArgumentException( + Messages.format( + "error.write.tablet_expected_type", + "Binary or String", + getDisplayName(), + value.getClass().getName())); + } + if (value instanceof Binary) { + ((Binary[]) array)[rowIndex] = (Binary) value; + } else { + ((Binary[]) array)[rowIndex] = + value != null + ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) + : Binary.EMPTY_VALUE; + } + } + + @Override + public void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + ((Binary[]) target)[targetIndex] = ((Binary[]) source)[sourceIndex]; + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((Binary[]) array, newLength); + } + + @Override + public Object createArray(int capacity) { + return new Binary[capacity]; + } + + @Override + public long estimateArraySize(int size) { + return RamUsageEstimator.sizeOfObjectArray(size); + } + + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + + @Override + public int serializedSize(Object array, int rowSize) { + return serializedSizeOfBinaryValues(array, rowSize); + } + + @Override + public int serializedSize(Object array, int startRow, int endRow) { + return serializedSizeOfBinaryValues(array, startRow, endRow); + } + + @Override + public void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + serializeBinaryValues(array, rowSize, stream); + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + serializeBinaryValues(array, length, buffer); + } + + @Override + public Object deserializeArray(ByteBuffer buffer, int rowSize) { + return deserializeBinaryValues(buffer, rowSize); + } + + @Override + public Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + return deserializeBinaryValues(stream, rowSize); + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + array[index] = ReadWriteIOUtils.readBinary(buffer); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) throws IOException { + array[index] = ReadWriteIOUtils.readBinary(stream); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + Binary[] values = deserializeBinaryValues(buffer, rowSize, nullIndicators); + return new BinaryColumn(0, rowSize, nullIndicators, values); + } + + @Override + public boolean arrayEquals(Object left, Object right, int rowSize) { + return binaryArrayEquals(left, right, rowSize); + } + + @Override + public long estimateArraySize(Object array) { + return RamUsageEstimator.sizeOf((Binary[]) array); + } + + @Override + public Column createColumnWithMaxPosition(int positionCount) { + return new BinaryColumn( + positionCount, Optional.of(new boolean[positionCount]), new Binary[positionCount]); + } + + @Override + public Column createColumnWithZeroPosition(int positionCount) { + return new BinaryColumn(positionCount); + } + + @Override + public Column createNullColumn(int positionCount) { + return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BinaryColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + return new BinaryColumnBuilder(columnBuilderStatus, expectedEntries); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.OBJECT; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java index 96bd3b380..6123fb30b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java @@ -20,6 +20,8 @@ package org.apache.tsfile.read.common.type; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.utils.TsPrimitiveType; import java.util.ArrayList; import java.util.Arrays; @@ -73,6 +75,31 @@ public static RowType createWithTypeSignature(List fields) { return new RowType(fields); } + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsVector(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsVector((TsPrimitiveType[]) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsVector(((TsPrimitiveType[][]) array)[rowIndex]); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); + } + public static Field field(String name, Type type) { return new Field(Optional.of(name), type); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java index 725ae9e3b..f5d54ed7d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/StringType.java @@ -19,12 +19,35 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.file.metadata.statistics.StringStatistics; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.utils.Binary; + public class StringType extends AbstractVarcharType { public static final StringType STRING = new StringType(); private StringType() {} + @Override + public Statistics createStatistics() { + return new StringStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return StringStatistics.INSTANCE_SIZE; + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.STRING); + field.setBinaryV((Binary) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.STRING; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java index c4e5e8434..143c577f0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TimestampType.java @@ -19,12 +19,34 @@ package org.apache.tsfile.read.common.type; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.file.metadata.statistics.TimestampStatistics; +import org.apache.tsfile.read.common.Field; + public class TimestampType extends AbstractLongType { public static final TimestampType TIMESTAMP = new TimestampType(); private TimestampType() {} + @Override + public Statistics createStatistics() { + return new TimestampStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return TimestampStatistics.INSTANCE_SIZE; + } + + @Override + public Field getField(Object value) { + Field field = new Field(TSDataType.TIMESTAMP); + field.setLongV((long) value); + return field; + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.TIMESTAMP; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index cb6b60944..d7be3dbab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -21,12 +21,48 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.UnknownColumnTypeException; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; import java.util.List; +/** + * Type provides common interfaces associated with a specific TsDataType to remove uses of + * "switch(dataType)", which make the extension of datatypes very difficult. + * + *

Interfaces that are less generic should be defined using TypeService and checked in a proper + * context. + */ public interface Type { + static Type fromTsDataType(TSDataType tsDataType) { + return TypeFactory.getType(tsDataType); + } + /** Gets a boolean at {@code position}. */ default boolean getBoolean(Column c, int position) { throw new UnsupportedOperationException(getClass().getName()); @@ -62,6 +98,89 @@ default Object getObject(Column c, int position) { return c.getObject(position); } + /** Creates an empty TsPrimitiveType for this type. */ + default TsPrimitiveType getTsPrimitiveType() { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Creates a TsPrimitiveType initialized with {@code value}. */ + default TsPrimitiveType getTsPrimitiveType(Object value) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Serializes {@code value} into {@code valueBytes} starting at {@code offset}. */ + default void toBytes(TsPrimitiveType value, byte[] valueBytes, int offset) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the number of bytes required to serialize {@code value}. */ + default int calcTypeSize(TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the number of bytes required to serialize {@code value}. */ + default int calcTypeSize(Object value) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Initializes the value storage in {@code batchData} for this type. */ + default void init(BatchData batchData) { + throw new UnSupportedDataTypeException(String.valueOf(batchData.getDataType())); + } + + /** Adds a timestamp-value pair to {@code batchData}. */ + default void put(BatchData batchData, long timestamp, Object value) { + throw new UnSupportedDataTypeException(String.valueOf(batchData.getDataType())); + } + + /** Serializes all timestamp-value pairs in {@code batchData} and returns the serialized size. */ + default int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + throw new IllegalArgumentException( + Messages.format("error.read.batch_data_unknown_type", batchData.getDataType())); + } + + /** Serializes {@code value} to {@code stream} and returns the serialized size. */ + default int serialize(TsPrimitiveType value, DataOutputStream stream) throws IOException { + throw new IllegalArgumentException( + Messages.format("error.read.batch_data_unknown_type", value.getDataType())); + } + + /** Serializes {@code value} to {@code buffer} and returns the serialized size. */ + default int serializeValue(Object value, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Serializes {@code value} to {@code stream} and returns the serialized size. */ + default int serializeValue(Object value, DataOutputStream stream) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Writes a TsPrimitiveType value to the current entry. */ + default void write(ColumnBuilder builder, TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Writes the value serialized in {@code bytes} at {@code offset} to the current entry. */ + default void write(ColumnBuilder builder, byte[] bytes, int offset) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Writes the value at {@code index} in {@code column} to the current entry. */ + default void write(ColumnBuilder builder, Column column, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Writes a TsPrimitiveType value to {@code array} at {@code index}. */ + default void write(TsPrimitiveType from, Object toArray, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Writes the current value from {@code from} to {@code toArray} at {@code index}. */ + default void write(BatchData from, Object toArray, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + /** Write a boolean to the current entry; */ default void writeBoolean(ColumnBuilder builder, boolean value) { throw new UnsupportedOperationException(getClass().getName()); @@ -97,12 +216,323 @@ default void writeObject(ColumnBuilder builder, Object value) { builder.writeObject(value); } + /** Adds a value to the array at {@code rowIndex}. */ + default void addValue(int rowIndex, Object value, Object array) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Copies one element from {@code source} to {@code target}. */ + default void copyArrayElement(Object source, int sourceIndex, Object target, int targetIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Copies {@code array}, truncating or padding it to {@code newLength}. */ + default Object arrayCopyOf(Object array, int newLength) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Creates an array column with {@code capacity} entries for this type. */ + default Object createArray(int capacity) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Estimates the retained memory size of an array with {@code size} entries. */ + default long estimateArraySize(int size) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Estimates the retained memory size of {@code array}. */ + default long estimateArraySize(Object array) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Estimates the retained memory size of a value of this type. */ + default long estimateValueSize() { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the serialized byte size of the array with {@code rowSize} entries. */ + default int serializedSize(Object array, int rowSize) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the serialized byte size of the array within the given range. */ + default int serializedSize(Object array, int startRow, int endRow) { + return serializedSize(array, endRow - startRow); + } + + /** Serializes the array column with {@code rowSize} entries. */ + default void serializeArray(Object array, int rowSize, DataOutputStream stream) + throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Serializes the first {@code length} entries in {@code array} to {@code buffer}. */ + default void serializeArray(Object array, int length, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Deserializes an array with {@code rowSize} entries from {@code buffer}. */ + default Object deserializeArray(ByteBuffer buffer, int rowSize) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Deserializes an array with {@code rowSize} entries from {@code stream}. */ + default Object deserializeArray(DataInputStream stream, int rowSize) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Deserializes one value from {@code buffer} into {@code array} at {@code index}. */ + default void deserialize(Object[] array, int index, ByteBuffer buffer) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Deserializes one value from {@code stream} into {@code array} at {@code index}. */ + default void deserialize(Object[] array, int index, InputStream stream) throws IOException { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Deserializes an array with {@code rowSize} entries from {@code buffer}. */ + default Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns whether two array columns are equal in the first {@code rowSize} entries. */ + default boolean arrayEquals(Object left, Object right, int rowSize) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the value at {@code rowIndex} from an array. */ + default Object getValue(Object array, int rowIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the value at {@code rowIndex} from an array as a TsPrimitiveType. */ + default TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the value in {@code field}. */ + default Object getValue(Field field) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + + /** Creates a field initialized with {@code value}. */ + default Field getField(Object value) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + + /** Copies the value from {@code from} to {@code to}. */ + default void setTo(TsPrimitiveType from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + + /** Copies the value from {@code from} to {@code to} at {@code index}. */ + default void setTo(TsPrimitiveType from, Column to, int index) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + + /** Copies {@code value} to {@code to} from {@code startIndex} to {@code endIndex}. */ + default void setTo(Object value, Column to, int startIndex, int endIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Copies the value from {@code from} to {@code to}. */ + default void setTo(Field from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + + /** Copies the current value from {@code from} to {@code to}. */ + default void setTo(BatchData from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + + /** Copies the current value from {@code from} to {@code to} at {@code index}. */ + default void setTo(BatchData from, Column to, int index) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + + /** Copies a value from {@code from} at {@code fromIndex} to {@code to} at {@code toIndex}. */ + default void setTo(Column from, int fromIndex, Column to, int toIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** + * Copies a value from {@code from} at {@code fromIndex} to {@code toArray} at {@code toIndex}. + */ + default void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Adds the value of {@code columnName} in {@code resultSet} to {@code record}. */ + default void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", this.getTypeEnum())); + } + + /** Adds the value in {@code field} to {@code record}. */ + default void addPoint(TSRecord record, String columnName, Field field) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", this.getTypeEnum())); + } + + /** Creates a data point by parsing a string value. */ + default DataPoint getDataPoint(String measurementId, String value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Creates a data point from a long value for test file generation. */ + default DataPoint getDataPoint(String measurementId, long value) { + return new StringDataPoint( + measurementId, new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + } + + /** Writes a value to an aligned value chunk. */ + default void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes a TsPrimitiveType value to an aligned value chunk. */ + default void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes a value at {@code rowIndex} in a Tablet column to an aligned value chunk. */ + default void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes a value at {@code rowIndex} in a TsBlock column to an aligned value chunk. */ + default void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes a batch of values in a TsBlock column to an aligned value chunk. */ + default void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + throw new UnsupportedOperationException( + Messages.format("error.write.chunk_unknown_type", getTypeEnum())); + } + + /** Writes a value at {@code rowIndex} in a Tablet column to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes a TsPrimitiveType value to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes the current value in {@code data} to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, BatchData data) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Writes a value to a non-aligned chunk. */ + default void write(ChunkWriterImpl writer, long time, Object value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + /** Creates a column with {@code positionCount} capacity for this type. */ + default Column createColumnWithMaxPosition(int positionCount) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Creates a column with {@code positionCount} capacity for this type. */ + default Column createColumnWithZeroPosition(int positionCount) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Creates an all-null column with {@code positionCount} positions. */ + default Column createNullColumn(int positionCount) { + throw new IllegalArgumentException( + Messages.format("error.read.null_col_unknown_type", getTypeEnum())); + } + + /** Returns the default encoding configured for this type. */ + default TSEncoding getDefaultEncoding(TSFileConfig config) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the default compressor configured for this type. */ + default CompressionType getDefaultCompressor(TSFileConfig config) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns a decoder for the specified encoding. */ + default Decoder getDecoder(TSEncoding encoding) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the maximum encoded size of one item. */ + default int getOneItemMaxSize(int valveLength) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Creates empty statistics for this type. */ + default Statistics createStatistics() { + throw new UnknownColumnTypeException(getTypeEnum().toString()); + } + + /** Returns the estimated instance size of statistics for this type. */ + default long getStatisticsInstanceSize() { + throw new UnknownColumnTypeException(getTypeEnum().toString()); + } + + /** Updates statistics with a timestamp-value pair. */ + default void update(Statistics stats, long timestamp, TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Updates statistics with the current timestamp-value pair in {@code batchData}. */ + default void update(Statistics stats, BatchData batchData) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Updates statistics with the value at {@code rowIndex} in a TsBlock column. */ + default void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the current value in {@code batchData}. */ + default Object getCurrentValue(BatchData batchData) { + throw new UnsupportedOperationException(getClass().getName()); + } + + /** Returns the string representation of the value in {@code field}. */ + default String toString(Field field) { + throw new UnSupportedDataTypeException(field.getDataType().toString()); + } + /** * Creates the preferred column builder for this type. This is the builder used to store values * after an expression projection within the read. */ ColumnBuilder createColumnBuilder(int expectedEntries); + /** Creates the preferred column builder for this type with memory tracking. */ + default ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + throw new UnsupportedOperationException(getClass().getName()); + } + TypeEnum getTypeEnum(); /** Returns the name of this type that should be displayed to end-users. */ diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java index 52db17a39..8bcab3d10 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java @@ -39,5 +39,6 @@ public enum TypeEnum { DATE, BLOB, STRING, - OBJECT + OBJECT, + VECTOR } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java index bebefbdea..7e4d542f0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java @@ -29,62 +29,43 @@ private TypeFactory() { } public static Type getType(TSDataType tsDataType) { - switch (tsDataType) { - case DATE: - return DateType.getInstance(); - case INT32: - return IntType.getInstance(); - case INT64: - return LongType.getInstance(); - case TIMESTAMP: - return TimestampType.getInstance(); - case FLOAT: - return FloatType.getInstance(); - case DOUBLE: - return DoubleType.getInstance(); - case BOOLEAN: - return BooleanType.getInstance(); - case TEXT: - return BinaryType.getInstance(); - case STRING: - return StringType.getInstance(); - case BLOB: - return BlobType.getInstance(); - case OBJECT: - return ObjectType.getInstance(); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.typefactory_invalid_tsdata_type", tsDataType)); - } + return switch (tsDataType) { + case DATE -> DateType.getInstance(); + case INT32 -> IntType.getInstance(); + case INT64 -> LongType.getInstance(); + case TIMESTAMP -> TimestampType.getInstance(); + case FLOAT -> FloatType.getInstance(); + case DOUBLE -> DoubleType.getInstance(); + case BOOLEAN -> BooleanType.getInstance(); + case TEXT -> BinaryType.getInstance(); + case STRING -> StringType.getInstance(); + case BLOB -> BlobType.getInstance(); + case OBJECT -> ObjectType.getInstance(); + case VECTOR -> VectorType.getInstance(); + case UNKNOWN -> UnknownType.getInstance(); + default -> + throw new UnsupportedOperationException( + Messages.format("error.read.typefactory_invalid_tsdata_type", tsDataType)); + }; } public static Type getType(TypeEnum typeEnum) { - switch (typeEnum) { - case INT32: - return IntType.getInstance(); - case INT64: - return LongType.getInstance(); - case FLOAT: - return FloatType.getInstance(); - case DOUBLE: - return DoubleType.getInstance(); - case BOOLEAN: - return BooleanType.getInstance(); - case TEXT: - return BinaryType.getInstance(); - case UNKNOWN: - return UnknownType.getInstance(); - case DATE: - return DateType.getInstance(); - case TIMESTAMP: - return TimestampType.getInstance(); - case BLOB: - return BlobType.getInstance(); - case STRING: - return StringType.getInstance(); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.typefactory_invalid_type_enum", typeEnum)); - } + return switch (typeEnum) { + case INT32 -> IntType.getInstance(); + case INT64 -> LongType.getInstance(); + case FLOAT -> FloatType.getInstance(); + case DOUBLE -> DoubleType.getInstance(); + case BOOLEAN -> BooleanType.getInstance(); + case TEXT -> BinaryType.getInstance(); + case UNKNOWN -> UnknownType.getInstance(); + case DATE -> DateType.getInstance(); + case TIMESTAMP -> TimestampType.getInstance(); + case BLOB -> BlobType.getInstance(); + case STRING -> StringType.getInstance(); + case VECTOR -> VectorType.getInstance(); + default -> + throw new UnsupportedOperationException( + Messages.format("error.read.typefactory_invalid_type_enum", typeEnum)); + }; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java index 802165dc8..4df624811 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java @@ -21,6 +21,9 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; import java.util.Collections; @@ -51,11 +54,23 @@ public boolean getBoolean(Column column, int position) { return false; } + @Override + public Object getCurrentValue(BatchData batchData) { + return null; + } + @Override public ColumnBuilder createColumnBuilder(int expectedEntries) { return new BooleanColumnBuilder(null, expectedEntries); } + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + throw new IllegalArgumentException( + Messages.format("error.read.tsblock_builder_unknown_type", getTypeEnum())); + } + @Override public TypeEnum getTypeEnum() { return TypeEnum.UNKNOWN; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java new file mode 100644 index 000000000..680900af4 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/VectorType.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.common.type; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DeltaBinaryDecoder; +import org.apache.tsfile.encoding.decoder.LongChimpDecoder; +import org.apache.tsfile.encoding.decoder.LongGorillaDecoder; +import org.apache.tsfile.encoding.decoder.LongRleDecoder; +import org.apache.tsfile.encoding.decoder.RegularDataDecoder; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.file.metadata.statistics.TimeStatistics; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class VectorType extends AbstractLongType { + + public static final VectorType VECTOR = new VectorType(); + + private VectorType() {} + + @Override + public DataPoint getDataPoint(String measurementId, String value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public DataPoint getDataPoint(String measurementId, long value) { + return new StringDataPoint( + measurementId, new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + } + + @Override + public void write(ValueChunkWriter writer, long time, Object value, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write(ValueChunkWriter writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write(ColumnBuilder builder, TsPrimitiveType value) { + throw new UnsupportedOperationException(getClass().getName()); + } + + @Override + public void write(ColumnBuilder builder, byte[] bytes, int offset) { + throw new UnsupportedOperationException(getClass().getName()); + } + + @Override + public void write(ColumnBuilder builder, Column column, int index) { + throw new UnsupportedOperationException(getClass().getName()); + } + + @Override + public void write(TsPrimitiveType from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); + } + + @Override + public void write(BatchData from, Object toArray, int index) { + ((TsPrimitiveType[][]) toArray)[index] = from.getVector(); + } + + @Override + public Object arrayCopyOf(Object array, int newLength) { + return Arrays.copyOf((long[]) array, newLength); + } + + @Override + public void setTo(Column from, int fromIndex, Object toArray, int toIndex) { + ((long[]) toArray)[toIndex] = from.getLong(fromIndex); + } + + @Override + public void setTo(Column from, int fromIndex, Column to, int toIndex) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Object array, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write( + ValueChunkWriter writer, long time, Column column, int rowIndex, boolean isNull) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write( + ValueChunkWriter writer, long[] times, Column column, int batchSize, int arrayOffset) { + throw new UnsupportedOperationException( + Messages.format("error.write.chunk_unknown_type", getTypeEnum())); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object column, int rowIndex) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write(ChunkWriterImpl writer, long time, TsPrimitiveType value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write(ChunkWriterImpl writer, long time, BatchData data) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public void write(ChunkWriterImpl writer, long time, Object value) { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", getTypeEnum())); + } + + @Override + public ColumnBuilder createColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { + throw new IllegalArgumentException( + Messages.format("error.read.tsblock_builder_unknown_type", getTypeEnum())); + } + + @Override + public void init(BatchData batchData) { + batchData.initVectorValues(); + } + + @Override + public void put(BatchData batchData, long timestamp, Object value) { + batchData.putVector(timestamp, (TsPrimitiveType[]) value); + } + + @Override + public int serialize(BatchData batchData, DataOutputStream outputStream, boolean isDesc) + throws IOException { + int size = 0; + for (int i = 0; i < batchData.length(); i++) { + int index = isDesc ? batchData.length() - 1 - i : i; + outputStream.writeLong(batchData.getTimeByIndex(index)); + TsPrimitiveType[] values = batchData.getVectorByIndex(index); + outputStream.writeInt(values.length); + size = Math.addExact(size, Long.BYTES + Integer.BYTES); + for (TsPrimitiveType value : values) { + size = Math.addExact(size, Byte.BYTES); + if (value == null) { + outputStream.write(0); + continue; + } + outputStream.write(1); + outputStream.write(value.getDataType().serialize()); + size = Math.addExact(size, Byte.BYTES); + size = + Math.addExact( + size, Type.fromTsDataType(value.getDataType()).serialize(value, outputStream)); + } + } + return size; + } + + @Override + public int serialize(TsPrimitiveType value, DataOutputStream stream) { + throw new IllegalArgumentException( + Messages.format("error.read.batch_data_unknown_type", value.getDataType())); + } + + @Override + public int serializeValue(Object value, ByteBuffer buffer) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public int serializeValue(Object value, DataOutputStream stream) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void serializeArray(Object array, int length, ByteBuffer buffer) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public Decoder getDecoder(TSEncoding encoding) { + return switch (encoding) { + case PLAIN, DICTIONARY -> super.getDecoder(encoding); + case RLE -> new LongRleDecoder(); + case TS_2DIFF -> new DeltaBinaryDecoder.LongDeltaDecoder(); + case REGULAR -> new RegularDataDecoder.LongRegularDecoder(); + case GORILLA -> new LongGorillaDecoder(); + case CHIMP -> new LongChimpDecoder(); + default -> throw decoderNotFound(encoding); + }; + } + + @Override + public int getOneItemMaxSize(int valveLength) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public Statistics createStatistics() { + return new TimeStatistics(); + } + + @Override + public long getStatisticsInstanceSize() { + return TimeStatistics.INSTANCE_SIZE; + } + + @Override + public void update(Statistics stats, long timestamp, TsPrimitiveType value) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void update(Statistics stats, BatchData batchData) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void update(Statistics stats, TsBlock block, int columnIndex, int rowIndex) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public Object getCurrentValue(BatchData batchData) { + return batchData.getVector(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType() { + return new TsPrimitiveType.TsVector(); + } + + @Override + public TsPrimitiveType getTsPrimitiveType(Object value) { + return new TsPrimitiveType.TsVector((TsPrimitiveType[]) value); + } + + @Override + public TsPrimitiveType getValueAsTsPrimitiveType(Object array, int rowIndex) { + return new TsPrimitiveType.TsVector(((TsPrimitiveType[][]) array)[rowIndex]); + } + + @Override + public void addPoint(TSRecord record, String columnName, ResultSet resultSet) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", getTypeEnum())); + } + + @Override + public void addPoint(TSRecord record, String columnName, Field field) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", getTypeEnum())); + } + + @Override + public Object deserializeColumn(ByteBuffer buffer, int rowSize, boolean[] nullIndicators) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void deserialize(Object[] array, int index, ByteBuffer buffer) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public void deserialize(Object[] array, int index, InputStream stream) { + throw new UnsupportedOperationException(getDisplayName()); + } + + @Override + public Column createNullColumn(int positionCount) { + throw new IllegalArgumentException( + Messages.format("error.read.null_col_unknown_type", getTypeEnum())); + } + + @Override + public String toString(Field field) { + throw new UnSupportedDataTypeException(field.getDataType().toString()); + } + + @Override + public Object getValue(Field field) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + + @Override + public Field getField(Object value) { + throw new UnSupportedDataTypeException(getTypeEnum().toString()); + } + + @Override + public void setTo(TsPrimitiveType from, Field to) { + throw new UnSupportedDataTypeException( + Messages.format("error.common.unsupported_data_type", from.getDataType())); + } + + @Override + public void setTo(BatchData from, Field to) { + TsPrimitiveType value = from.getVector()[0]; + Type.fromTsDataType(value.getDataType()).setTo(value, to); + } + + @Override + public long estimateValueSize() { + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + } + + @Override + public TypeEnum getTypeEnum() { + return TypeEnum.VECTOR; + } + + @Override + public String getDisplayName() { + return "VECTOR"; + } + + public static VectorType getInstance() { + return VECTOR; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/BooleanTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/BooleanTypeService.java new file mode 100644 index 000000000..22e3ce7fa --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/BooleanTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A BooleanTypeService wraps a switch(dataType) clause into a checked service class. + * Implementations should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns boolean only. + */ +public interface BooleanTypeService { + + boolean call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/DoubleTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/DoubleTypeService.java new file mode 100644 index 000000000..7f3af905d --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/DoubleTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A DoubleTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns double only. + */ +public interface DoubleTypeService { + + double call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/FloatTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/FloatTypeService.java new file mode 100644 index 000000000..2d19f6abc --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/FloatTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A FloatTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns float only. + */ +public interface FloatTypeService { + + float call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/IntTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/IntTypeService.java new file mode 100644 index 000000000..e0900b299 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/IntTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * An IntTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns int only. + */ +public interface IntTypeService { + + int call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/LongTypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/LongTypeService.java new file mode 100644 index 000000000..c589fb060 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/LongTypeService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; + +/** + * A LongTypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

It returns long only. + */ +public interface LongTypeService { + + long call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + call(type); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java new file mode 100644 index 000000000..7c4b28615 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/service/TypeService.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.read.common.type.service; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +/** + * A TypeService wraps a switch(dataType) clause into a checked service class. Implementations + * should cover all Types that can be instanced from TsDataTypes, and throw + * UnsupportedDataTypeException when an unrecognized type is encountered. + * + *

Use enhanced-switch and list all branches in the implementation to expose uncovered datatypes + * during compilation. + * + * @param the return type of the service. + */ +public interface TypeService { + + T call(Type type); + + default void check() { + for (TSDataType tsDataType : TSDataType.values()) { + Type type = null; + try { + type = Type.fromTsDataType(tsDataType); + } catch (UnsupportedOperationException e) { + // ignore types that cannot be instanced + } + + if (type != null) { + try { + call(type); + } catch (UnSupportedDataTypeException e) { + if (!e.isChecked()) { + // the exception is caused by an uncovered datatype, throw it to fail any tests + // involving this service + throw e; + } + } + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java index 4423792f5..2b25915d5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/ValueFilterApi.java @@ -21,20 +21,12 @@ import org.apache.tsfile.common.regexp.LikePattern; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; -import org.apache.tsfile.read.filter.operator.BinaryFilterOperators; -import org.apache.tsfile.read.filter.operator.BooleanFilterOperators; -import org.apache.tsfile.read.filter.operator.DoubleFilterOperators; import org.apache.tsfile.read.filter.operator.ExtractTimeFilterOperators; import org.apache.tsfile.read.filter.operator.ExtractValueFilterOperators; -import org.apache.tsfile.read.filter.operator.FloatFilterOperators; -import org.apache.tsfile.read.filter.operator.IntegerFilterOperators; -import org.apache.tsfile.read.filter.operator.LongFilterOperators; -import org.apache.tsfile.read.filter.operator.StringFilterOperators; import org.apache.tsfile.read.filter.operator.ValueIsNotNullOperator; import org.apache.tsfile.read.filter.operator.ValueIsNullOperator; -import org.apache.tsfile.utils.Binary; import java.time.ZoneId; import java.util.Objects; @@ -42,6 +34,21 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_BETWEEN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_GT_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_GT_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_IN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_LIKE_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_LT_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_LT_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_BETWEEN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_EQ_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_IN_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_LIKE_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_NOT_REGEXP_FILTER_SERVICE; +import static org.apache.tsfile.utils.ValueFilterTypeServices.VALUE_REGEXP_FILTER_SERVICE; + public class ValueFilterApi { public static final int DEFAULT_MEASUREMENT_INDEX = 0; @@ -56,166 +63,43 @@ private ValueFilterApi() { public static Filter gt(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueGt(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueGt(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueGt(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueGt(measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueGt(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueGt(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueGt(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_GT_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, value); } public static Filter gtEq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueGtEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueGtEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueGtEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueGtEq( - measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueGtEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueGtEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueGtEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_GT_EQ_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value); } public static Filter lt(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueLt(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueLt(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueLt(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueLt(measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueLt(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueLt(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueLt(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_LT_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, value); } public static Filter ltEq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueLtEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueLtEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueLtEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueLtEq( - measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueLtEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueLtEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueLtEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_LT_EQ_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value); } public static Filter eq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueEq(measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_EQ_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, value); } public static Filter notEq(int measurementIndex, Object value, TSDataType type) { // constant cannot be null Objects.requireNonNull(value, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotEq(measurementIndex, (boolean) value); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotEq(measurementIndex, ((Number) value).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotEq(measurementIndex, ((Number) value).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueNotEq( - measurementIndex, ((Number) value).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueNotEq(measurementIndex, ((Number) value).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotEq(measurementIndex, (Binary) value); - case STRING: - return new StringFilterOperators.ValueNotEq(measurementIndex, (Binary) value); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_EQ_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value); } public static Filter isNull(int measurementIndex) { @@ -231,35 +115,9 @@ public static Filter between( // constant cannot be null Objects.requireNonNull(value1, CONSTANT_CANNOT_BE_NULL_MSG); Objects.requireNonNull(value2, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueBetweenAnd( - measurementIndex, (boolean) value1, (boolean) value2); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).intValue(), ((Number) value2).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).longValue(), ((Number) value2).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueBetweenAnd( - measurementIndex, ((Number) value1).floatValue(), ((Number) value2).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - case STRING: - return new StringFilterOperators.ValueBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_BETWEEN_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value1, value2); } public static Filter notBetween( @@ -267,195 +125,53 @@ public static Filter notBetween( // constant cannot be null Objects.requireNonNull(value1, CONSTANT_CANNOT_BE_NULL_MSG); Objects.requireNonNull(value2, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotBetweenAnd( - measurementIndex, (boolean) value1, (boolean) value2); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).intValue(), ((Number) value2).intValue()); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).longValue(), ((Number) value2).longValue()); - case DOUBLE: - return new DoubleFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); - case FLOAT: - return new FloatFilterOperators.ValueNotBetweenAnd( - measurementIndex, ((Number) value1).floatValue(), ((Number) value2).floatValue()); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - case STRING: - return new StringFilterOperators.ValueNotBetweenAnd( - measurementIndex, (Binary) value1, (Binary) value2); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_BETWEEN_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, value1, value2); } public static Filter like(int measurementIndex, LikePattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueLike(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueLike(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueLike(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueLike(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueLike(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueLike(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueLike(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_LIKE_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static Filter notLike(int measurementIndex, LikePattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotLike(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotLike(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotLike(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueNotLike(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueNotLike(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotLike(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueNotLike(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_LIKE_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static Filter regexp(int measurementIndex, Pattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueRegexp(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueRegexp(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueRegexp(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueRegexp(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueRegexp(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueRegexp(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueRegexp(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_REGEXP_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static Filter notRegexp(int measurementIndex, Pattern pattern, TSDataType type) { Objects.requireNonNull(pattern, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case DOUBLE: - return new DoubleFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case FLOAT: - return new FloatFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotRegexp(measurementIndex, pattern); - case STRING: - return new StringFilterOperators.ValueNotRegexp(measurementIndex, pattern); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_REGEXP_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, pattern); } public static > Filter in( int measurementIndex, Set values, TSDataType type) { // constants cannot be null Objects.requireNonNull(values, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueIn(measurementIndex, (Set) values); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueIn(measurementIndex, (Set) values); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueIn(measurementIndex, (Set) values); - case FLOAT: - return new FloatFilterOperators.ValueIn(measurementIndex, (Set) values); - case DOUBLE: - return new DoubleFilterOperators.ValueIn(measurementIndex, (Set) values); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueIn(measurementIndex, (Set) values); - case STRING: - return new StringFilterOperators.ValueIn(measurementIndex, (Set) values); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_IN_FILTER_SERVICE.call(Type.fromTsDataType(type)).create(measurementIndex, values); } public static > Filter notIn( int measurementIndex, Set values, TSDataType type) { // constants cannot be null Objects.requireNonNull(values, CONSTANT_CANNOT_BE_NULL_MSG); - switch (type) { - case BOOLEAN: - return new BooleanFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case INT32: - case DATE: - return new IntegerFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case INT64: - case TIMESTAMP: - return new LongFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case FLOAT: - return new FloatFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case DOUBLE: - return new DoubleFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case TEXT: - case BLOB: - return new BinaryFilterOperators.ValueNotIn(measurementIndex, (Set) values); - case STRING: - return new StringFilterOperators.ValueNotIn(measurementIndex, (Set) values); - default: - throw new UnsupportedOperationException( - Messages.format("error.read.filter_api_unsupported_type", type)); - } + return VALUE_NOT_IN_FILTER_SERVICE + .call(Type.fromTsDataType(type)) + .create(measurementIndex, values); } public static Filter extractValueGt( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java index 4e3b870ca..966131732 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java @@ -20,13 +20,12 @@ package org.apache.tsfile.read.query.dataset; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.series.AbstractFileSeriesReader; -import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.IOException; import java.util.ArrayList; @@ -152,44 +151,10 @@ private Long timeHeapGet() { } private Field putValueToField(BatchData col) { - TSDataType type = col.getDataType(); - Field field; - if (type == TSDataType.VECTOR) { - field = new Field((col.getVector())[0].getDataType()); - } else { - field = new Field(col.getDataType()); - } - switch (col.getDataType()) { - case BOOLEAN: - field.setBoolV(col.getBoolean()); - break; - case INT32: - case DATE: - field.setIntV(col.getInt()); - break; - case INT64: - case TIMESTAMP: - field.setLongV(col.getLong()); - break; - case FLOAT: - field.setFloatV(col.getFloat()); - break; - case DOUBLE: - field.setDoubleV(col.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - field.setBinaryV(col.getBinary()); - break; - case VECTOR: - Field.setTsPrimitiveValue((col.getVector())[0], field); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.read.dataset_unsupported_type", col.getDataType())); - } + TSDataType dataType = col.getDataType(); + Field field = + new Field(dataType == TSDataType.VECTOR ? col.getVector()[0].getDataType() : dataType); + Type.fromTsDataType(dataType).setTo(col, field); return field; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java index dabe7533b..74f501429 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java @@ -22,6 +22,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.block.TsBlockReader; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.record.TSRecord; @@ -184,36 +185,7 @@ private boolean cacheNextRecord() throws IOException { cachedRecord.dataPointList.add(null); continue; } - switch (dataType) { - case INT32: - case DATE: - cachedRecord.addPoint(columnName, getInt(columnName)); - break; - case INT64: - case TIMESTAMP: - cachedRecord.addPoint(columnName, getLong(columnName)); - break; - case FLOAT: - cachedRecord.addPoint(columnName, getFloat(columnName)); - break; - case DOUBLE: - cachedRecord.addPoint(columnName, getDouble(columnName)); - break; - case STRING: - case TEXT: - cachedRecord.addPoint(columnName, getString(columnName)); - break; - case BLOB: - cachedRecord.addPoint(columnName, getBinary(columnName)); - break; - case BOOLEAN: - cachedRecord.addPoint(columnName, getBoolean(columnName)); - break; - case VECTOR: - case UNKNOWN: - default: - break; - } + Type.fromTsDataType(dataType).addPoint(cachedRecord, columnName, TableResultSet.this); } return true; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java index 9483e1e7e..6321e0622 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java @@ -23,6 +23,7 @@ import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.write.record.TSRecord; import java.io.IOException; @@ -119,36 +120,7 @@ private boolean fetchRecords() throws IOException { Integer pathIdx = pathIndexMap.get(path); if (pathIdx != null) { Field field = currentRow.getField(pathIdx); - switch (field.getDataType()) { - case INT32: - case DATE: - record.addPoint(measurement, field.getIntV()); - break; - case INT64: - case TIMESTAMP: - record.addPoint(measurement, field.getLongV()); - break; - case FLOAT: - record.addPoint(measurement, field.getFloatV()); - break; - case DOUBLE: - record.addPoint(measurement, field.getDoubleV()); - break; - case STRING: - case TEXT: - record.addPoint(measurement, field.getStringValue()); - break; - case BLOB: - record.addPoint(measurement, field.getBinaryV().getValues()); - break; - case BOOLEAN: - record.addPoint(measurement, field.getBoolV()); - break; - case VECTOR: - case UNKNOWN: - default: - break; - } + Type.fromTsDataType(field.getDataType()).addPoint(record, measurement, field); } else { record.dataPointList.add(null); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java index 83a454007..8700a850e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java @@ -32,6 +32,7 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.read.reader.page.ValuePageReader; import org.apache.tsfile.utils.Pair; @@ -231,11 +232,14 @@ private Pair convertToLastPoint( seriesMeta.getMeasurementId(), new TimeValuePair( seriesMeta.getStatistics().getEndTime(), - seriesMeta.getTsDataType() == TSDataType.VECTOR - ? TsPrimitiveType.getByType( - TSDataType.INT64, seriesMeta.getStatistics().getEndTime()) - : TsPrimitiveType.getByType( - seriesMeta.getTsDataType(), seriesMeta.getStatistics().getLastValue()))); + Type.fromTsDataType( + seriesMeta.getTsDataType() == TSDataType.VECTOR + ? TSDataType.INT64 + : seriesMeta.getTsDataType()) + .getTsPrimitiveType( + seriesMeta.getTsDataType() == TSDataType.VECTOR + ? seriesMeta.getStatistics().getEndTime() + : seriesMeta.getStatistics().getLastValue()))); } else { return readLastPoint(seriesMeta, isAligned); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java index 8eaab414f..e149a648a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/block/SingleDeviceTsBlockReader.java @@ -25,6 +25,7 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.controller.IMetadataQuerier; import org.apache.tsfile.read.expression.ExpressionTree; @@ -32,16 +33,13 @@ import org.apache.tsfile.read.query.executor.task.DeviceQueryTask; import org.apache.tsfile.read.reader.series.AbstractFileSeriesReader; import org.apache.tsfile.read.reader.series.FileSeriesReader; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -232,69 +230,12 @@ private void advanceColumn(BatchData batchData, MeasurementColumnContext columnC } private void fillIdColumn(Column column, Object val, int startPos, int endPos) { - switch (column.getDataType()) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - if (val instanceof String) { - val = new Binary(((String) val), StandardCharsets.UTF_8); - } - Arrays.fill(column.getBinaries(), startPos, endPos, val); - break; - case BOOLEAN: - Arrays.fill(column.getBooleans(), startPos, endPos, ((boolean) val)); - break; - case INT32: - case DATE: - Arrays.fill(column.getInts(), startPos, endPos, ((int) val)); - break; - case INT64: - case TIMESTAMP: - Arrays.fill(column.getLongs(), startPos, endPos, ((long) val)); - break; - case FLOAT: - Arrays.fill(column.getFloats(), startPos, endPos, ((float) val)); - break; - case DOUBLE: - Arrays.fill(column.getDoubles(), startPos, endPos, ((double) val)); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.block_reader_unsupported_type", column.getDataType())); - } + Type.fromTsDataType(column.getDataType()).setTo(val, column, startPos, endPos); column.setPositionCount(endPos); } private static void fillSingleMeasurementColumn(Column column, BatchData batchData, int pos) { - switch (batchData.getDataType()) { - case BOOLEAN: - column.getBooleans()[pos] = batchData.getBoolean(); - break; - case DOUBLE: - column.getDoubles()[pos] = batchData.getDouble(); - break; - case FLOAT: - column.getFloats()[pos] = batchData.getFloat(); - break; - case INT32: - case DATE: - column.getInts()[pos] = batchData.getInt(); - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - column.getBinaries()[pos] = batchData.getBinary(); - break; - case INT64: - case TIMESTAMP: - column.getLongs()[pos] = batchData.getLong(); - break; - default: - throw new IllegalArgumentException( - Messages.format("error.read.block_reader_unsupported_type", batchData.getDataType())); - } + Type.fromTsDataType(batchData.getDataType()).setTo(batchData, column, pos); column.setPositionCount(pos + 1); } @@ -383,35 +324,8 @@ void fillInto(TsBlock block, int blockRowNum) { final List columnPositions = posInResult.get(i); for (Integer pos : columnPositions) { if (value != null) { - switch (value.getDataType()) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - block.getColumn(pos).getBinaries()[blockRowNum] = value.getBinary(); - break; - case INT32: - case DATE: - block.getColumn(pos).getInts()[blockRowNum] = value.getInt(); - break; - case INT64: - case TIMESTAMP: - block.getColumn(pos).getLongs()[blockRowNum] = value.getLong(); - break; - case BOOLEAN: - block.getColumn(pos).getBooleans()[blockRowNum] = value.getBoolean(); - break; - case FLOAT: - block.getColumn(pos).getFloats()[blockRowNum] = value.getFloat(); - break; - case DOUBLE: - block.getColumn(pos).getDoubles()[blockRowNum] = value.getDouble(); - break; - default: - throw new IllegalArgumentException( - Messages.format( - "error.read.block_reader_unsupported_type", value.getDataType())); - } + Type.fromTsDataType(value.getDataType()) + .setTo(value, block.getColumn(pos), blockRowNum); } else { block.getColumn(pos).setNull(blockRowNum, blockRowNum + 1); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java index 2db020cf8..60a877a41 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java @@ -19,7 +19,6 @@ package org.apache.tsfile.read.reader.page; -import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.PageHeader; @@ -29,14 +28,16 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; -import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.reader.IPageReader; import org.apache.tsfile.read.reader.series.PaginationController; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.PageDataBlockValueReader; +import org.apache.tsfile.utils.TypeServices.PageDataReadStatus; +import org.apache.tsfile.utils.TypeServices.PageDataValueReader; import java.io.IOException; import java.io.Serializable; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Optional; import java.util.function.LongConsumer; +import java.util.function.LongPredicate; import static org.apache.tsfile.read.reader.series.PaginationController.UNLIMITED_PAGINATION_CONTROLLER; import static org.apache.tsfile.utils.Preconditions.checkArgument; @@ -149,64 +151,18 @@ private void uncompressDataIfNecessary() throws IOException { /** * @return the returned BatchData may be empty, but never be null */ - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning @Override public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { uncompressDataIfNecessary(); BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false); boolean allSatisfy = recordFilter == null || recordFilter.allSatisfy(this); + PageDataValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_BATCHDATA_SERVICE.call(Type.fromTsDataType(dataType)); + LongPredicate isDeleted = this::isDeleted; while (timeDecoder.hasNext(timeBuffer)) { long timestamp = timeDecoder.readLong(timeBuffer); - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyBoolean(timestamp, aBoolean))) { - pageData.putBoolean(timestamp, aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyInteger(timestamp, anInt))) { - pageData.putInt(timestamp, anInt); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timestamp) && (allSatisfy || recordFilter.satisfyLong(timestamp, aLong))) { - pageData.putLong(timestamp, aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyFloat(timestamp, aFloat))) { - pageData.putFloat(timestamp, aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyDouble(timestamp, aDouble))) { - pageData.putDouble(timestamp, aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timestamp) - && (allSatisfy || recordFilter.satisfyBinary(timestamp, aBinary))) { - pageData.putBinary(timestamp, aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read( + valueDecoder, valueBuffer, recordFilter, pageData, timestamp, allSatisfy, isDeleted); } return pageData.flip(); } @@ -220,175 +176,35 @@ public TsBlock getAllSatisfiedData() throws IOException { public TsBlock getAllSatisfiedData(LongConsumer filterRowsRecorder) throws IOException { uncompressDataIfNecessary(); TsBlockBuilder builder; - int initialExpectedEntries = (int) pageHeader.getStatistics().getCount(); + int initialExpectedEntries = pageHeader.getStatistics().getCount(); if (paginationController.hasSetLimit()) { initialExpectedEntries = (int) Math.min(initialExpectedEntries, paginationController.getCurLimit()); } builder = new TsBlockBuilder(initialExpectedEntries, Collections.singletonList(dataType)); - TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); - ColumnBuilder valueBuilder = builder.getColumnBuilder(0); long allFilteredRows = 0; boolean allSatisfy = recordFilter == null || recordFilter.allSatisfy(this); - switch (dataType) { - case BOOLEAN: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyBoolean(timestamp, aBoolean)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeBoolean(aBoolean); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case INT32: - case DATE: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - int anInt = valueDecoder.readInt(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyInteger(timestamp, anInt)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeInt(anInt); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case INT64: - case TIMESTAMP: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - long aLong = valueDecoder.readLong(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyLong(timestamp, aLong)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeLong(aLong); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case FLOAT: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - float aFloat = valueDecoder.readFloat(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyFloat(timestamp, aFloat)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeFloat(aFloat); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case DOUBLE: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - double aDouble = valueDecoder.readDouble(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyDouble(timestamp, aDouble)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeDouble(aDouble); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - while (timeDecoder.hasNext(timeBuffer)) { - long timestamp = timeDecoder.readLong(timeBuffer); - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (isDeleted(timestamp)) { - continue; - } - if (!allSatisfy && !recordFilter.satisfyBinary(timestamp, aBinary)) { - allFilteredRows++; - continue; - } - if (paginationController.hasCurOffset()) { - paginationController.consumeOffset(); - continue; - } - if (paginationController.hasCurLimit()) { - timeBuilder.writeLong(timestamp); - valueBuilder.writeBinary(aBinary); - builder.declarePosition(); - paginationController.consumeLimit(); - } else { - break; - } - } + PageDataBlockValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_TSBLOCK_SERVICE.call(Type.fromTsDataType(dataType)); + LongPredicate isDeleted = this::isDeleted; + while (timeDecoder.hasNext(timeBuffer)) { + long timestamp = timeDecoder.readLong(timeBuffer); + PageDataReadStatus status = + valueReader.read( + valueDecoder, + valueBuffer, + recordFilter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController); + if (status == PageDataReadStatus.FILTERED) { + allFilteredRows++; + } else if (status == PageDataReadStatus.STOP) { break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); + } } if (filterRowsRecorder != null && allFilteredRows > 0) { filterRowsRecorder.accept(allFilteredRows); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index 50835f5f8..cd0799f3f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -27,18 +27,21 @@ import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.BatchDataFactory; import org.apache.tsfile.read.common.TimeRange; -import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; -import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.PageDataColumnBuilderValueReader; +import org.apache.tsfile.utils.TypeServices.PageDataTsPrimitiveValueReader; +import org.apache.tsfile.utils.TypeServices.PageDataValueReader; import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.function.LongPredicate; public class ValuePageReader { @@ -115,120 +118,29 @@ public BatchData nextBatch(long[] timeBatch, boolean ascending, Filter filter) throws IOException { uncompressDataIfNecessary(); BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false); + PageDataValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_BATCHDATA_SERVICE.call(Type.fromTsDataType(dataType)); + boolean allSatisfy = filter == null; + LongPredicate isDeleted = this::isDeleted; for (int i = 0; i < timeBatch.length; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { continue; } long timestamp = timeBatch[i]; - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyBoolean(timestamp, aBoolean))) { - pageData.putBoolean(timestamp, aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyInteger(timestamp, anInt))) { - pageData.putInt(timestamp, anInt); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timestamp) && (filter == null || filter.satisfyLong(timestamp, aLong))) { - pageData.putLong(timestamp, aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timestamp) && (filter == null || filter.satisfyFloat(timestamp, aFloat))) { - pageData.putFloat(timestamp, aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyDouble(timestamp, aDouble))) { - pageData.putDouble(timestamp, aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timestamp) - && (filter == null || filter.satisfyBinary(timestamp, aBinary))) { - pageData.putBinary(timestamp, aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read( + valueDecoder, valueBuffer, filter, pageData, timestamp, allSatisfy, isDeleted); } return pageData.flip(); } public TsPrimitiveType nextValue(long timestamp, int timeIndex) throws IOException { uncompressDataIfNecessary(); - TsPrimitiveType resultValue = null; if (valueBuffer == null || ((bitmap[timeIndex / 8] & 0xFF) & (MASK >>> (timeIndex % 8))) == 0) { return null; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsBoolean(aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = - dataType.equals(TSDataType.INT32) - ? new TsPrimitiveType.TsInt(anInt) - : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsLong(aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsFloat(aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } - - return resultValue; + PageDataTsPrimitiveValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_TSPRIMITIVETYPE_SERVICE.call(Type.fromTsDataType(dataType)); + return valueReader.read(valueDecoder, valueBuffer, timestamp, this::isDeleted); } /** @@ -241,58 +153,14 @@ public TsPrimitiveType[] nextValueBatch(long[] timeBatch) throws IOException { if (valueBuffer == null) { return valueBatch; } + PageDataTsPrimitiveValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_TSPRIMITIVETYPE_SERVICE.call(Type.fromTsDataType(dataType)); + LongPredicate isDeleted = this::isDeleted; for (int i = 0; i < size; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { continue; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsBoolean(aBoolean); - } - break; - case INT32: - case DATE: - int anInt = valueDecoder.readInt(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = - dataType.equals(TSDataType.INT32) - ? new TsPrimitiveType.TsInt(anInt) - : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsLong(aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsFloat(aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueBatch[i] = valueReader.read(valueDecoder, valueBuffer, timeBatch[i], isDeleted); } return valueBatch; } @@ -309,6 +177,8 @@ public void writeColumnBuilderWithNextBatch( } return; } + PageDataColumnBuilderValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE.call(Type.fromTsDataType(dataType)); for (int i = 0; i < readEndIndex; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { if (keepCurrentRow[i]) { @@ -316,88 +186,7 @@ public void writeColumnBuilderWithNextBatch( } continue; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeBoolean(aBoolean); - } - } - break; - case INT32: - int anInt = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeInt(anInt); - } - } - break; - case DATE: - int anDate = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - if (columnBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) columnBuilder).writeDate(anDate); - } else { - columnBuilder.writeInt(anDate); - } - } - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeLong(aLong); - } - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeFloat(aFloat); - } - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeDouble(aDouble); - } - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (keepCurrentRow[i]) { - if (isDeleted[i]) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeBinary(aBinary); - } - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read(valueDecoder, valueBuffer, columnBuilder, keepCurrentRow[i], isDeleted[i]); } } @@ -412,6 +201,8 @@ public void writeColumnBuilderWithNextBatch( } return; } + PageDataColumnBuilderValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE.call(Type.fromTsDataType(dataType)); for (int i = 0; i < readEndIndex; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { if (keepCurrentRow[i]) { @@ -419,60 +210,7 @@ public void writeColumnBuilderWithNextBatch( } continue; } - switch (dataType) { - case BOOLEAN: - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeBoolean(aBoolean); - } - break; - case INT32: - int anInt = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeInt(anInt); - } - break; - case DATE: - int anDate = valueDecoder.readInt(valueBuffer); - if (keepCurrentRow[i]) { - if (columnBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) columnBuilder).writeDate(anDate); - } else { - columnBuilder.writeInt(anDate); - } - } - break; - case INT64: - case TIMESTAMP: - long aLong = valueDecoder.readLong(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeLong(aLong); - } - break; - case FLOAT: - float aFloat = valueDecoder.readFloat(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeFloat(aFloat); - } - break; - case DOUBLE: - double aDouble = valueDecoder.readDouble(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - Binary aBinary = valueDecoder.readBinary(valueBuffer); - if (keepCurrentRow[i]) { - columnBuilder.writeBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); - } + valueReader.read(valueDecoder, valueBuffer, columnBuilder, keepCurrentRow[i], false); } } @@ -483,131 +221,20 @@ public void writeColumnBuilderWithNextBatch( columnBuilder.appendNull(readEndIndex - readStartIndex); return; } - - switch (dataType) { - case BOOLEAN: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readBoolean(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - boolean aBoolean = valueDecoder.readBoolean(valueBuffer); - columnBuilder.writeBoolean(aBoolean); - } - break; - case INT32: - case DATE: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readInt(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - int aInt = valueDecoder.readInt(valueBuffer); - if (dataType == TSDataType.INT32) { - columnBuilder.writeInt(aInt); - } else { - if (columnBuilder instanceof BinaryColumnBuilder) { - ((BinaryColumnBuilder) columnBuilder).writeDate(aInt); - } else { - columnBuilder.writeInt(aInt); - } - } - } - break; - case INT64: - case TIMESTAMP: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readLong(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - long aLong = valueDecoder.readLong(valueBuffer); - columnBuilder.writeLong(aLong); - } - break; - case FLOAT: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readFloat(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - float aFloat = valueDecoder.readFloat(valueBuffer); - columnBuilder.writeFloat(aFloat); - } - break; - case DOUBLE: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readDouble(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - double aDouble = valueDecoder.readDouble(valueBuffer); - columnBuilder.writeDouble(aDouble); - } - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - // skip useless data - for (int i = 0; i < readStartIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - continue; - } - valueDecoder.readBinary(valueBuffer); - } - - for (int i = readStartIndex; i < readEndIndex; i++) { - if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { - columnBuilder.appendNull(); - continue; - } - Binary aBinary = valueDecoder.readBinary(valueBuffer); - columnBuilder.writeBinary(aBinary); - } - break; - default: - throw new UnSupportedDataTypeException(String.valueOf(dataType)); + PageDataColumnBuilderValueReader valueReader = + TypeServices.READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE.call(Type.fromTsDataType(dataType)); + // skip useless data + for (int i = 0; i < readStartIndex; i++) { + if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) != 0) { + valueReader.read(valueDecoder, valueBuffer, columnBuilder, false, false); + } + } + for (int i = readStartIndex; i < readEndIndex; i++) { + if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { + columnBuilder.appendNull(); + continue; + } + valueReader.read(valueDecoder, valueBuffer, columnBuilder, true, false); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/EncodingTypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/EncodingTypeServices.java new file mode 100644 index 000000000..145f5abd0 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/EncodingTypeServices.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.apache.tsfile.encoding.encoder.DeltaBinaryEncoder; +import org.apache.tsfile.encoding.encoder.DoublePrecisionChimpEncoder; +import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV1; +import org.apache.tsfile.encoding.encoder.DoublePrecisionEncoderV2; +import org.apache.tsfile.encoding.encoder.DoubleRLBE; +import org.apache.tsfile.encoding.encoder.DoubleSprintzEncoder; +import org.apache.tsfile.encoding.encoder.Encoder; +import org.apache.tsfile.encoding.encoder.FloatEncoder; +import org.apache.tsfile.encoding.encoder.FloatRLBE; +import org.apache.tsfile.encoding.encoder.FloatSprintzEncoder; +import org.apache.tsfile.encoding.encoder.IntChimpEncoder; +import org.apache.tsfile.encoding.encoder.IntGorillaEncoder; +import org.apache.tsfile.encoding.encoder.IntRLBE; +import org.apache.tsfile.encoding.encoder.IntRleEncoder; +import org.apache.tsfile.encoding.encoder.IntSprintzEncoder; +import org.apache.tsfile.encoding.encoder.IntZigzagEncoder; +import org.apache.tsfile.encoding.encoder.LongChimpEncoder; +import org.apache.tsfile.encoding.encoder.LongGorillaEncoder; +import org.apache.tsfile.encoding.encoder.LongRLBE; +import org.apache.tsfile.encoding.encoder.LongRleEncoder; +import org.apache.tsfile.encoding.encoder.LongSprintzEncoder; +import org.apache.tsfile.encoding.encoder.LongZigzagEncoder; +import org.apache.tsfile.encoding.encoder.RegularDataEncoder; +import org.apache.tsfile.encoding.encoder.SinglePrecisionChimpEncoder; +import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV1; +import org.apache.tsfile.encoding.encoder.SinglePrecisionEncoderV2; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +public final class EncodingTypeServices { + + private static final String ERROR_MSG_KEY = "error.encoding.ts_encoding_builder_unsupported_type"; + + public static final TypeService RLE_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE, BOOLEAN -> maxPointNumber -> new IntRleEncoder(); + case INT64, TIMESTAMP -> maxPointNumber -> new LongRleEncoder(); + case FLOAT -> + maxPointNumber -> + new FloatEncoder(TSEncoding.RLE, TSDataType.FLOAT, maxPointNumber); + case DOUBLE -> + maxPointNumber -> + new FloatEncoder(TSEncoding.RLE, TSDataType.DOUBLE, maxPointNumber); + case TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + maxPointNumber -> { + throw unsupported(TSEncoding.RLE, type.getTypeEnum()); + }; + }; + + public static final TypeService TS_2DIFF_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> maxPointNumber -> new DeltaBinaryEncoder.IntDeltaEncoder(); + case INT64, TIMESTAMP -> maxPointNumber -> new DeltaBinaryEncoder.LongDeltaEncoder(); + case FLOAT -> + maxPointNumber -> + new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT, maxPointNumber); + case DOUBLE -> + maxPointNumber -> + new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.DOUBLE, maxPointNumber); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + maxPointNumber -> { + throw unsupported(TSEncoding.TS_2DIFF, type.getTypeEnum()); + }; + }; + + public static final TypeService GORILLA_V1_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionEncoderV1(); + case DOUBLE -> new DoublePrecisionEncoderV1(); + case BOOLEAN, + INT32, + DATE, + INT64, + TIMESTAMP, + TEXT, + BLOB, + STRING, + OBJECT, + ROW, + UNKNOWN, + VECTOR -> + throw unsupported(TSEncoding.GORILLA_V1, type.getTypeEnum()); + }; + + public static final TypeService REGULAR_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new RegularDataEncoder.IntRegularEncoder(); + case INT64, TIMESTAMP -> new RegularDataEncoder.LongRegularEncoder(); + case BOOLEAN, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.REGULAR, type.getTypeEnum()); + }; + + public static final TypeService GORILLA_V2_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionEncoderV2(); + case DOUBLE -> new DoublePrecisionEncoderV2(); + case INT32, DATE -> new IntGorillaEncoder(); + case INT64, TIMESTAMP -> new LongGorillaEncoder(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.GORILLA, type.getTypeEnum()); + }; + + public static final TypeService SPRINTZ_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new IntSprintzEncoder(); + case INT64, TIMESTAMP -> new LongSprintzEncoder(); + case FLOAT -> new FloatSprintzEncoder(); + case DOUBLE -> new DoubleSprintzEncoder(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.SPRINTZ, type.getTypeEnum()); + }; + + public static final TypeService RLBE_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new IntRLBE(); + case INT64, TIMESTAMP -> new LongRLBE(); + case FLOAT -> new FloatRLBE(); + case DOUBLE -> new DoubleRLBE(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.RLBE, type.getTypeEnum()); + }; + + public static final TypeService ZIGZAG_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> new IntZigzagEncoder(); + case INT64, TIMESTAMP -> new LongZigzagEncoder(); + case BOOLEAN, FLOAT, DOUBLE, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.ZIGZAG, type.getTypeEnum()); + }; + + public static final TypeService CHIMP_ENCODER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case FLOAT -> new SinglePrecisionChimpEncoder(); + case DOUBLE -> new DoublePrecisionChimpEncoder(); + case INT32, DATE -> new IntChimpEncoder(); + case INT64, TIMESTAMP -> new LongChimpEncoder(); + case BOOLEAN, TEXT, BLOB, STRING, OBJECT, ROW, UNKNOWN, VECTOR -> + throw unsupported(TSEncoding.CHIMP, type.getTypeEnum()); + }; + + static { + RLE_ENCODER_SERVICE.check(); + TS_2DIFF_ENCODER_SERVICE.check(); + GORILLA_V1_ENCODER_SERVICE.check(); + REGULAR_ENCODER_SERVICE.check(); + GORILLA_V2_ENCODER_SERVICE.check(); + SPRINTZ_ENCODER_SERVICE.check(); + RLBE_ENCODER_SERVICE.check(); + ZIGZAG_ENCODER_SERVICE.check(); + CHIMP_ENCODER_SERVICE.check(); + } + + private EncodingTypeServices() {} + + private static UnSupportedDataTypeException unsupported(TSEncoding encoding, Object type) { + return new UnSupportedDataTypeException(Messages.format(ERROR_MSG_KEY, encoding, type)) + .setChecked(true); + } + + @FunctionalInterface + public interface EncoderBuilder { + + Encoder build(int maxPointNumber); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java index 6b941e73f..6d1dd9e15 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java @@ -27,15 +27,11 @@ import org.apache.tsfile.fileSystem.fsFactory.FSFactory; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; import org.apache.tsfile.write.record.datapoint.DataPoint; -import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; -import org.apache.tsfile.write.record.datapoint.FloatDataPoint; -import org.apache.tsfile.write.record.datapoint.IntDataPoint; -import org.apache.tsfile.write.record.datapoint.LongDataPoint; import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -68,35 +64,9 @@ public static void writeWithTsRecord( // construct TsRecord TSRecord tsRecord = new TSRecord(deviceId, time); for (IMeasurementSchema schema : schemas) { - DataPoint dPoint; - switch (schema.getType()) { - case INT64: - case TIMESTAMP: - dPoint = new LongDataPoint(schema.getMeasurementName(), startValue); - break; - case INT32: - case DATE: - dPoint = new IntDataPoint(schema.getMeasurementName(), (int) startValue); - break; - case DOUBLE: - dPoint = new DoubleDataPoint(schema.getMeasurementName(), (double) startValue); - break; - case FLOAT: - dPoint = new FloatDataPoint(schema.getMeasurementName(), (float) startValue); - break; - case BOOLEAN: - dPoint = new BooleanDataPoint(schema.getMeasurementName(), true); - break; - case TEXT: - case BLOB: - case STRING: - default: - dPoint = - new StringDataPoint( - schema.getMeasurementName(), - new Binary(String.valueOf(startValue), TSFileConfig.STRING_CHARSET)); - break; - } + DataPoint dPoint = + Type.fromTsDataType(schema.getType()) + .getDataPoint(schema.getMeasurementName(), startValue); tsRecord.addTuple(dPoint); } // write diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java new file mode 100644 index 000000000..34c5c16d6 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TypeServices.java @@ -0,0 +1,510 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.reader.series.PaginationController; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ValueChunkWriter; + +import java.nio.ByteBuffer; +import java.util.function.LongPredicate; + +public final class TypeServices { + + // Page value decoding services for BatchData and TsBlock outputs. + + public static final TypeService READ_PAGE_VALUE_TO_BATCHDATA_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + boolean value = decoder.readBoolean(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyBoolean(timestamp, value))) { + pageData.putBoolean(timestamp, value); + } + }; + case INT32, DATE -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + int value = decoder.readInt(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyInteger(timestamp, value))) { + pageData.putInt(timestamp, value); + } + }; + case INT64, TIMESTAMP -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + long value = decoder.readLong(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyLong(timestamp, value))) { + pageData.putLong(timestamp, value); + } + }; + case FLOAT -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + float value = decoder.readFloat(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyFloat(timestamp, value))) { + pageData.putFloat(timestamp, value); + } + }; + case DOUBLE -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + double value = decoder.readDouble(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyDouble(timestamp, value))) { + pageData.putDouble(timestamp, value); + } + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + Binary value = decoder.readBinary(buffer); + if (!isDeleted.test(timestamp) + && (allSatisfy || filter.satisfyBinary(timestamp, value))) { + pageData.putBinary(timestamp, value); + } + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, buffer, filter, pageData, timestamp, allSatisfy, isDeleted) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); + }; + }; + + public static final TypeService READ_PAGE_VALUE_TO_TSBLOCK_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + boolean value = decoder.readBoolean(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyBoolean(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeBoolean(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case INT32, DATE -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + int value = decoder.readInt(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyInteger(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeInt(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case INT64, TIMESTAMP -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + long value = decoder.readLong(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyLong(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeLong(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case FLOAT -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + float value = decoder.readFloat(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyFloat(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeFloat(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case DOUBLE -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + double value = decoder.readDouble(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyDouble(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeDouble(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + Binary value = decoder.readBinary(buffer); + if (isDeleted.test(timestamp)) { + return PageDataReadStatus.CONTINUE; + } + if (!allSatisfy && !filter.satisfyBinary(timestamp, value)) { + return PageDataReadStatus.FILTERED; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + return PageDataReadStatus.CONTINUE; + } + if (!paginationController.hasCurLimit()) { + return PageDataReadStatus.STOP; + } + builder.getTimeColumnBuilder().writeLong(timestamp); + builder.getColumnBuilder(0).writeBinary(value); + builder.declarePosition(); + paginationController.consumeLimit(); + return PageDataReadStatus.CONTINUE; + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, + buffer, + filter, + builder, + timestamp, + allSatisfy, + isDeleted, + paginationController) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); + }; + }; + + // Value page decoding services for TsPrimitiveType and ColumnBuilder outputs. + + public static final TypeService + READ_PAGE_VALUE_TO_TSPRIMITIVETYPE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, buffer, timestamp, isDeleted) -> { + boolean value = decoder.readBoolean(buffer); + return isDeleted.test(timestamp) + ? null + : new TsPrimitiveType.TsBoolean(value); + }; + case INT32 -> + (decoder, buffer, timestamp, isDeleted) -> { + int value = decoder.readInt(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsInt(value); + }; + case DATE -> + (decoder, buffer, timestamp, isDeleted) -> { + int value = decoder.readInt(buffer); + return isDeleted.test(timestamp) + ? null + : new TsPrimitiveType.TsInt(value, TSDataType.DATE); + }; + case INT64, TIMESTAMP -> + (decoder, buffer, timestamp, isDeleted) -> { + long value = decoder.readLong(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsLong(value); + }; + case FLOAT -> + (decoder, buffer, timestamp, isDeleted) -> { + float value = decoder.readFloat(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsFloat(value); + }; + case DOUBLE -> + (decoder, buffer, timestamp, isDeleted) -> { + double value = decoder.readDouble(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsDouble(value); + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, buffer, timestamp, isDeleted) -> { + Binary value = decoder.readBinary(buffer); + return isDeleted.test(timestamp) ? null : new TsPrimitiveType.TsBinary(value); + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, buffer, timestamp, isDeleted) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); + }; + }; + + public static final TypeService + READ_PAGE_VALUE_TO_COLUMNBUILDER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + boolean value = decoder.readBoolean(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeBoolean(value); + } + } + }; + case INT32 -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + int value = decoder.readInt(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeInt(value); + } + } + }; + case DATE -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + int value = decoder.readInt(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else if (builder instanceof BinaryColumnBuilder) { + ((BinaryColumnBuilder) builder).writeDate(value); + } else { + builder.writeInt(value); + } + } + }; + case INT64, TIMESTAMP -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + long value = decoder.readLong(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeLong(value); + } + } + }; + case FLOAT -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + float value = decoder.readFloat(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeFloat(value); + } + } + }; + case DOUBLE -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + double value = decoder.readDouble(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeDouble(value); + } + } + }; + case TEXT, BLOB, STRING, OBJECT -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + Binary value = decoder.readBinary(buffer); + if (keepCurrentRow) { + if (isDeleted) { + builder.appendNull(); + } else { + builder.writeBinary(value); + } + } + }; + case ROW, UNKNOWN, VECTOR -> + (decoder, buffer, builder, keepCurrentRow, isDeleted) -> { + throw new UnSupportedDataTypeException(String.valueOf(type.getTypeEnum())) + .setChecked(true); + }; + }; + + // Chunk writing services. + + public static final TypeService WRITE_EMPTY_VALUE_TO_CHUNK_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> writer -> writer.write(-1, false, true); + case INT32, DATE -> writer -> writer.write(-1, 0, true); + case INT64, TIMESTAMP -> writer -> writer.write(-1, 0L, true); + case FLOAT -> writer -> writer.write(-1, 0.0F, true); + case DOUBLE -> writer -> writer.write(-1, 0.0D, true); + case TEXT, BLOB, STRING, OBJECT -> writer -> writer.write(-1, null, true); + case ROW, UNKNOWN, VECTOR -> + writer -> { + throw new UnSupportedDataTypeException( + Messages.format("error.write.type_not_supported", type.getTypeEnum())) + .setChecked(true); + }; + }; + + private TypeServices() {} + + @FunctionalInterface + public interface PageDataValueReader { + + void read( + Decoder decoder, + ByteBuffer buffer, + Filter filter, + BatchData pageData, + long timestamp, + boolean allSatisfy, + LongPredicate isDeleted); + } + + @FunctionalInterface + public interface PageDataBlockValueReader { + + PageDataReadStatus read( + Decoder decoder, + ByteBuffer buffer, + Filter filter, + TsBlockBuilder builder, + long timestamp, + boolean allSatisfy, + LongPredicate isDeleted, + PaginationController paginationController); + } + + @FunctionalInterface + public interface PageDataTsPrimitiveValueReader { + + TsPrimitiveType read( + Decoder decoder, ByteBuffer buffer, long timestamp, LongPredicate isDeleted); + } + + @FunctionalInterface + public interface PageDataColumnBuilderValueReader { + + void read( + Decoder decoder, + ByteBuffer buffer, + ColumnBuilder builder, + boolean keepCurrentRow, + boolean isDeleted); + } + + @FunctionalInterface + public interface EmptyValueChunkWriter { + + void write(ValueChunkWriter writer); + } + + public enum PageDataReadStatus { + CONTINUE, + FILTERED, + STOP + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java new file mode 100644 index 000000000..a868ae8a5 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ValueFilterTypeServices.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.apache.tsfile.common.regexp.LikePattern; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.filter.operator.BinaryFilterOperators; +import org.apache.tsfile.read.filter.operator.BooleanFilterOperators; +import org.apache.tsfile.read.filter.operator.DoubleFilterOperators; +import org.apache.tsfile.read.filter.operator.FloatFilterOperators; +import org.apache.tsfile.read.filter.operator.IntegerFilterOperators; +import org.apache.tsfile.read.filter.operator.LongFilterOperators; +import org.apache.tsfile.read.filter.operator.StringFilterOperators; + +import java.util.Set; +import java.util.regex.Pattern; + +@SuppressWarnings("unchecked") +public final class ValueFilterTypeServices { + + public static final TypeService VALUE_GT_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueGt(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueGt(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueGt(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueGt(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueGt(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueGt(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueGt(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_GT_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueGtEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueGtEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueGtEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueGtEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueGtEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueGtEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueGtEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_LT_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueLt(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueLt(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueLt(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueLt(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueLt(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueLt(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueLt(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_LT_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueLtEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueLtEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueLtEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueLtEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueLtEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueLtEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueLtEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_EQ_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value) -> new BooleanFilterOperators.ValueNotEq(index, (boolean) value); + case INT32, DATE -> + (index, value) -> + new IntegerFilterOperators.ValueNotEq(index, ((Number) value).intValue()); + case INT64, TIMESTAMP -> + (index, value) -> + new LongFilterOperators.ValueNotEq(index, ((Number) value).longValue()); + case FLOAT -> + (index, value) -> + new FloatFilterOperators.ValueNotEq(index, ((Number) value).floatValue()); + case DOUBLE -> + (index, value) -> + new DoubleFilterOperators.ValueNotEq(index, ((Number) value).doubleValue()); + case TEXT, BLOB -> + (index, value) -> new BinaryFilterOperators.ValueNotEq(index, (Binary) value); + case STRING -> + (index, value) -> new StringFilterOperators.ValueNotEq(index, (Binary) value); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value) -> unsupportedType(type); + }; + + public static final TypeService VALUE_BETWEEN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value1, value2) -> + new BooleanFilterOperators.ValueBetweenAnd( + index, (boolean) value1, (boolean) value2); + case INT32, DATE -> + (index, value1, value2) -> + new IntegerFilterOperators.ValueBetweenAnd( + index, ((Number) value1).intValue(), ((Number) value2).intValue()); + case INT64, TIMESTAMP -> + (index, value1, value2) -> + new LongFilterOperators.ValueBetweenAnd( + index, ((Number) value1).longValue(), ((Number) value2).longValue()); + case FLOAT -> + (index, value1, value2) -> + new FloatFilterOperators.ValueBetweenAnd( + index, ((Number) value1).floatValue(), ((Number) value2).floatValue()); + case DOUBLE -> + (index, value1, value2) -> + new DoubleFilterOperators.ValueBetweenAnd( + index, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); + case TEXT, BLOB -> + (index, value1, value2) -> + new BinaryFilterOperators.ValueBetweenAnd( + index, (Binary) value1, (Binary) value2); + case STRING -> + (index, value1, value2) -> + new StringFilterOperators.ValueBetweenAnd( + index, (Binary) value1, (Binary) value2); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value1, value2) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_BETWEEN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, value1, value2) -> + new BooleanFilterOperators.ValueNotBetweenAnd( + index, (boolean) value1, (boolean) value2); + case INT32, DATE -> + (index, value1, value2) -> + new IntegerFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).intValue(), ((Number) value2).intValue()); + case INT64, TIMESTAMP -> + (index, value1, value2) -> + new LongFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).longValue(), ((Number) value2).longValue()); + case FLOAT -> + (index, value1, value2) -> + new FloatFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).floatValue(), ((Number) value2).floatValue()); + case DOUBLE -> + (index, value1, value2) -> + new DoubleFilterOperators.ValueNotBetweenAnd( + index, ((Number) value1).doubleValue(), ((Number) value2).doubleValue()); + case TEXT, BLOB -> + (index, value1, value2) -> + new BinaryFilterOperators.ValueNotBetweenAnd( + index, (Binary) value1, (Binary) value2); + case STRING -> + (index, value1, value2) -> + new StringFilterOperators.ValueNotBetweenAnd( + index, (Binary) value1, (Binary) value2); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, value1, value2) -> unsupportedType(type); + }; + + public static final TypeService VALUE_LIKE_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueLike::new; + case INT32, DATE -> IntegerFilterOperators.ValueLike::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueLike::new; + case FLOAT -> FloatFilterOperators.ValueLike::new; + case DOUBLE -> DoubleFilterOperators.ValueLike::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueLike::new; + case STRING -> StringFilterOperators.ValueLike::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_LIKE_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueNotLike::new; + case INT32, DATE -> IntegerFilterOperators.ValueNotLike::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueNotLike::new; + case FLOAT -> FloatFilterOperators.ValueNotLike::new; + case DOUBLE -> DoubleFilterOperators.ValueNotLike::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueNotLike::new; + case STRING -> StringFilterOperators.ValueNotLike::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_REGEXP_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueRegexp::new; + case INT32, DATE -> IntegerFilterOperators.ValueRegexp::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueRegexp::new; + case FLOAT -> FloatFilterOperators.ValueRegexp::new; + case DOUBLE -> DoubleFilterOperators.ValueRegexp::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueRegexp::new; + case STRING -> StringFilterOperators.ValueRegexp::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_REGEXP_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> BooleanFilterOperators.ValueNotRegexp::new; + case INT32, DATE -> IntegerFilterOperators.ValueNotRegexp::new; + case INT64, TIMESTAMP -> LongFilterOperators.ValueNotRegexp::new; + case FLOAT -> FloatFilterOperators.ValueNotRegexp::new; + case DOUBLE -> DoubleFilterOperators.ValueNotRegexp::new; + case TEXT, BLOB -> BinaryFilterOperators.ValueNotRegexp::new; + case STRING -> StringFilterOperators.ValueNotRegexp::new; + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, pattern) -> unsupportedType(type); + }; + + public static final TypeService VALUE_IN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, values) -> new BooleanFilterOperators.ValueIn(index, (Set) values); + case INT32, DATE -> + (index, values) -> new IntegerFilterOperators.ValueIn(index, (Set) values); + case INT64, TIMESTAMP -> + (index, values) -> new LongFilterOperators.ValueIn(index, (Set) values); + case FLOAT -> + (index, values) -> new FloatFilterOperators.ValueIn(index, (Set) values); + case DOUBLE -> + (index, values) -> new DoubleFilterOperators.ValueIn(index, (Set) values); + case TEXT, BLOB -> + (index, values) -> new BinaryFilterOperators.ValueIn(index, (Set) values); + case STRING -> + (index, values) -> new StringFilterOperators.ValueIn(index, (Set) values); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, values) -> unsupportedType(type); + }; + + public static final TypeService VALUE_NOT_IN_FILTER_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (index, values) -> + new BooleanFilterOperators.ValueNotIn(index, (Set) values); + case INT32, DATE -> + (index, values) -> + new IntegerFilterOperators.ValueNotIn(index, (Set) values); + case INT64, TIMESTAMP -> + (index, values) -> new LongFilterOperators.ValueNotIn(index, (Set) values); + case FLOAT -> + (index, values) -> new FloatFilterOperators.ValueNotIn(index, (Set) values); + case DOUBLE -> + (index, values) -> + new DoubleFilterOperators.ValueNotIn(index, (Set) values); + case TEXT, BLOB -> + (index, values) -> + new BinaryFilterOperators.ValueNotIn(index, (Set) values); + case STRING -> + (index, values) -> + new StringFilterOperators.ValueNotIn(index, (Set) values); + case OBJECT, ROW, UNKNOWN, VECTOR -> (index, values) -> unsupportedType(type); + }; + + static { + VALUE_GT_FILTER_SERVICE.check(); + VALUE_GT_EQ_FILTER_SERVICE.check(); + VALUE_LT_FILTER_SERVICE.check(); + VALUE_LT_EQ_FILTER_SERVICE.check(); + VALUE_EQ_FILTER_SERVICE.check(); + VALUE_NOT_EQ_FILTER_SERVICE.check(); + VALUE_BETWEEN_FILTER_SERVICE.check(); + VALUE_NOT_BETWEEN_FILTER_SERVICE.check(); + VALUE_LIKE_FILTER_SERVICE.check(); + VALUE_NOT_LIKE_FILTER_SERVICE.check(); + VALUE_REGEXP_FILTER_SERVICE.check(); + VALUE_NOT_REGEXP_FILTER_SERVICE.check(); + VALUE_IN_FILTER_SERVICE.check(); + VALUE_NOT_IN_FILTER_SERVICE.check(); + } + + private ValueFilterTypeServices() {} + + private static Filter unsupportedType(Type type) { + throw new UnsupportedOperationException( + Messages.format("error.read.filter_api_unsupported_type", type.getTypeEnum())); + } + + @FunctionalInterface + public interface SingleValueFilterFactory { + Filter create(int measurementIndex, Object value); + } + + @FunctionalInterface + public interface BetweenValueFilterFactory { + Filter create(int measurementIndex, Object value1, Object value2); + } + + @FunctionalInterface + public interface LikeFilterFactory { + Filter create(int measurementIndex, LikePattern pattern); + } + + @FunctionalInterface + public interface RegexpFilterFactory { + Filter create(int measurementIndex, Pattern pattern); + } + + @FunctionalInterface + public interface SetFilterFactory { + Filter create(int measurementIndex, Set values); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 07e82767d..8896b58a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -31,9 +31,9 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.i18n.Messages; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.TypeServices; +import org.apache.tsfile.utils.TypeServices.EmptyValueChunkWriter; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.time.LocalDate; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -171,34 +170,7 @@ public int write(long time, List data) throws WriteProcessException, if (valueChunkWriter == null) { valueChunkWriter = tryToAddSeriesWriterInternal(point.getMeasurementSchema()); } - switch (point.getType()) { - case BOOLEAN: - valueChunkWriter.write(time, (boolean) point.getValue(), isNull); - break; - case INT32: - case DATE: - valueChunkWriter.write(time, isNull ? 0 : (int) point.getValue(), isNull); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(time, (long) point.getValue(), isNull); - break; - case FLOAT: - valueChunkWriter.write(time, (float) point.getValue(), isNull); - break; - case DOUBLE: - valueChunkWriter.write(time, (double) point.getValue(), isNull); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueChunkWriter.write(time, (Binary) point.getValue(), isNull); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", point.getType())); - } + Type.fromTsDataType(point.getType()).write(valueChunkWriter, time, point.getValue(), isNull); } if (!emptyValueChunkWriters.isEmpty()) { writeEmptyDataInOneRow(emptyValueChunkWriters); @@ -255,45 +227,8 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) // check isNull by bitMap in tablet ValueChunkWriter valueChunkWriter = tryToAddSeriesWriterInternal(measurementSchemas.get(columnIndex)); - switch (measurementSchemas.get(columnIndex).getType()) { - case BOOLEAN: - valueChunkWriter.write( - time, ((boolean[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case INT32: - valueChunkWriter.write(time, ((int[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case DATE: - valueChunkWriter.write( - time, - isNull - ? 0 - : DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.getValues()[columnIndex])[row]), - isNull); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(time, ((long[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case FLOAT: - valueChunkWriter.write(time, ((float[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case DOUBLE: - valueChunkWriter.write(time, ((double[]) tablet.getValues()[columnIndex])[row], isNull); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueChunkWriter.write(time, ((Binary[]) tablet.getValues()[columnIndex])[row], isNull); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format( - "error.write.type_not_supported", - measurementSchemas.get(columnIndex).getType())); - } + Type.fromTsDataType(measurementSchemas.get(columnIndex).getType()) + .write(valueChunkWriter, time, tablet.getValues()[columnIndex], row, isNull); } // TODO: we can write the null columns after whole insertion, according to the point number // in the time chunk before and after, no need to do it in a row-by-row manner @@ -357,35 +292,10 @@ public void tryToAddEmptyPageAndData(ValueChunkWriter valueChunkWriter) throws I private void writeEmptyDataInOneRow(List valueChunkWriterList) { for (ValueChunkWriter valueChunkWriter : valueChunkWriterList) { - TSDataType dataType = valueChunkWriter.getDataType(); - switch (dataType) { - case BOOLEAN: - valueChunkWriter.write(-1, false, true); - break; - case INT32: - case DATE: - valueChunkWriter.write(-1, 0, true); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(-1, 0L, true); - break; - case FLOAT: - valueChunkWriter.write(-1, 0.0f, true); - break; - case DOUBLE: - valueChunkWriter.write(-1, 0.0d, true); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - valueChunkWriter.write(-1, null, true); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + EmptyValueChunkWriter emptyValueWriter = + TypeServices.WRITE_EMPTY_VALUE_TO_CHUNK_SERVICE.call( + Type.fromTsDataType(valueChunkWriter.getDataType())); + emptyValueWriter.write(valueChunkWriter); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java index 17c74cf62..ecdf14151 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -29,8 +29,8 @@ import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -39,7 +39,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -48,6 +47,7 @@ public class AlignedChunkWriterImpl implements IChunkWriter { protected TimeChunkWriter timeChunkWriter; protected List valueChunkWriterList; + private Type[] valueTypes; protected int valueIndex; protected EncryptParameter encryptParam; @@ -303,41 +303,24 @@ public void write(long time, Binary value, boolean isNull, int valueIndex) { } public void write(long time, TsPrimitiveType[] points) { + if (valueTypes == null) { + initValueTypes(); + } valueIndex = 0; for (TsPrimitiveType point : points) { ValueChunkWriter writer = valueChunkWriterList.get(valueIndex++); - switch (writer.getDataType()) { - case INT64: - case TIMESTAMP: - writer.write(time, point != null ? point.getLong() : Long.MAX_VALUE, point == null); - break; - case INT32: - case DATE: - writer.write(time, point != null ? point.getInt() : Integer.MAX_VALUE, point == null); - break; - case FLOAT: - writer.write(time, point != null ? point.getFloat() : Float.MAX_VALUE, point == null); - break; - case DOUBLE: - writer.write(time, point != null ? point.getDouble() : Double.MAX_VALUE, point == null); - break; - case BOOLEAN: - writer.write(time, point != null ? point.getBoolean() : false, point == null); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - writer.write( - time, - point != null ? point.getBinary() : new Binary("".getBytes(StandardCharsets.UTF_8)), - point == null); - break; - } + valueTypes[valueIndex - 1].write(writer, time, point); } write(time); } + private void initValueTypes() { + valueTypes = new Type[valueChunkWriterList.size()]; + for (int i = 0; i < valueChunkWriterList.size(); i++) { + valueTypes[i] = Type.fromTsDataType(valueChunkWriterList.get(i).getDataType()); + } + } + public void write(long time) { valueIndex = 0; timeChunkWriter.write(time); @@ -351,6 +334,9 @@ public void writeTime(long time) { } public void write(TimeColumn timeColumn, Column[] valueColumns, int batchSize) { + if (valueTypes == null) { + initValueTypes(); + } if (remainingPointsNumber < batchSize) { int pointsHasWritten = (int) remainingPointsNumber; batchWrite(timeColumn, valueColumns, pointsHasWritten, 0); @@ -367,35 +353,7 @@ private void batchWrite( for (Column column : valueColumns) { ValueChunkWriter chunkWriter = valueChunkWriterList.get(valueIndex++); - TSDataType tsDataType = chunkWriter.getDataType(); - switch (tsDataType) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkWriter.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); - break; - case DOUBLE: - chunkWriter.write(times, column.getDoubles(), column.isNull(), batchSize, arrayOffset); - break; - case BOOLEAN: - chunkWriter.write(times, column.getBooleans(), column.isNull(), batchSize, arrayOffset); - break; - case INT64: - case TIMESTAMP: - chunkWriter.write(times, column.getLongs(), column.isNull(), batchSize, arrayOffset); - break; - case INT32: - case DATE: - chunkWriter.write(times, column.getInts(), column.isNull(), batchSize, arrayOffset); - break; - case FLOAT: - chunkWriter.write(times, column.getFloats(), column.isNull(), batchSize, arrayOffset); - break; - default: - throw new UnsupportedOperationException( - Messages.format("error.write.chunk_unknown_type", tsDataType)); - } + valueTypes[valueIndex - 1].write(chunkWriter, times, column, batchSize, arrayOffset); } write(times, batchSize, arrayOffset); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index b525064fa..ea73884d2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -22,13 +22,10 @@ import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.i18n.Messages; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -38,7 +35,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.time.LocalDate; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -118,7 +114,8 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) continue; } String measurementId = timeseries.get(column).getMeasurementName(); - TSDataType tsDataType = timeseries.get(column).getType(); + Type type = Type.fromTsDataType(timeseries.get(column).getType()); + ChunkWriterImpl chunkWriter = chunkWriters.get(measurementId); pointCount = 0; for (int row = startRowIndex; row < endRowIndex; row++) { // check isNull in tablet @@ -130,49 +127,7 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) long time = tablet.getTimestamps()[row]; checkIsHistoryData(measurementId, time); pointCount++; - switch (tsDataType) { - case INT32: - chunkWriters.get(measurementId).write(time, ((int[]) tablet.getValues()[column])[row]); - break; - case DATE: - chunkWriters - .get(measurementId) - .write( - time, - DateUtils.parseDateExpressionToInt( - ((LocalDate[]) tablet.getValues()[column])[row])); - break; - case INT64: - case TIMESTAMP: - chunkWriters.get(measurementId).write(time, ((long[]) tablet.getValues()[column])[row]); - break; - case FLOAT: - chunkWriters - .get(measurementId) - .write(time, ((float[]) tablet.getValues()[column])[row]); - break; - case DOUBLE: - chunkWriters - .get(measurementId) - .write(time, ((double[]) tablet.getValues()[column])[row]); - break; - case BOOLEAN: - chunkWriters - .get(measurementId) - .write(time, ((boolean[]) tablet.getValues()[column])[row]); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - chunkWriters - .get(measurementId) - .write(time, ((Binary[]) tablet.getValues()[column])[row]); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", tsDataType)); - } + type.write(chunkWriter, time, tablet.getValues()[column], row); lastTimeMap.put(measurementId, time); } maxPointCount = Math.max(pointCount, maxPointCount); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 2bad6c953..843588c59 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -28,15 +28,14 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -67,7 +66,6 @@ public class Tablet implements Accountable { private static final long TABLET_SIZE = RamUsageEstimator.shallowSizeOfInstance(Tablet.class); private static final int DEFAULT_SIZE = 1024; - private static final LocalDate EMPTY_DATE = LocalDate.of(1000, 1, 1); /** DeviceId if using tree-view interfaces or TableName when using table-view interfaces. */ private String insertTargetName; @@ -310,120 +308,7 @@ private void addValueOfDataType( // Mark the null value position updateBitMap(rowIndex, indexOfSchema, value == null); - switch (dataType) { - case TEXT: - case STRING: - case BLOB: - case OBJECT: - { - if (value != null && !(value instanceof Binary) && !(value instanceof String)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Binary or String", - dataType, - value.getClass().getName())); - } - final Binary[] sensor = (Binary[]) values[indexOfSchema]; - if (value instanceof Binary) { - sensor[rowIndex] = (Binary) value; - } else { - sensor[rowIndex] = - value != null - ? new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)) - : Binary.EMPTY_VALUE; - } - break; - } - case FLOAT: - { - if (value != null && !(value instanceof Float)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Float", - dataType, - value.getClass().getName())); - } - final float[] sensor = (float[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (float) value : Float.MIN_VALUE; - break; - } - case INT32: - { - if (value != null && !(value instanceof Integer)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Integer", - dataType, - value.getClass().getName())); - } - final int[] sensor = (int[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; - break; - } - case DATE: - { - if (value != null && !(value instanceof LocalDate)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "LocalDate", - dataType, - value.getClass().getName())); - } - final LocalDate[] sensor = (LocalDate[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (LocalDate) value : EMPTY_DATE; - break; - } - case INT64: - case TIMESTAMP: - { - if (value != null && !(value instanceof Long)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Long", - dataType, - value.getClass().getName())); - } - final long[] sensor = (long[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; - break; - } - case DOUBLE: - { - if (value != null && !(value instanceof Double)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Double", - dataType, - value.getClass().getName())); - } - final double[] sensor = (double[]) values[indexOfSchema]; - sensor[rowIndex] = value != null ? (double) value : Double.MIN_VALUE; - break; - } - case BOOLEAN: - { - if (value != null && !(value instanceof Boolean)) { - throw new IllegalArgumentException( - Messages.format( - "error.write.tablet_expected_type", - "Boolean", - dataType, - value.getClass().getName())); - } - final boolean[] sensor = (boolean[]) values[indexOfSchema]; - sensor[rowIndex] = value != null && (boolean) value; - break; - } - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + Type.fromTsDataType(dataType).addValue(rowIndex, value, values[indexOfSchema]); } @TsFileApi @@ -711,39 +596,7 @@ private Object createValueColumnOfDataType(TSDataType dataType) { } private Object createValueColumnOfDataType(TSDataType dataType, int capacity) { - - Object valueColumn; - switch (dataType) { - case INT32: - valueColumn = new int[capacity]; - break; - case INT64: - case TIMESTAMP: - valueColumn = new long[capacity]; - break; - case FLOAT: - valueColumn = new float[capacity]; - break; - case DOUBLE: - valueColumn = new double[capacity]; - break; - case BOOLEAN: - valueColumn = new boolean[capacity]; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - valueColumn = new Binary[capacity]; - break; - case DATE: - valueColumn = new LocalDate[capacity]; - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } - return valueColumn; + return Type.fromTsDataType(dataType).createArray(capacity); } /** Serialize {@link Tablet} */ @@ -824,55 +677,21 @@ private int serializedSizeOfValues() { if (values != null) { final int columnCount = schemas == null ? 0 : schemas.size(); for (int i = 0; i < columnCount; i++) { - size = Math.addExact(size, serializedSizeOfColumn(schemas.get(i).getType(), values[i])); + size = + Math.addExact( + size, + serializedSizeOfColumn(Type.fromTsDataType(schemas.get(i).getType()), values[i])); } } return size; } - private int serializedSizeOfColumn(final TSDataType dataType, final Object column) { + private int serializedSizeOfColumn(final Type type, final Object column) { int size = Byte.BYTES; if (column == null) { return size; } - switch (dataType) { - case INT32: - return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize)); - case DATE: - return Math.addExact(size, Math.multiplyExact(Integer.BYTES, rowSize)); - case INT64: - case TIMESTAMP: - return Math.addExact(size, Math.multiplyExact(Long.BYTES, rowSize)); - case FLOAT: - return Math.addExact(size, Math.multiplyExact(Float.BYTES, rowSize)); - case DOUBLE: - return Math.addExact(size, Math.multiplyExact(Double.BYTES, rowSize)); - case BOOLEAN: - return Math.addExact(size, rowSize); - case TEXT: - case STRING: - case BLOB: - case OBJECT: - return Math.addExact(size, serializedSizeOfBinaryValues((Binary[]) column)); - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } - } - - private static int serializedSizeOfBinaryValues(final Binary[] binaryValues, final int rowSize) { - int size = 0; - for (int j = 0; j < rowSize; j++) { - size = Math.addExact(size, Byte.BYTES); - if (binaryValues[j] != null) { - size = Math.addExact(size, ReadWriteIOUtils.sizeToWrite(binaryValues[j])); - } - } - return size; - } - - private int serializedSizeOfBinaryValues(final Binary[] binaryValues) { - return serializedSizeOfBinaryValues(binaryValues, rowSize); + return Math.addExact(size, type.serializedSize(column, rowSize)); } /** Serialize {@link MeasurementSchema}s */ @@ -926,75 +745,17 @@ private void writeValues(DataOutputStream stream) throws IOException { if (values != null) { int size = (schemas == null ? 0 : schemas.size()); for (int i = 0; i < size; i++) { - serializeColumn(schemas.get(i).getType(), values[i], stream, columnCategories.get(i)); + serializeColumn(Type.fromTsDataType(schemas.get(i).getType()), values[i], stream); } } } - private void serializeColumn( - TSDataType dataType, Object column, DataOutputStream stream, ColumnCategory columnCategory) + private void serializeColumn(Type type, Object column, DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(column != null), stream); if (column != null) { - switch (dataType) { - case INT32: - int[] intValues = (int[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(intValues[j], stream); - } - break; - case DATE: - LocalDate[] dateValues = (LocalDate[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write( - dateValues[j] == null - ? DateUtils.EMPTY_DATE_INT - : DateUtils.parseDateExpressionToInt(dateValues[j]), - stream); - } - break; - case INT64: - case TIMESTAMP: - long[] longValues = (long[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(longValues[j], stream); - } - break; - case FLOAT: - float[] floatValues = (float[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(floatValues[j], stream); - } - break; - case DOUBLE: - double[] doubleValues = (double[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(doubleValues[j], stream); - } - break; - case BOOLEAN: - boolean[] boolValues = (boolean[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(BytesUtils.boolToByte(boolValues[j]), stream); - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[] binaryValues = (Binary[]) column; - for (int j = 0; j < rowSize; j++) { - ReadWriteIOUtils.write(BytesUtils.boolToByte(binaryValues[j] != null), stream); - if (binaryValues[j] != null) { - ReadWriteIOUtils.write(binaryValues[j], stream); - } - } - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + type.serializeArray(column, rowSize, stream); } } @@ -1081,70 +842,7 @@ public static Object[] readvaluesFromBuffer( boolean isValueColumnsNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); if (isValueColumnsNotNull) { - switch (types[i]) { - case BOOLEAN: - boolean[] boolValues = new boolean[rowSize]; - for (int index = 0; index < rowSize; index++) { - boolValues[index] = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); - } - values[i] = boolValues; - break; - case INT32: - int[] intValues = new int[rowSize]; - for (int index = 0; index < rowSize; index++) { - intValues[index] = ReadWriteIOUtils.readInt(byteBuffer); - } - values[i] = intValues; - break; - case DATE: - LocalDate[] dateValues = new LocalDate[rowSize]; - for (int index = 0; index < rowSize; index++) { - dateValues[index] = - DateUtils.parseIntToLocalDate(ReadWriteIOUtils.readInt(byteBuffer)); - } - values[i] = dateValues; - break; - case INT64: - case TIMESTAMP: - long[] longValues = new long[rowSize]; - for (int index = 0; index < rowSize; index++) { - longValues[index] = ReadWriteIOUtils.readLong(byteBuffer); - } - values[i] = longValues; - break; - case FLOAT: - float[] floatValues = new float[rowSize]; - for (int index = 0; index < rowSize; index++) { - floatValues[index] = ReadWriteIOUtils.readFloat(byteBuffer); - } - values[i] = floatValues; - break; - case DOUBLE: - double[] doubleValues = new double[rowSize]; - for (int index = 0; index < rowSize; index++) { - doubleValues[index] = ReadWriteIOUtils.readDouble(byteBuffer); - } - values[i] = doubleValues; - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[] binaryValues = new Binary[rowSize]; - for (int index = 0; index < rowSize; index++) { - boolean isNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); - if (isNotNull) { - binaryValues[index] = ReadWriteIOUtils.readBinary(byteBuffer); - } else { - binaryValues[index] = Binary.EMPTY_VALUE; - } - } - values[i] = binaryValues; - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.tablet_client_type_not_supported", types[i])); - } + values[i] = Type.fromTsDataType(types[i]).deserializeArray(byteBuffer, rowSize); } } return values; @@ -1209,98 +907,9 @@ public boolean equals(Object o) { return false; } - switch (schemas.get(i).getType()) { - case INT32: - int[] thisIntValues = (int[]) values[i]; - int[] thatIntValues = (int[]) thatValues[i]; - if (thisIntValues.length < rowSize || thatIntValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisIntValues[j] != thatIntValues[j]) { - return false; - } - } - break; - case DATE: - LocalDate[] thisDateValues = (LocalDate[]) values[i]; - LocalDate[] thatDateValues = (LocalDate[]) thatValues[i]; - if (thisDateValues.length < rowSize || thatDateValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (!thisDateValues[j].equals(thatDateValues[j])) { - return false; - } - } - break; - case INT64: - case TIMESTAMP: - long[] thisLongValues = (long[]) values[i]; - long[] thatLongValues = (long[]) thatValues[i]; - if (thisLongValues.length < rowSize || thatLongValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisLongValues[j] != thatLongValues[j]) { - return false; - } - } - break; - case FLOAT: - float[] thisFloatValues = (float[]) values[i]; - float[] thatFloatValues = (float[]) thatValues[i]; - if (thisFloatValues.length < rowSize || thatFloatValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisFloatValues[j] != thatFloatValues[j]) { - return false; - } - } - break; - case DOUBLE: - double[] thisDoubleValues = (double[]) values[i]; - double[] thatDoubleValues = (double[]) thatValues[i]; - if (thisDoubleValues.length < rowSize || thatDoubleValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisDoubleValues[j] != thatDoubleValues[j]) { - return false; - } - } - break; - case BOOLEAN: - boolean[] thisBooleanValues = (boolean[]) values[i]; - boolean[] thatBooleanValues = (boolean[]) thatValues[i]; - if (thisBooleanValues.length < rowSize || thatBooleanValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (thisBooleanValues[j] != thatBooleanValues[j]) { - return false; - } - } - break; - case TEXT: - case STRING: - case BLOB: - case OBJECT: - Binary[] thisBinaryValues = (Binary[]) values[i]; - Binary[] thatBinaryValues = (Binary[]) thatValues[i]; - if (thisBinaryValues.length < rowSize || thatBinaryValues.length < rowSize) { - return false; - } - for (int j = 0; j < rowSize; j++) { - if (!thisBinaryValues[j].equals(thatBinaryValues[j])) { - return false; - } - } - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", schemas.get(i).getType())); + if (!Type.fromTsDataType(schemas.get(i).getType()) + .arrayEquals(values[i], thatValues[i], rowSize)) { + return false; } } @@ -1378,29 +987,7 @@ public Object getValue(int i, int j) { if (isNull(i, j)) { return null; } - switch (schemas.get(j).getType()) { - case BLOB: - case TEXT: - case STRING: - case OBJECT: - return ((Binary[]) values[j])[i]; - case INT32: - return ((int[]) values[j])[i]; - case FLOAT: - return ((float[]) values[j])[i]; - case DOUBLE: - return ((double[]) values[j])[i]; - case BOOLEAN: - return ((boolean[]) values[j])[i]; - case INT64: - case TIMESTAMP: - return ((long[]) values[j])[i]; - case DATE: - return ((LocalDate[]) values[j])[i]; - default: - throw new IllegalArgumentException( - Messages.format("error.write.tablet_unsupported_type", schemas.get(j).getType())); - } + return Type.fromTsDataType(schemas.get(j).getType()).getValue(values[j], i); } /** @@ -1700,33 +1287,7 @@ public long ramBytesUsed() { if (values == null || values.length <= column) { continue; } - switch (tsDataType) { - case INT64: - case TIMESTAMP: - totalSizeInBytes += RamUsageEstimator.sizeOf((long[]) values[column]); - break; - case DATE: - totalSizeInBytes += RamUsageEstimator.sizeOf((LocalDate[]) values[column]); - break; - case INT32: - totalSizeInBytes += RamUsageEstimator.sizeOf((int[]) values[column]); - break; - case DOUBLE: - totalSizeInBytes += RamUsageEstimator.sizeOf((double[]) values[column]); - break; - case FLOAT: - totalSizeInBytes += RamUsageEstimator.sizeOf((float[]) values[column]); - break; - case BOOLEAN: - totalSizeInBytes += RamUsageEstimator.sizeOf((boolean[]) values[column]); - break; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - totalSizeInBytes += RamUsageEstimator.sizeOf((Binary[]) values[column]); - break; - } + totalSizeInBytes += Type.fromTsDataType(tsDataType).estimateArraySize(values[column]); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java index 103e7503d..388f6aa3b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java @@ -19,11 +19,10 @@ package org.apache.tsfile.write.record.datapoint; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.StringContainer; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.ChunkWriterImpl; @@ -67,46 +66,13 @@ protected DataPoint(TSDataType type, String measurementId) { * @return data point class according to data type */ public static DataPoint getDataPoint(TSDataType dataType, String measurementId, String value) { - DataPoint dataPoint = null; try { - switch (dataType) { - case INT32: - dataPoint = new IntDataPoint(measurementId, Integer.parseInt(value)); - break; - case DATE: - dataPoint = new IntDataPoint(measurementId, DateUtils.parseDateExpressionToInt(value)); - break; - case INT64: - case TIMESTAMP: - dataPoint = new LongDataPoint(measurementId, Long.parseLong(value)); - break; - case FLOAT: - dataPoint = new FloatDataPoint(measurementId, Float.parseFloat(value)); - break; - case DOUBLE: - dataPoint = new DoubleDataPoint(measurementId, Double.parseDouble(value)); - break; - case BOOLEAN: - dataPoint = new BooleanDataPoint(measurementId, Boolean.parseBoolean(value)); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - dataPoint = - new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format("error.write.type_not_supported", dataType)); - } + return Type.fromTsDataType(dataType).getDataPoint(measurementId, value); } catch (Exception e) { throw new UnSupportedDataTypeException( Messages.format( "error.write.datapoint_type_value_mismatch", measurementId, dataType, value)); } - - return dataPoint; } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java index 8e1561864..6c0503832 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/TableTsBlock2TsFileWriter.java @@ -28,8 +28,8 @@ import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl; import org.apache.tsfile.write.chunk.IChunkGroupWriter; import org.apache.tsfile.write.chunk.TableChunkGroupWriterImpl; @@ -198,13 +198,16 @@ public int getRowCount() { */ private class TableTsBlockChunkGroupWriterImpl extends TableChunkGroupWriterImpl { private final ValueChunkWriter[] valueChunkWriters; + private final Type[] valueTypes; public TableTsBlockChunkGroupWriterImpl(IDeviceID deviceId) throws IOException { super(deviceId); // Initialize ValueChunkWriter for each measurement this.valueChunkWriters = new ValueChunkWriter[fieldColumnSchemas.length]; + this.valueTypes = new Type[fieldColumnSchemas.length]; for (int i = 0; i < fieldColumnSchemas.length; i++) { valueChunkWriters[i] = tryToAddSeriesWriterInternal(fieldColumnSchemas[i]); + valueTypes[i] = Type.fromTsDataType(valueChunkWriters[i].getDataType()); } } @@ -227,35 +230,7 @@ public int write(Column timeColumn, Column[] valueColumns, int startRowIndex, in Column valueColumn = valueColumns[valueColumnIndex]; ValueChunkWriter valueChunkWriter = valueChunkWriters[valueColumnIndex]; boolean isNull = valueColumn.isNull(rowIndex); - switch (valueChunkWriter.getDataType()) { - case BOOLEAN: - valueChunkWriter.write( - time, isNull ? false : valueColumn.getBoolean(rowIndex), isNull); - break; - case INT32: - case DATE: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getInt(rowIndex), isNull); - break; - case INT64: - case TIMESTAMP: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getLong(rowIndex), isNull); - break; - case FLOAT: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getFloat(rowIndex), isNull); - break; - case DOUBLE: - valueChunkWriter.write(time, isNull ? 0 : valueColumn.getDouble(rowIndex), isNull); - break; - case TEXT: - case BLOB: - case STRING: - valueChunkWriter.write(time, isNull ? null : valueColumn.getBinary(rowIndex), isNull); - break; - default: - throw new UnSupportedDataTypeException( - Messages.format( - "error.write.type_not_supported", valueChunkWriter.getDataType().getType())); - } + valueTypes[valueColumnIndex].write(valueChunkWriter, time, valueColumn, rowIndex, isNull); } timeChunkWriter.write(time); lastTime = time; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/ColumnSerializeWithoutNullsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/ColumnSerializeWithoutNullsTest.java new file mode 100644 index 000000000..c412a8b9c --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/ColumnSerializeWithoutNullsTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.common.block; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.block.column.NullColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.utils.Binary; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +public class ColumnSerializeWithoutNullsTest { + + private static final boolean[] NULL_INDICATORS = {false, true, false, true}; + + @Test + public void testSerializeWithoutNulls() throws IOException { + ByteBuffer buffer = + serialize( + new BooleanColumn( + 4, Optional.of(NULL_INDICATORS), new boolean[] {true, true, false, true})); + Assert.assertEquals((byte) 0b1000_0000, buffer.get()); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = serialize(new IntColumn(4, Optional.of(NULL_INDICATORS), new int[] {1, 2, 3, 4})); + Assert.assertEquals(1, buffer.getInt()); + Assert.assertEquals(3, buffer.getInt()); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize(new LongColumn(4, Optional.of(NULL_INDICATORS), new long[] {1L, 2L, 3L, 4L})); + Assert.assertEquals(1L, buffer.getLong()); + Assert.assertEquals(3L, buffer.getLong()); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize(new FloatColumn(4, Optional.of(NULL_INDICATORS), new float[] {1F, 2F, 3F, 4F})); + Assert.assertEquals(1F, Float.intBitsToFloat(buffer.getInt()), 0); + Assert.assertEquals(3F, Float.intBitsToFloat(buffer.getInt()), 0); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize(new DoubleColumn(4, Optional.of(NULL_INDICATORS), new double[] {1D, 2D, 3D, 4D})); + Assert.assertEquals(1D, Double.longBitsToDouble(buffer.getLong()), 0); + Assert.assertEquals(3D, Double.longBitsToDouble(buffer.getLong()), 0); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = + serialize( + new BinaryColumn( + 4, + Optional.of(NULL_INDICATORS), + new Binary[] {binary("a"), binary("b"), binary("cc"), binary("d")})); + Assert.assertEquals(binary("a"), readBinary(buffer)); + Assert.assertEquals(binary("cc"), readBinary(buffer)); + Assert.assertFalse(buffer.hasRemaining()); + + buffer = serialize(new TimeColumn(4, new long[] {1L, 2L, 3L, 4L})); + Assert.assertEquals(1L, buffer.getLong()); + Assert.assertEquals(2L, buffer.getLong()); + Assert.assertEquals(3L, buffer.getLong()); + Assert.assertEquals(4L, buffer.getLong()); + Assert.assertFalse(buffer.hasRemaining()); + + Assert.assertFalse(serialize(new NullColumn(4)).hasRemaining()); + } + + private static ByteBuffer serialize(Column column) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + column.serializeWithoutNulls(new DataOutputStream(output)); + return ByteBuffer.wrap(output.toByteArray()); + } + + private static Binary binary(String value) { + return new Binary(value, StandardCharsets.UTF_8); + } + + private static Binary readBinary(ByteBuffer buffer) { + byte[] value = new byte[buffer.getInt()]; + buffer.get(value); + return new Binary(value); + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java index 8c82a11dc..51f6c833d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/Int32ArrayColumnEncoderTest.java @@ -37,6 +37,13 @@ import java.util.Random; public class Int32ArrayColumnEncoderTest { + + @Test(expected = IllegalArgumentException.class) + public void testInvalidDataType() { + ColumnEncoderFactory.get(ColumnEncoding.INT32_ARRAY) + .readColumn(ByteBuffer.allocate(0), TSDataType.DOUBLE, 0); + } + @Test public void testIntColumn() { final int positionCount = 10; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java index 1c0fa5b2b..b1fcb3e8d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/block/NullColumnUnitTest.java @@ -22,12 +22,46 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.column.NullColumn; +import org.apache.tsfile.read.common.type.Type; import org.junit.Assert; import org.junit.Test; public class NullColumnUnitTest { + @Test + public void testCreatingNullColumnsByType() { + Object[][] testCases = { + {TSDataType.BOOLEAN, TSDataType.BOOLEAN}, + {TSDataType.INT32, TSDataType.INT32}, + {TSDataType.DATE, TSDataType.INT32}, + {TSDataType.INT64, TSDataType.INT64}, + {TSDataType.TIMESTAMP, TSDataType.INT64}, + {TSDataType.FLOAT, TSDataType.FLOAT}, + {TSDataType.DOUBLE, TSDataType.DOUBLE}, + {TSDataType.TEXT, TSDataType.TEXT}, + {TSDataType.STRING, TSDataType.TEXT}, + {TSDataType.BLOB, TSDataType.TEXT}, + {TSDataType.OBJECT, TSDataType.TEXT} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + TSDataType expectedDataType = (TSDataType) testCase[1]; + assertNullColumn(Type.fromTsDataType(dataType).createNullColumn(10), expectedDataType); + assertNullColumn(NullColumn.create(dataType, 10), expectedDataType); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + NullColumn.create(dataType, 10); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException ignored) { + // Expected. + } + } + } + @Test public void testCreatingBooleanNullColumn() { Column nullColumn = NullColumn.create(TSDataType.BOOLEAN, 10); @@ -87,4 +121,12 @@ public void testCreatingDoubleNullColumn() { Assert.assertTrue(nullColumn.isNull(0)); Assert.assertTrue(nullColumn.isNull(9)); } + + private static void assertNullColumn(Column column, TSDataType expectedDataType) { + Assert.assertEquals(expectedDataType, column.getDataType()); + Assert.assertEquals(10, column.getPositionCount()); + Assert.assertTrue(column.mayHaveNull()); + Assert.assertTrue(column.isNull(0)); + Assert.assertTrue(column.isNull(9)); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java index 1dc6214d7..668fee698 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/common/conf/TSFileDescriptorTest.java @@ -19,6 +19,9 @@ package org.apache.tsfile.common.conf; import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.Assert; import org.junit.Before; @@ -82,4 +85,57 @@ public void testGetInstanceOverWriteFromFile() { Assert.assertEquals(tsFileConfig1.getMaxStringLength(), tsFileConfig2.getMaxStringLength()); } + + @Test + public void testGetValueEncoder() { + tsFileConfig1.setBooleanEncoding(TSEncoding.PLAIN.name()); + tsFileConfig1.setInt32Encoding(TSEncoding.RLE.name()); + tsFileConfig1.setInt64Encoding(TSEncoding.GORILLA.name()); + tsFileConfig1.setFloatEncoding(TSEncoding.TS_2DIFF.name()); + tsFileConfig1.setDoubleEncoding(TSEncoding.RLE.name()); + tsFileConfig1.setTextEncoding(TSEncoding.DICTIONARY.name()); + + Assert.assertEquals(TSEncoding.PLAIN, tsFileConfig1.getValueEncoder(TSDataType.BOOLEAN)); + Assert.assertEquals(TSEncoding.RLE, tsFileConfig1.getValueEncoder(TSDataType.INT32)); + Assert.assertEquals(TSEncoding.RLE, tsFileConfig1.getValueEncoder(TSDataType.DATE)); + Assert.assertEquals(TSEncoding.GORILLA, tsFileConfig1.getValueEncoder(TSDataType.INT64)); + Assert.assertEquals(TSEncoding.GORILLA, tsFileConfig1.getValueEncoder(TSDataType.TIMESTAMP)); + Assert.assertEquals(TSEncoding.TS_2DIFF, tsFileConfig1.getValueEncoder(TSDataType.FLOAT)); + Assert.assertEquals(TSEncoding.RLE, tsFileConfig1.getValueEncoder(TSDataType.DOUBLE)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.TEXT)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.STRING)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.BLOB)); + Assert.assertEquals(TSEncoding.DICTIONARY, tsFileConfig1.getValueEncoder(TSDataType.OBJECT)); + } + + @Test + public void testGetCompressor() { + tsFileConfig1.setCompressor(CompressionType.ZSTD.name()); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.BOOLEAN)); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.INT32)); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.TEXT)); + + tsFileConfig1.setBooleanCompression(CompressionType.UNCOMPRESSED.name()); + tsFileConfig1.setInt32Compression(CompressionType.SNAPPY.name()); + tsFileConfig1.setInt64Compression(CompressionType.GZIP.name()); + tsFileConfig1.setFloatCompression(CompressionType.LZ4.name()); + tsFileConfig1.setDoubleCompression(CompressionType.ZSTD.name()); + tsFileConfig1.setTextCompression(CompressionType.LZMA2.name()); + + Assert.assertEquals( + CompressionType.UNCOMPRESSED, tsFileConfig1.getCompressor(TSDataType.BOOLEAN)); + Assert.assertEquals(CompressionType.SNAPPY, tsFileConfig1.getCompressor(TSDataType.INT32)); + Assert.assertEquals(CompressionType.SNAPPY, tsFileConfig1.getCompressor(TSDataType.DATE)); + Assert.assertEquals(CompressionType.GZIP, tsFileConfig1.getCompressor(TSDataType.INT64)); + Assert.assertEquals(CompressionType.GZIP, tsFileConfig1.getCompressor(TSDataType.TIMESTAMP)); + Assert.assertEquals(CompressionType.LZ4, tsFileConfig1.getCompressor(TSDataType.FLOAT)); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.DOUBLE)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.TEXT)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.STRING)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.BLOB)); + Assert.assertEquals(CompressionType.LZMA2, tsFileConfig1.getCompressor(TSDataType.OBJECT)); + + tsFileConfig1.setInt32Compression(null); + Assert.assertEquals(CompressionType.ZSTD, tsFileConfig1.getCompressor(TSDataType.INT32)); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/DecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/DecoderTest.java new file mode 100644 index 000000000..000e08206 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/DecoderTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.encoding.decoder; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; + +import org.junit.Assert; +import org.junit.Test; + +public class DecoderTest { + + @Test + public void testGetDecoderByType() { + Object[][] testCases = { + {TSEncoding.PLAIN, TSDataType.UNKNOWN, PlainDecoder.class}, + {TSEncoding.DICTIONARY, TSDataType.BLOB, DictionaryDecoder.class}, + {TSEncoding.RLE, TSDataType.BOOLEAN, IntRleDecoder.class}, + {TSEncoding.RLE, TSDataType.INT32, IntRleDecoder.class}, + {TSEncoding.RLE, TSDataType.DATE, IntRleDecoder.class}, + {TSEncoding.RLE, TSDataType.INT64, LongRleDecoder.class}, + {TSEncoding.RLE, TSDataType.VECTOR, LongRleDecoder.class}, + {TSEncoding.RLE, TSDataType.TIMESTAMP, LongRleDecoder.class}, + {TSEncoding.RLE, TSDataType.FLOAT, FloatDecoder.class}, + {TSEncoding.RLE, TSDataType.DOUBLE, FloatDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.INT32, DeltaBinaryDecoder.IntDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.DATE, DeltaBinaryDecoder.IntDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.INT64, DeltaBinaryDecoder.LongDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.VECTOR, DeltaBinaryDecoder.LongDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.TIMESTAMP, DeltaBinaryDecoder.LongDeltaDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.FLOAT, FloatDecoder.class}, + {TSEncoding.TS_2DIFF, TSDataType.DOUBLE, FloatDecoder.class}, + {TSEncoding.GORILLA_V1, TSDataType.FLOAT, SinglePrecisionDecoderV1.class}, + {TSEncoding.GORILLA_V1, TSDataType.DOUBLE, DoublePrecisionDecoderV1.class}, + {TSEncoding.REGULAR, TSDataType.INT32, RegularDataDecoder.IntRegularDecoder.class}, + {TSEncoding.REGULAR, TSDataType.VECTOR, RegularDataDecoder.LongRegularDecoder.class}, + {TSEncoding.GORILLA, TSDataType.INT32, IntGorillaDecoder.class}, + {TSEncoding.GORILLA, TSDataType.VECTOR, LongGorillaDecoder.class}, + {TSEncoding.GORILLA, TSDataType.FLOAT, SinglePrecisionDecoderV2.class}, + {TSEncoding.GORILLA, TSDataType.DOUBLE, DoublePrecisionDecoderV2.class}, + {TSEncoding.ZIGZAG, TSDataType.INT32, IntZigzagDecoder.class}, + {TSEncoding.ZIGZAG, TSDataType.INT64, LongZigzagDecoder.class}, + {TSEncoding.CHIMP, TSDataType.INT32, IntChimpDecoder.class}, + {TSEncoding.CHIMP, TSDataType.VECTOR, LongChimpDecoder.class}, + {TSEncoding.CHIMP, TSDataType.FLOAT, SinglePrecisionChimpDecoder.class}, + {TSEncoding.CHIMP, TSDataType.DOUBLE, DoublePrecisionChimpDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.INT32, IntSprintzDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.INT64, LongSprintzDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.FLOAT, FloatSprintzDecoder.class}, + {TSEncoding.SPRINTZ, TSDataType.DOUBLE, DoubleSprintzDecoder.class}, + {TSEncoding.RLBE, TSDataType.INT32, IntRLBEDecoder.class}, + {TSEncoding.RLBE, TSDataType.INT64, LongRLBEDecoder.class}, + {TSEncoding.RLBE, TSDataType.FLOAT, FloatRLBEDecoder.class}, + {TSEncoding.RLBE, TSDataType.DOUBLE, DoubleRLBEDecoder.class}, + {TSEncoding.CAMEL, TSDataType.DOUBLE, CamelDecoder.class} + }; + + for (Object[] testCase : testCases) { + Assert.assertEquals( + testCase[2], + Decoder.getDecoderByType((TSEncoding) testCase[0], (TSDataType) testCase[1]).getClass()); + } + } + + @Test + public void testUnsupportedDecoder() { + Object[][] testCases = { + {TSEncoding.RLE, TSDataType.TEXT}, + {TSEncoding.TS_2DIFF, TSDataType.BOOLEAN}, + {TSEncoding.GORILLA_V1, TSDataType.INT32}, + {TSEncoding.REGULAR, TSDataType.FLOAT}, + {TSEncoding.GORILLA, TSDataType.BOOLEAN}, + {TSEncoding.ZIGZAG, TSDataType.VECTOR}, + {TSEncoding.CHIMP, TSDataType.TEXT}, + {TSEncoding.SPRINTZ, TSDataType.VECTOR}, + {TSEncoding.RLBE, TSDataType.VECTOR}, + {TSEncoding.CAMEL, TSDataType.FLOAT} + }; + + for (Object[] testCase : testCases) { + try { + Decoder.getDecoderByType((TSEncoding) testCase[0], (TSDataType) testCase[1]); + Assert.fail("Expected TsFileDecodingException"); + } catch (TsFileDecodingException ignored) { + // Expected. + } + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java index ba12c6358..47ad21a2d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java @@ -20,6 +20,8 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.junit.Test; @@ -34,6 +36,131 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public class StatisticsTest { + private static final TypeService CHECK_STATISTICS_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, INT64, FLOAT, DOUBLE, TIMESTAMP, DATE -> + (statistics, min, max, sum) -> { + assertEquals(min, ((Number) statistics.getMinValue()).intValue()); + assertEquals(max, ((Number) statistics.getMaxValue()).intValue()); + assertEquals(min, ((Number) statistics.getFirstValue()).intValue()); + assertEquals(max, ((Number) statistics.getLastValue()).intValue()); + assertEquals(sum, statistics.getSumDoubleValue(), 0.001); + }; + case BOOLEAN -> + (statistics, min, max, sum) -> { + assertEquals(min % 2 == 1, statistics.getFirstValue()); + assertEquals(max % 2 == 1, statistics.getLastValue()); + assertEquals(sum, statistics.getSumDoubleValue(), 0.001); + }; + case TEXT -> + (statistics, min, max, sum) -> { + assertEquals( + new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), + statistics.getFirstValue()); + assertEquals( + new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), + statistics.getLastValue()); + }; + case STRING -> + (statistics, min, max, sum) -> { + assertEquals( + new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), + statistics.getMinValue()); + assertEquals( + new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), + statistics.getMaxValue()); + assertEquals( + new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), + statistics.getFirstValue()); + assertEquals( + new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), + statistics.getLastValue()); + }; + case BLOB, OBJECT -> (statistics, min, max, sum) -> {}; + case ROW, UNKNOWN, VECTOR -> + (statistics, min, max, sum) -> { + throw new IllegalArgumentException(type.getTypeEnum().toString()); + }; + }; + + private static final TypeService GENERATE_STATISTICS_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + value -> { + IntegerStatistics statistics = new IntegerStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case INT64 -> + value -> { + LongStatistics statistics = new LongStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case FLOAT -> + value -> { + FloatStatistics statistics = new FloatStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case DOUBLE -> + value -> { + DoubleStatistics statistics = new DoubleStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case TEXT -> + value -> { + BinaryStatistics statistics = new BinaryStatistics(); + statistics.initializeStats( + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + return statistics; + }; + case STRING -> + value -> { + StringStatistics statistics = new StringStatistics(); + statistics.initializeStats( + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET), + new Binary(String.valueOf(value), TSFileConfig.STRING_CHARSET)); + return statistics; + }; + case BOOLEAN -> + value -> { + BooleanStatistics statistics = new BooleanStatistics(); + boolean booleanValue = value % 2 == 1; + statistics.initializeStats(booleanValue, booleanValue, booleanValue ? 1 : 0); + return statistics; + }; + case BLOB -> value -> new BlobStatistics(); + case OBJECT -> value -> new ObjectStatistics(); + case DATE -> + value -> { + DateStatistics statistics = new DateStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case TIMESTAMP -> + value -> { + TimestampStatistics statistics = new TimestampStatistics(); + statistics.initializeStats(value, value, value, value, value); + return statistics; + }; + case ROW, UNKNOWN, VECTOR -> + value -> { + throw new IllegalArgumentException(type.getTypeEnum().toString()); + }; + }; + + static { + CHECK_STATISTICS_SERVICE.check(); + GENERATE_STATISTICS_SERVICE.check(); + } + @Test public void testCrossTypeMerge() { Set dataTypes = new HashSet<>(); @@ -69,120 +196,29 @@ public void testCrossTypeMerge() { private static void checkStatistics(Statistics statistics, int min, int max, double sum) { assertEquals(min, statistics.getStartTime()); assertEquals(max, statistics.getEndTime()); - switch (statistics.getType()) { - case INT32: - case INT64: - case FLOAT: - case DOUBLE: - case TIMESTAMP: - case DATE: - assertEquals(min, ((Number) statistics.getMinValue()).intValue()); - assertEquals(max, ((Number) statistics.getMaxValue()).intValue()); - assertEquals(min, ((Number) statistics.getFirstValue()).intValue()); - assertEquals(max, ((Number) statistics.getLastValue()).intValue()); - assertEquals(sum, statistics.getSumDoubleValue(), 0.001); - break; - case BOOLEAN: - assertEquals(min % 2 == 1, statistics.getFirstValue()); - assertEquals(max % 2 == 1, statistics.getLastValue()); - assertEquals(sum, statistics.getSumDoubleValue(), 0.001); - break; - case TEXT: - assertEquals( - new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), - statistics.getFirstValue()); - assertEquals( - new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), - statistics.getLastValue()); - break; - case STRING: - assertEquals( - new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), statistics.getMinValue()); - assertEquals( - new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), statistics.getMaxValue()); - assertEquals( - new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), - statistics.getFirstValue()); - assertEquals( - new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), - statistics.getLastValue()); - break; - case BLOB: - case OBJECT: - break; - default: - throw new IllegalArgumentException(statistics.getType().toString()); - } + CHECK_STATISTICS_SERVICE + .call(Type.fromTsDataType(statistics.getType())) + .check(statistics, min, max, sum); } private static Statistics genStatistics(TSDataType dataType, int val) { - Statistics result; - switch (dataType) { - case INT32: - IntegerStatistics intStat = new IntegerStatistics(); - intStat.initializeStats(val, val, val, val, val); - result = intStat; - break; - case INT64: - LongStatistics longStat = new LongStatistics(); - longStat.initializeStats(val, val, val, val, val); - result = longStat; - break; - case FLOAT: - FloatStatistics floatStat = new FloatStatistics(); - floatStat.initializeStats(val, val, val, val, val); - result = floatStat; - break; - case DOUBLE: - DoubleStatistics doubleStat = new DoubleStatistics(); - doubleStat.initializeStats(val, val, val, val, val); - result = doubleStat; - break; - case TEXT: - BinaryStatistics binaryStat = new BinaryStatistics(); - binaryStat.initializeStats( - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET)); - result = binaryStat; - break; - case STRING: - StringStatistics stringStat = new StringStatistics(); - stringStat.initializeStats( - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET), - new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET)); - result = stringStat; - break; - case BOOLEAN: - BooleanStatistics boolStat = new BooleanStatistics(); - boolStat.initializeStats(val % 2 == 1, val % 2 == 1, val % 2 == 1 ? 1 : 0); - result = boolStat; - break; - case BLOB: - BlobStatistics blobStat = new BlobStatistics(); - result = blobStat; - break; - case OBJECT: - ObjectStatistics objectStat = new ObjectStatistics(); - result = objectStat; - break; - case DATE: - DateStatistics dateStat = new DateStatistics(); - dateStat.initializeStats(val, val, val, val, val); - result = dateStat; - break; - case TIMESTAMP: - TimestampStatistics timestampStat = new TimestampStatistics(); - timestampStat.initializeStats(val, val, val, val, val); - result = timestampStat; - break; - default: - throw new IllegalArgumentException(dataType.toString()); - } + Statistics result = + GENERATE_STATISTICS_SERVICE.call(Type.fromTsDataType(dataType)).generate(val); result.setStartTime(val); result.setEndTime(val); result.setEmpty(false); return result; } + + @FunctionalInterface + private interface StatisticsChecker { + + void check(Statistics statistics, int min, int max, double sum); + } + + @FunctionalInterface + private interface StatisticsGenerator { + + Statistics generate(int value); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java index e388fa51e..fde2d0b9c 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/BatchDataTest.java @@ -21,18 +21,58 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.TsPrimitiveType; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class BatchDataTest { + private static final TypeService ASSERT_SERIALIZED_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> + (inputStream, expectedValue) -> + assertEquals(expectedValue, inputStream.readBoolean()); + case INT32, DATE -> + (inputStream, expectedValue) -> assertEquals(expectedValue, inputStream.readInt()); + case INT64, TIMESTAMP -> + (inputStream, expectedValue) -> assertEquals(expectedValue, inputStream.readLong()); + case FLOAT -> + (inputStream, expectedValue) -> + assertEquals((float) expectedValue, inputStream.readFloat(), 0); + case DOUBLE -> + (inputStream, expectedValue) -> + assertEquals((double) expectedValue, inputStream.readDouble(), 0); + case TEXT, STRING, BLOB, OBJECT -> + (inputStream, expectedValue) -> { + Binary binary = (Binary) expectedValue; + assertArrayEquals( + binary.getValues(), inputStream.readNBytes(inputStream.readInt())); + }; + case ROW, UNKNOWN, VECTOR -> + (inputStream, expectedValue) -> fail("Unexpected data type: " + type.getTypeEnum()); + }; + + static { + ASSERT_SERIALIZED_VALUE_SERVICE.check(); + } + @Test public void testInt() { BatchData batchData = new BatchData(TSDataType.INT32); @@ -81,4 +121,88 @@ public void testSignal() { } fail(); } + + @Test + public void testSerializeData() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260714}, + {TSDataType.INT64, 1L}, + {TSDataType.TIMESTAMP, 2L}, + {TSDataType.FLOAT, 1.0F}, + {TSDataType.DOUBLE, 2.0D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Object value = testCase[1]; + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(100L, value); + + try (DataInputStream inputStream = serialize(batchData)) { + assertEquals(100L, inputStream.readLong()); + assertSerializedValue(inputStream, dataType, value); + assertEquals(-1, inputStream.read()); + } + } + } + + @Test + public void testSerializeVectorData() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + TsPrimitiveType[] values = { + null, + new TsPrimitiveType.TsBoolean(true), + new TsPrimitiveType.TsInt(1), + new TsPrimitiveType.TsInt(20260714, TSDataType.DATE), + new TsPrimitiveType.TsLong(1L), + new TsPrimitiveType.TsFloat(1.0F), + new TsPrimitiveType.TsDouble(2.0D), + new TsPrimitiveType.TsBinary(binary) + }; + BatchData batchData = new BatchData(TSDataType.VECTOR); + batchData.putAnObject(100L, values); + + try (DataInputStream inputStream = serialize(batchData)) { + assertEquals(100L, inputStream.readLong()); + assertEquals(values.length, inputStream.readInt()); + for (TsPrimitiveType value : values) { + if (value == null) { + assertEquals(0, inputStream.readByte()); + continue; + } + assertEquals(1, inputStream.readByte()); + assertEquals(value.getDataType().serialize(), inputStream.readByte()); + assertSerializedValue(inputStream, value.getDataType(), value.getValue()); + } + assertEquals(-1, inputStream.read()); + } + } + + private DataInputStream serialize(BatchData batchData) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (DataOutputStream outputStream = new DataOutputStream(byteStream)) { + batchData.serializeData(outputStream); + } + return new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())); + } + + private void assertSerializedValue( + DataInputStream inputStream, TSDataType dataType, Object expectedValue) throws IOException { + ASSERT_SERIALIZED_VALUE_SERVICE + .call(Type.fromTsDataType(dataType)) + .assertValue(inputStream, expectedValue); + } + + @FunctionalInterface + private interface SerializedValueAsserter { + + void assertValue(DataInputStream inputStream, Object expectedValue) throws IOException; + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index f3a784324..51203e0db 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -21,6 +21,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.BooleanColumn; @@ -38,17 +39,272 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Optional; public class ColumnTest { + @Test + public void testConvertTo() { + Column source = + new IntColumn( + 4, Optional.of(new boolean[] {false, true, false, false}), new int[] {0, 1, 2, 3}) + .getRegion(1, 3); + + Column converted = source.convertTo(TSDataType.DOUBLE); + Assert.assertEquals(TSDataType.DOUBLE, converted.getDataType()); + Assert.assertEquals(3, converted.getPositionCount()); + Assert.assertTrue(converted.isNull(0)); + Assert.assertEquals(2.0D, converted.getDouble(1), 0); + Assert.assertEquals(3.0D, converted.getDouble(2), 0); + + Assert.assertSame(source, source.convertTo(TSDataType.INT32)); + Assert.assertThrows(ClassCastException.class, () -> source.convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows(NullPointerException.class, () -> source.convertTo(null)); + } + + @Test + public void testConvertToText() { + Column dateColumn = + new IntColumn( + 3, + Optional.of(new boolean[] {false, true, false}), + new int[] {20260714, 20260715, 20260716}, + TSDataType.DATE); + + Column converted = dateColumn.convertTo(TSDataType.TEXT); + Assert.assertEquals(TSDataType.TEXT, converted.getDataType()); + Assert.assertEquals(new Binary("2026-07-14", StandardCharsets.UTF_8), converted.getBinary(0)); + Assert.assertTrue(converted.isNull(1)); + Assert.assertEquals(new Binary("2026-07-16", StandardCharsets.UTF_8), converted.getBinary(2)); + } + + @Test + public void testConvertEncodedColumns() { + Column dictionary = + DictionaryColumn.create( + 3, new IntColumn(3, Optional.empty(), new int[] {1, 2, 3}), new int[] {2, 0, 2}); + Column convertedDictionary = dictionary.convertTo(TSDataType.DOUBLE); + Assert.assertTrue(convertedDictionary instanceof DictionaryColumn); + Assert.assertEquals(3.0D, convertedDictionary.getDouble(0), 0); + Assert.assertEquals(1.0D, convertedDictionary.getDouble(1), 0); + Assert.assertEquals(3.0D, convertedDictionary.getDouble(2), 0); + + Column runLengthEncoded = + new RunLengthEncodedColumn(new FloatColumn(1, Optional.empty(), new float[] {1.25F}), 4); + Column convertedRle = runLengthEncoded.convertTo(TSDataType.DOUBLE); + Assert.assertTrue(convertedRle instanceof RunLengthEncodedColumn); + Assert.assertEquals(4, convertedRle.getPositionCount()); + Assert.assertEquals(1.25D, convertedRle.getDouble(3), 0); + + Column nullColumn = new NullColumn(3).convertTo(TSDataType.DOUBLE); + Assert.assertEquals(TSDataType.DOUBLE, nullColumn.getDataType()); + Assert.assertEquals(3, nullColumn.getPositionCount()); + Assert.assertTrue(nullColumn.isNull(0)); + Assert.assertTrue(nullColumn.isNull(2)); + } + + @Test + public void testConvertToRejectsIncompatibleEmptyColumns() { + Column emptyDictionary = + DictionaryColumn.create( + 2, new IntColumn(2, Optional.empty(), new int[] {1, 2}), new int[] {0, 1}) + .getRegion(0, 0); + Column emptyRle = + new RunLengthEncodedColumn( + new IntColumn(1, Optional.of(new boolean[] {true}), new int[1]), 0); + + Assert.assertThrows( + ClassCastException.class, + () -> new IntColumn(0, Optional.empty(), new int[0]).convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows( + ClassCastException.class, + () -> + new IntColumn(2, Optional.of(new boolean[] {true, true}), new int[2]) + .convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows( + ClassCastException.class, () -> emptyDictionary.convertTo(TSDataType.BOOLEAN)); + Assert.assertThrows(ClassCastException.class, () -> emptyRle.convertTo(TSDataType.BOOLEAN)); + } + + @Test + public void testConvertToDoesNotReadBoxedValues() { + Column booleanColumn = + new BooleanColumn(1, Optional.empty(), new boolean[] {true}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column intColumn = + new IntColumn(1, Optional.empty(), new int[] {1}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column longColumn = + new LongColumn(1, Optional.empty(), new long[] {2L}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column floatColumn = + new FloatColumn(1, Optional.empty(), new float[] {3.0F}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column doubleColumn = + new DoubleColumn(1, Optional.empty(), new double[] {4.0D}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column binaryColumn = + new BinaryColumn( + 1, Optional.empty(), new Binary[] {new Binary("5", StandardCharsets.UTF_8)}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + Column timeColumn = + new TimeColumn(1, new long[] {6L}) { + @Override + public Object getObject(int position) { + throw new AssertionError("convertTo must use primitive accessors"); + } + }; + + Assert.assertEquals( + new Binary("true", StandardCharsets.UTF_8), + booleanColumn.convertTo(TSDataType.TEXT).getBinary(0)); + Assert.assertEquals(1.0D, intColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + Assert.assertEquals(2.0D, longColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + Assert.assertEquals(3.0D, floatColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + Assert.assertEquals( + new Binary("4.0", StandardCharsets.UTF_8), + doubleColumn.convertTo(TSDataType.TEXT).getBinary(0)); + Assert.assertEquals( + new Binary("5", StandardCharsets.UTF_8), + binaryColumn.convertTo(TSDataType.BLOB).getBinary(0)); + Assert.assertEquals(6.0D, timeColumn.convertTo(TSDataType.DOUBLE).getDouble(0), 0); + } + + @Test + public void testWriteTo() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Column intDictionary = + DictionaryColumn.create( + 2, new IntColumn(3, Optional.empty(), new int[] {7, 8, 9}), new int[] {2, 0}); + Column binaryRle = + new RunLengthEncodedColumn(new BinaryColumn(1, Optional.empty(), new Binary[] {binary}), 3); + + Object[][] testCases = { + {new BooleanColumn(2, Optional.empty(), new boolean[] {false, true}).getRegion(1, 1), 0}, + {new IntColumn(2, Optional.empty(), new int[] {0, 42}).getRegion(1, 1), 0}, + {new LongColumn(2, Optional.empty(), new long[] {0L, 42L}).getRegion(1, 1), 0}, + {new FloatColumn(2, Optional.empty(), new float[] {0.0F, 1.25F}).getRegion(1, 1), 0}, + {new DoubleColumn(2, Optional.empty(), new double[] {0.0D, 2.5D}).getRegion(1, 1), 0}, + { + new BinaryColumn(2, Optional.empty(), new Binary[] {Binary.EMPTY_VALUE, binary}) + .getRegion(1, 1), + 0 + }, + {new TimeColumn(2, new long[] {0L, 42L}).getRegion(1, 1), 0}, + {intDictionary, 0}, + {binaryRle, 2} + }; + + for (Object[] testCase : testCases) { + Column column = (Column) testCase[0]; + int index = (int) testCase[1]; + ByteArrayOutputStream expectedOutput = new ByteArrayOutputStream(); + Type.fromTsDataType(column.getDataType()) + .serializeValue(column.getObject(index), new DataOutputStream(expectedOutput)); + byte[] expected = expectedOutput.toByteArray(); + + ByteBuffer buffer = ByteBuffer.allocate(expected.length); + column.writeTo(index, buffer); + Assert.assertEquals(expected.length, buffer.position()); + Assert.assertArrayEquals(expected, Arrays.copyOf(buffer.array(), buffer.position())); + + ByteArrayOutputStream actualOutput = new ByteArrayOutputStream(); + column.writeTo(index, new DataOutputStream(actualOutput)); + Assert.assertArrayEquals(expected, actualOutput.toByteArray()); + } + + Column nullColumn = new NullColumn(1); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + nullColumn.writeTo(0, emptyBuffer); + ByteArrayOutputStream emptyOutput = new ByteArrayOutputStream(); + nullColumn.writeTo(0, new DataOutputStream(emptyOutput)); + Assert.assertEquals(0, emptyBuffer.position()); + Assert.assertEquals(0, emptyOutput.size()); + } + + @Test + public void testArePositionsEqual() { + IntColumnBuilder builder = new IntColumnBuilder(null, 5); + builder.writeInt(1); + builder.writeInt(1); + builder.writeInt(2); + builder.appendNull(); + builder.appendNull(); + Column column = builder.build(); + + Assert.assertTrue(column.arePositionsEqual(0, 1)); + Assert.assertFalse(column.arePositionsEqual(0, 2)); + Assert.assertFalse(column.arePositionsEqual(0, 3)); + Assert.assertTrue(column.arePositionsEqual(3, 4)); + + IntColumnBuilder thatBuilder = new IntColumnBuilder(null, 3); + thatBuilder.writeInt(1); + thatBuilder.writeInt(2); + thatBuilder.appendNull(); + Column that = thatBuilder.build(); + Assert.assertTrue(column.arePositionsEqual(0, that, 0)); + Assert.assertFalse(column.arePositionsEqual(0, that, 1)); + Assert.assertFalse(column.arePositionsEqual(0, that, 2)); + Assert.assertTrue(column.arePositionsEqual(3, that, 2)); + + Column dictionaryColumn = column.getPositions(new int[] {0, 2, 1}, 0, 3); + Assert.assertTrue(dictionaryColumn.arePositionsEqual(0, 2)); + Assert.assertFalse(dictionaryColumn.arePositionsEqual(0, 1)); + Assert.assertTrue(dictionaryColumn.arePositionsEqual(0, that, 0)); + Assert.assertTrue(that.arePositionsEqual(0, dictionaryColumn, 2)); + + Column runLengthEncodedColumn = new RunLengthEncodedColumn(column.getRegion(0, 1), 3); + Assert.assertTrue(runLengthEncodedColumn.arePositionsEqual(0, 2)); + Assert.assertTrue(runLengthEncodedColumn.arePositionsEqual(0, that, 0)); + Assert.assertTrue(that.arePositionsEqual(0, runLengthEncodedColumn, 2)); + + Column nanColumn = new DoubleColumn(1, Optional.empty(), new double[] {Double.NaN}); + Column nanRunLengthEncodedColumn = new RunLengthEncodedColumn(nanColumn, 2); + Assert.assertFalse(nanRunLengthEncodedColumn.arePositionsEqual(0, 1)); + + Column nullColumn = new NullColumn(2); + Assert.assertTrue(nullColumn.arePositionsEqual(0, 1)); + Assert.assertTrue(nullColumn.arePositionsEqual(0, that, 2)); + Assert.assertFalse(nullColumn.arePositionsEqual(0, that, 0)); + } + @Test public void timeColumnSubColumnTest() { TimeColumnBuilder columnBuilder = new TimeColumnBuilder(null, 10); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java index 1be38da0e..0ee21c4fd 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/FieldTest.java @@ -18,10 +18,17 @@ */ package org.apache.tsfile.read.common; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NullFieldException; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.junit.Assert; import org.junit.Test; +import java.nio.charset.StandardCharsets; + public class FieldTest { @Test(expected = NullFieldException.class) @@ -29,4 +36,29 @@ public void construct() { Field field = new Field(null); field.getIntV(); } + + @Test + public void copy() { + Object[][] values = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260716}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, new Binary("text", StandardCharsets.UTF_8)}, + {TSDataType.STRING, new Binary("string", StandardCharsets.UTF_8)}, + {TSDataType.BLOB, new Binary(new byte[] {1, 2})}, + {TSDataType.OBJECT, new Binary(BytesUtils.longToBytes(2L))} + }; + + for (Object[] value : values) { + TSDataType dataType = (TSDataType) value[0]; + Field source = Type.fromTsDataType(dataType).getField(value[1]); + Field copy = Field.copy(source); + Assert.assertEquals(source.getDataType(), copy.getDataType()); + Assert.assertEquals(source.getObjectValue(dataType), copy.getObjectValue(dataType)); + } + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java new file mode 100644 index 000000000..6f36ec3a9 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/type/TypeTest.java @@ -0,0 +1,1946 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.common.type; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.encoder.PlainEncoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.UnknownColumnTypeException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Array; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.Arrays; + +public class TypeTest { + + private static final int OFFSET = 2; + private static final int VALUE_LENGTH = 10; + + @Test + public void testWriteTsPrimitiveTypeToColumnBuilder() { + ColumnBuilder builder = Mockito.mock(ColumnBuilder.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260721, TSDataType.DATE)}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L)}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L)}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D)} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]).write(builder, (TsPrimitiveType) testCase[1]); + } + + Mockito.verify(builder).writeBoolean(true); + Mockito.verify(builder).writeInt(1); + Mockito.verify(builder).writeInt(20260721); + Mockito.verify(builder).writeLong(2L); + Mockito.verify(builder).writeLong(3L); + Mockito.verify(builder).writeFloat(1.25F); + Mockito.verify(builder).writeDouble(2.5D); + Mockito.reset(builder); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType binaryValue = new TsPrimitiveType.TsBinary(binary); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(builder, binaryValue); + Mockito.verify(builder).writeBinary(binary); + Mockito.reset(builder); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(builder, new TsPrimitiveType.TsLong(1L)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testWriteBytesToColumnBuilder() { + ColumnBuilder builder = Mockito.mock(ColumnBuilder.class); + Binary binary = new Binary("value", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260721, TSDataType.DATE)}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L)}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L)}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D)}, + {TSDataType.TEXT, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.STRING, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.BLOB, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.OBJECT, new TsPrimitiveType.TsBinary(binary)} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + TsPrimitiveType value = (TsPrimitiveType) testCase[1]; + byte[] bytes = new byte[OFFSET + type.calcTypeSize(value)]; + type.toBytes(value, bytes, OFFSET); + type.write(builder, bytes, OFFSET); + } + + Mockito.verify(builder).writeBoolean(true); + Mockito.verify(builder).writeInt(1); + Mockito.verify(builder).writeInt(20260721); + Mockito.verify(builder).writeLong(2L); + Mockito.verify(builder).writeLong(3L); + Mockito.verify(builder).writeFloat(1.25F); + Mockito.verify(builder).writeDouble(2.5D); + Mockito.verify(builder, Mockito.times(4)).writeBinary(binary); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(builder, new byte[Long.BYTES], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testWriteColumnValueToColumnBuilder() { + ColumnBuilder builder = Mockito.mock(ColumnBuilder.class); + Column column = Mockito.mock(Column.class); + Binary binary = new Binary("value", StandardCharsets.UTF_8); + + Mockito.when(column.getBoolean(0)).thenReturn(true); + Mockito.when(column.getInt(1)).thenReturn(1); + Mockito.when(column.getInt(2)).thenReturn(20260729); + Mockito.when(column.getLong(3)).thenReturn(2L); + Mockito.when(column.getLong(4)).thenReturn(3L); + Mockito.when(column.getFloat(5)).thenReturn(1.25F); + Mockito.when(column.getDouble(6)).thenReturn(2.5D); + Mockito.when(column.getBinary(7)).thenReturn(binary); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 0}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 2}, + {TSDataType.INT64, 3}, + {TSDataType.TIMESTAMP, 4}, + {TSDataType.FLOAT, 5}, + {TSDataType.DOUBLE, 6} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]).write(builder, column, (int) testCase[1]); + } + + Mockito.verify(builder).writeBoolean(true); + Mockito.verify(builder).writeInt(1); + Mockito.verify(builder).writeInt(20260729); + Mockito.verify(builder).writeLong(2L); + Mockito.verify(builder).writeLong(3L); + Mockito.verify(builder).writeFloat(1.25F); + Mockito.verify(builder).writeDouble(2.5D); + Mockito.reset(builder); + + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(builder, column, 7); + Mockito.verify(builder).writeBinary(binary); + Mockito.reset(builder); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(builder, column, 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testWriteTsPrimitiveTypeToArray() { + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType[] vector = {new TsPrimitiveType.TsInt(1), new TsPrimitiveType.TsLong(2L)}; + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true), new boolean[2]}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1), new int[2]}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260723, TSDataType.DATE), new int[2]}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260723, TSDataType.DATE), new LocalDate[2]}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L), new long[2]}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L), new long[2]}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F), new float[2]}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D), new double[2]}, + {TSDataType.TEXT, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.STRING, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.BLOB, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.OBJECT, new TsPrimitiveType.TsBinary(binary), new Binary[2]}, + {TSDataType.VECTOR, new TsPrimitiveType.TsVector(vector), new TsPrimitiveType[2][]} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + TsPrimitiveType expected = (TsPrimitiveType) testCase[1]; + Object array = testCase[2]; + type.write(expected, array, 1); + TsPrimitiveType actual = type.getValueAsTsPrimitiveType(array, 1); + Assert.assertEquals(expected, actual); + Assert.assertEquals(expected.getDataType(), actual.getDataType()); + } + + Type rowType = RowType.anonymousRow(IntType.getInstance(), LongType.getInstance()); + TsPrimitiveType rowValue = new TsPrimitiveType.TsVector(vector); + TsPrimitiveType[][] rowArray = new TsPrimitiveType[2][]; + rowType.write(rowValue, rowArray, 1); + Assert.assertEquals(rowValue, rowType.getValueAsTsPrimitiveType(rowArray, 1)); + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).write(new TsPrimitiveType.TsInt(1), new int[1], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testWriteBatchDataToArray() { + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType[] vector = {new TsPrimitiveType.TsInt(1), new TsPrimitiveType.TsLong(2L)}; + Object[][] testCases = { + {TSDataType.BOOLEAN, true, new boolean[2]}, + {TSDataType.INT32, 1, new int[2]}, + {TSDataType.DATE, 20260723, new int[2]}, + {TSDataType.DATE, 20260723, new LocalDate[2]}, + {TSDataType.INT64, 2L, new long[2]}, + {TSDataType.TIMESTAMP, 3L, new long[2]}, + {TSDataType.FLOAT, 1.25F, new float[2]}, + {TSDataType.DOUBLE, 2.5D, new double[2]}, + {TSDataType.TEXT, binary, new Binary[2]}, + {TSDataType.STRING, binary, new Binary[2]}, + {TSDataType.BLOB, binary, new Binary[2]}, + {TSDataType.OBJECT, binary, new Binary[2]}, + {TSDataType.VECTOR, vector, new TsPrimitiveType[2][]} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Type type = Type.fromTsDataType(dataType); + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(1L, testCase[1]); + Object array = testCase[2]; + type.write(batchData, array, 1); + TsPrimitiveType actual = type.getValueAsTsPrimitiveType(array, 1); + Assert.assertEquals(batchData.currentTsPrimitiveType(), actual); + Assert.assertEquals(batchData.currentTsPrimitiveType().getDataType(), actual.getDataType()); + } + + Type rowType = RowType.anonymousRow(IntType.getInstance(), LongType.getInstance()); + BatchData vectorBatchData = new BatchData(TSDataType.VECTOR); + vectorBatchData.putAnObject(1L, vector); + TsPrimitiveType[][] rowArray = new TsPrimitiveType[2][]; + rowType.write(vectorBatchData, rowArray, 1); + Assert.assertEquals( + vectorBatchData.currentTsPrimitiveType(), rowType.getValueAsTsPrimitiveType(rowArray, 1)); + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).write((BatchData) null, new Object[1], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testGetValueAsTsPrimitiveType() { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + TsPrimitiveType[] vector = {new TsPrimitiveType.TsInt(1), null}; + Object[][] testCases = { + {TSDataType.BOOLEAN, new boolean[] {false, true}, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new int[] {0, 1}, new TsPrimitiveType.TsInt(1)}, + { + TSDataType.DATE, + new int[] {0, 20260717}, + new TsPrimitiveType.TsInt(20260717, TSDataType.DATE) + }, + { + TSDataType.DATE, + new LocalDate[] {null, LocalDate.of(2026, 7, 17)}, + new TsPrimitiveType.TsInt(20260717, TSDataType.DATE) + }, + {TSDataType.INT64, new long[] {0L, 1L}, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.TIMESTAMP, new long[] {0L, 1L}, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.FLOAT, new float[] {0.0F, 1.25F}, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, new double[] {0.0D, 2.5D}, new TsPrimitiveType.TsDouble(2.5D)}, + {TSDataType.TEXT, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.STRING, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.BLOB, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.OBJECT, new Binary[] {null, binary}, new TsPrimitiveType.TsBinary(binary)}, + { + TSDataType.VECTOR, + new TsPrimitiveType[][] {null, vector}, + new TsPrimitiveType.TsVector(vector) + } + }; + + for (Object[] testCase : testCases) { + TsPrimitiveType expected = (TsPrimitiveType) testCase[2]; + TsPrimitiveType actual = + Type.fromTsDataType((TSDataType) testCase[0]).getValueAsTsPrimitiveType(testCase[1], 1); + Assert.assertEquals(expected, actual); + Assert.assertEquals(expected.getDataType(), actual.getDataType()); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN) + .getValueAsTsPrimitiveType(new Object[] {new Object()}, 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testToBytes() { + byte[] valueBytes = new byte[16]; + + Type.fromTsDataType(TSDataType.BOOLEAN) + .toBytes(new TsPrimitiveType.TsBoolean(true), valueBytes, OFFSET); + Assert.assertTrue(BytesUtils.bytesToBool(valueBytes, OFFSET)); + + Type.fromTsDataType(TSDataType.DATE) + .toBytes(new TsPrimitiveType.TsInt(20260713, TSDataType.DATE), valueBytes, OFFSET); + Assert.assertEquals(20260713, BytesUtils.bytesToInt(valueBytes, OFFSET)); + + Type.fromTsDataType(TSDataType.TIMESTAMP) + .toBytes(new TsPrimitiveType.TsLong(123456789L), valueBytes, OFFSET); + Assert.assertEquals( + 123456789L, BytesUtils.bytesToLongFromOffset(valueBytes, Long.BYTES, OFFSET)); + + Type.fromTsDataType(TSDataType.FLOAT) + .toBytes(new TsPrimitiveType.TsFloat(1.25F), valueBytes, OFFSET); + Assert.assertEquals(1.25F, BytesUtils.bytesToFloat(valueBytes, OFFSET), 0); + + Type.fromTsDataType(TSDataType.DOUBLE) + .toBytes(new TsPrimitiveType.TsDouble(2.5D), valueBytes, OFFSET); + Assert.assertEquals(2.5D, BytesUtils.bytesToDouble(valueBytes, OFFSET), 0); + + Binary binary = new Binary("test", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType) + .toBytes(new TsPrimitiveType.TsBinary(binary), valueBytes, OFFSET); + Assert.assertEquals(binary.getLength(), BytesUtils.bytesToInt(valueBytes, OFFSET)); + Assert.assertArrayEquals( + binary.getValues(), + Arrays.copyOfRange( + valueBytes, OFFSET + Integer.BYTES, OFFSET + Integer.BYTES + binary.getLength())); + } + } + + @Test + public void testCalcTypeSize() { + Assert.assertEquals( + Byte.BYTES, + Type.fromTsDataType(TSDataType.BOOLEAN).calcTypeSize(new TsPrimitiveType.TsBoolean(true))); + Assert.assertEquals( + Integer.BYTES, + Type.fromTsDataType(TSDataType.INT32).calcTypeSize(new TsPrimitiveType.TsInt(1))); + Assert.assertEquals( + Integer.BYTES, + Type.fromTsDataType(TSDataType.DATE) + .calcTypeSize(new TsPrimitiveType.TsInt(20260713, TSDataType.DATE))); + Assert.assertEquals( + Long.BYTES, + Type.fromTsDataType(TSDataType.INT64).calcTypeSize(new TsPrimitiveType.TsLong(1L))); + Assert.assertEquals( + Long.BYTES, + Type.fromTsDataType(TSDataType.TIMESTAMP).calcTypeSize(new TsPrimitiveType.TsLong(1L))); + Assert.assertEquals( + Float.BYTES, + Type.fromTsDataType(TSDataType.FLOAT).calcTypeSize(new TsPrimitiveType.TsFloat(1.0F))); + Assert.assertEquals( + Double.BYTES, + Type.fromTsDataType(TSDataType.DOUBLE).calcTypeSize(new TsPrimitiveType.TsDouble(1.0D))); + + Binary binary = new Binary("test", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Assert.assertEquals( + Integer.BYTES + binary.getLength(), + Type.fromTsDataType(dataType).calcTypeSize(new TsPrimitiveType.TsBinary(binary))); + } + } + + @Test + public void testSerializeValue() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260716}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object value = testCase[1]; + ByteBuffer buffer = ByteBuffer.allocate(64); + int bufferSerializedSize = type.serializeValue(value, buffer); + Assert.assertEquals(buffer.position(), bufferSerializedSize); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + int serializedSize = + type.serialize(type.getTsPrimitiveType(value), new DataOutputStream(output)); + byte[] expected = output.toByteArray(); + Assert.assertEquals(expected.length, serializedSize); + Assert.assertArrayEquals(expected, Arrays.copyOf(buffer.array(), buffer.position())); + + output.reset(); + int streamSerializedSize = type.serializeValue(value, new DataOutputStream(output)); + Assert.assertEquals(output.size(), streamSerializedSize); + Assert.assertArrayEquals(expected, output.toByteArray()); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).serializeValue(null, ByteBuffer.allocate(0)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + + try { + Type.fromTsDataType(dataType) + .serializeValue(null, new DataOutputStream(new ByteArrayOutputStream())); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testSerializeBatchDataReturnsSerializedSize() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260722}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary}, + { + TSDataType.VECTOR, + new TsPrimitiveType[] { + null, new TsPrimitiveType.TsInt(1), new TsPrimitiveType.TsBinary(binary) + } + } + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + BatchData batchData = new BatchData((TSDataType) testCase[0]); + batchData.putAnObject(1L, testCase[1]); + batchData.putAnObject(2L, testCase[1]); + + for (boolean isDesc : new boolean[] {false, true}) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + int serializedSize = type.serialize(batchData, new DataOutputStream(output), isDesc); + Assert.assertEquals(output.size(), serializedSize); + } + } + } + + @Test + public void testDeserializeValue() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 20260716}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 3L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, binary}, + {TSDataType.STRING, binary}, + {TSDataType.BLOB, binary}, + {TSDataType.OBJECT, binary} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + type.serializeValue(testCase[1], new DataOutputStream(output)); + Object[] values = new Object[2]; + type.deserialize(values, 1, ByteBuffer.wrap(output.toByteArray())); + Assert.assertEquals(testCase[1], values[1]); + + values[1] = null; + type.deserialize(values, 1, new ByteArrayInputStream(output.toByteArray())); + Assert.assertEquals(testCase[1], values[1]); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).deserialize(new Object[1], 0, ByteBuffer.allocate(0)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + + try { + Type.fromTsDataType(dataType) + .deserialize(new Object[1], 0, new ByteArrayInputStream(new byte[0])); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testSerializeArrayToByteBuffer() throws IOException { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new boolean[] {true, false}}, + {TSDataType.INT32, new int[] {1, 2}}, + {TSDataType.DATE, new int[] {20260716, 20260717}}, + {TSDataType.INT64, new long[] {3L, 4L}}, + {TSDataType.TIMESTAMP, new long[] {5L, 6L}}, + {TSDataType.FLOAT, new float[] {1.25F, 2.5F}}, + {TSDataType.DOUBLE, new double[] {2.5D, 5.0D}}, + {TSDataType.TEXT, new Binary[] {binary, null}}, + {TSDataType.STRING, new Binary[] {binary, null}}, + {TSDataType.BLOB, new Binary[] {binary, null}}, + {TSDataType.OBJECT, new Binary[] {binary, null}} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object array = testCase[1]; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + type.serializeArray(array, 2, new DataOutputStream(output)); + + ByteBuffer buffer = ByteBuffer.allocate(type.serializedSize(array, 2)); + type.serializeArray(array, 2, buffer); + Assert.assertArrayEquals( + output.toByteArray(), Arrays.copyOf(buffer.array(), buffer.position())); + } + + LocalDate[] dates = {LocalDate.of(2026, 7, 16), null}; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Type dateType = Type.fromTsDataType(TSDataType.DATE); + dateType.serializeArray(dates, dates.length, new DataOutputStream(output)); + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * dates.length); + dateType.serializeArray(dates, dates.length, buffer); + Assert.assertArrayEquals(output.toByteArray(), buffer.array()); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).serializeArray(new Object[0], 0, ByteBuffer.allocate(0)); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testSerializedSizeRange() { + Assert.assertEquals( + Integer.BYTES * 2, + Type.fromTsDataType(TSDataType.INT32).serializedSize(new int[] {1, 2, 3}, 1, 3)); + + Binary[] binaries = { + new Binary("excluded", StandardCharsets.UTF_8), null, new Binary("x", StandardCharsets.UTF_8) + }; + int expectedSize = Byte.BYTES + Byte.BYTES + Integer.BYTES + 1; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type type = Type.fromTsDataType(dataType); + Assert.assertEquals(expectedSize, type.serializedSize(binaries, 1, 3)); + Assert.assertEquals(0, type.serializedSize(binaries, 2, 2)); + } + } + + @Test + public void testGetDataPoint() { + Object[][] testCases = { + {TSDataType.BOOLEAN, "true", true}, + {TSDataType.INT32, "1", 1}, + {TSDataType.DATE, "2026-07-15", 20260715}, + {TSDataType.INT64, "2", 2L}, + {TSDataType.TIMESTAMP, "3", 3L}, + {TSDataType.FLOAT, "1.25", 1.25F}, + {TSDataType.DOUBLE, "2.5", 2.5D}, + {TSDataType.TEXT, "text", new Binary("text", TSFileConfig.STRING_CHARSET)}, + {TSDataType.STRING, "string", new Binary("string", TSFileConfig.STRING_CHARSET)}, + {TSDataType.BLOB, "blob", new Binary("blob", TSFileConfig.STRING_CHARSET)}, + {TSDataType.OBJECT, "object", new Binary("object", TSFileConfig.STRING_CHARSET)} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + String value = (String) testCase[1]; + Object expectedValue = testCase[2]; + + DataPoint typeDataPoint = Type.fromTsDataType(dataType).getDataPoint("s", value); + Assert.assertEquals("s", typeDataPoint.getMeasurementId()); + Assert.assertEquals(expectedValue, typeDataPoint.getValue()); + + DataPoint factoryDataPoint = DataPoint.getDataPoint(dataType, "s", value); + Assert.assertEquals(typeDataPoint.getClass(), factoryDataPoint.getClass()); + Assert.assertEquals(expectedValue, factoryDataPoint.getValue()); + } + + try { + DataPoint.getDataPoint(TSDataType.INT32, "s", "invalid"); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + + try { + DataPoint.getDataPoint(TSDataType.VECTOR, "s", "1"); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + @Test + public void testGetDataPointFromLong() { + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 1}, + {TSDataType.INT64, 1L}, + {TSDataType.TIMESTAMP, 1L}, + {TSDataType.FLOAT, 1.0F}, + {TSDataType.DOUBLE, 1.0D}, + {TSDataType.TEXT, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.STRING, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.BLOB, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.OBJECT, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.VECTOR, new Binary("1", TSFileConfig.STRING_CHARSET)}, + {TSDataType.UNKNOWN, new Binary("1", TSFileConfig.STRING_CHARSET)} + }; + + for (Object[] testCase : testCases) { + DataPoint dataPoint = Type.fromTsDataType((TSDataType) testCase[0]).getDataPoint("s", 1L); + Assert.assertEquals("s", dataPoint.getMeasurementId()); + Assert.assertEquals(testCase[1], dataPoint.getValue()); + } + } + + @Test + public void testWriteValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, true, false); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, 1, false); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DATE).write(writer, 3L, null, true); + Mockito.verify(writer).write(3L, 0, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 4L, 2L, false); + Mockito.verify(writer).write(4L, 2L, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 5L, 1.25F, false); + Mockito.verify(writer).write(5L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 6L, 2.5D, false); + Mockito.verify(writer).write(6L, 2.5D, false); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 7L, binary, false); + Mockito.verify(writer).write(7L, binary, false); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 8L, null, true); + Mockito.verify(writer).write(8L, false, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT64).write(writer, 9L, null, true); + Mockito.verify(writer).write(9L, 0L, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 10L, null, true); + Mockito.verify(writer).write(10L, 0.0F, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 11L, null, true); + Mockito.verify(writer).write(11L, 0.0D, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TEXT).write(writer, 12L, null, true); + Mockito.verify(writer).write(12L, Binary.EMPTY_VALUE, true); + Mockito.reset(writer); + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 13L, null, true); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + @Test + public void testWriteTsPrimitiveValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, new TsPrimitiveType.TsBoolean(true)); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, new TsPrimitiveType.TsInt(1)); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 3L, new TsPrimitiveType.TsLong(2L)); + Mockito.verify(writer).write(3L, 2L, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 4L, new TsPrimitiveType.TsFloat(1.25F)); + Mockito.verify(writer).write(4L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 5L, new TsPrimitiveType.TsDouble(2.5D)); + Mockito.verify(writer).write(5L, 2.5D, false); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 6L, new TsPrimitiveType.TsBinary(binary)); + Mockito.verify(writer).write(6L, binary, false); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.INT32).write(writer, 7L, (TsPrimitiveType) null); + Mockito.verify(writer).write(7L, Integer.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 8L, (TsPrimitiveType) null); + Mockito.verify(writer).write(8L, Long.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 9L, (TsPrimitiveType) null); + Mockito.verify(writer).write(9L, Float.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 10L, (TsPrimitiveType) null); + Mockito.verify(writer).write(10L, Double.MAX_VALUE, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 11L, (TsPrimitiveType) null); + Mockito.verify(writer).write(11L, false, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TEXT).write(writer, 12L, (TsPrimitiveType) null); + Mockito.verify(writer).write(12L, Binary.EMPTY_VALUE, true); + Mockito.reset(writer); + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 13L, (TsPrimitiveType) null); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + @Test + public void testWriteTabletValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + + Type.fromTsDataType(TSDataType.BOOLEAN) + .write(writer, 1L, new boolean[] {false, true}, 1, false); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, new int[] {0, 1}, 1, false); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DATE) + .write( + writer, + 3L, + new LocalDate[] {LocalDate.of(2026, 7, 14), LocalDate.of(2026, 7, 15)}, + 1, + false); + Mockito.verify(writer).write(3L, 20260715, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 4L, new long[] {0L, 2L}, 1, false); + Mockito.verify(writer).write(4L, 2L, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 5L, new float[] {0F, 1.25F}, 1, false); + Mockito.verify(writer).write(5L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 6L, new double[] {0D, 2.5D}, 1, false); + Mockito.verify(writer).write(6L, 2.5D, false); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Binary[] binaries = {Binary.EMPTY_VALUE, binary}; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 7L, binaries, 1, false); + Mockito.verify(writer).write(7L, binary, false); + Mockito.reset(writer); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 8L, new long[] {1L}, 0, false); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + @Test + public void testWriteTsBlockValueChunk() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + Column column = Mockito.mock(Column.class); + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Mockito.when(column.getBoolean(1)).thenReturn(true); + Mockito.when(column.getInt(1)).thenReturn(1); + Mockito.when(column.getLong(1)).thenReturn(2L); + Mockito.when(column.getFloat(1)).thenReturn(1.25F); + Mockito.when(column.getDouble(1)).thenReturn(2.5D); + Mockito.when(column.getBinary(1)).thenReturn(binary); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, column, 1, false); + Mockito.verify(writer).write(1L, true, false); + Mockito.reset(writer); + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT32, TSDataType.DATE}) { + Type.fromTsDataType(dataType).write(writer, 2L, column, 1, false); + Mockito.verify(writer).write(2L, 1, false); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT64, TSDataType.TIMESTAMP}) { + Type.fromTsDataType(dataType).write(writer, 3L, column, 1, false); + Mockito.verify(writer).write(3L, 2L, false); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 4L, column, 1, false); + Mockito.verify(writer).write(4L, 1.25F, false); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 5L, column, 1, false); + Mockito.verify(writer).write(5L, 2.5D, false); + Mockito.reset(writer); + + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB}) { + Type.fromTsDataType(dataType).write(writer, 6L, column, 1, false); + Mockito.verify(writer).write(6L, binary, false); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.INT32).write(writer, 7L, column, 1, true); + Mockito.verify(writer).write(7L, 0, true); + Mockito.reset(writer); + + for (TSDataType dataType : new TSDataType[] {TSDataType.OBJECT, TSDataType.VECTOR}) { + try { + Type.fromTsDataType(dataType).write(writer, 8L, column, 1, false); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + + @Test + public void testWriteTsBlockValueChunkBatch() { + ValueChunkWriter writer = Mockito.mock(ValueChunkWriter.class); + Column column = Mockito.mock(Column.class); + long[] times = {1L, 2L}; + boolean[] isNull = {false, true}; + boolean[] booleans = {true, false}; + int[] ints = {1, 0}; + long[] longs = {2L, 0L}; + float[] floats = {1.25F, 0F}; + double[] doubles = {2.5D, 0D}; + Binary[] binaries = {new Binary("value", TSFileConfig.STRING_CHARSET), Binary.EMPTY_VALUE}; + Mockito.when(column.isNull()).thenReturn(isNull); + Mockito.when(column.getBooleans()).thenReturn(booleans); + Mockito.when(column.getInts()).thenReturn(ints); + Mockito.when(column.getLongs()).thenReturn(longs); + Mockito.when(column.getFloats()).thenReturn(floats); + Mockito.when(column.getDoubles()).thenReturn(doubles); + Mockito.when(column.getBinaries()).thenReturn(binaries); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, booleans, isNull, 1, 1); + Mockito.reset(writer); + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT32, TSDataType.DATE}) { + Type.fromTsDataType(dataType).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, ints, isNull, 1, 1); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT64, TSDataType.TIMESTAMP}) { + Type.fromTsDataType(dataType).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, longs, isNull, 1, 1); + Mockito.reset(writer); + } + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, floats, isNull, 1, 1); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, doubles, isNull, 1, 1); + Mockito.reset(writer); + + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, times, column, 1, 1); + Mockito.verify(writer).write(times, binaries, isNull, 1, 1); + Mockito.reset(writer); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, times, column, 1, 1); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testWriteNonAlignedTabletValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Type.fromTsDataType(TSDataType.BOOLEAN).write(writer, 1L, new boolean[] {false, true}, 1); + Mockito.verify(writer).write(1L, true); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.INT32).write(writer, 2L, new int[] {0, 1}, 1); + Mockito.verify(writer).write(2L, 1); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DATE) + .write( + writer, 3L, new LocalDate[] {LocalDate.of(2026, 7, 14), LocalDate.of(2026, 7, 15)}, 1); + Mockito.verify(writer).write(3L, 20260715); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.TIMESTAMP).write(writer, 4L, new long[] {0L, 2L}, 1); + Mockito.verify(writer).write(4L, 2L); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.FLOAT).write(writer, 5L, new float[] {0F, 1.25F}, 1); + Mockito.verify(writer).write(5L, 1.25F); + Mockito.reset(writer); + + Type.fromTsDataType(TSDataType.DOUBLE).write(writer, 6L, new double[] {0D, 2.5D}, 1); + Mockito.verify(writer).write(6L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Binary[] binaries = {Binary.EMPTY_VALUE, binary}; + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 7L, binaries, 1); + Mockito.verify(writer).write(7L, binary); + Mockito.reset(writer); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).write(writer, 8L, new long[] {1L}, 0); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + @Test + public void testWriteNonAlignedValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1L, true}, + {TSDataType.INT32, 2L, 1}, + {TSDataType.DATE, 3L, 20260717}, + {TSDataType.INT64, 4L, 2L}, + {TSDataType.TIMESTAMP, 5L, 3L}, + {TSDataType.FLOAT, 6L, 1.25F}, + {TSDataType.DOUBLE, 7L, 2.5D} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]).write(writer, (long) testCase[1], testCase[2]); + } + + Mockito.verify(writer).write(1L, true); + Mockito.verify(writer).write(2L, 1); + Mockito.verify(writer).write(3L, 20260717); + Mockito.verify(writer).write(4L, 2L); + Mockito.verify(writer).write(5L, 3L); + Mockito.verify(writer).write(6L, 1.25F); + Mockito.verify(writer).write(7L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 8L, binary); + Mockito.verify(writer).write(8L, binary); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(writer, 9L, 1L); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + + @Test + public void testWriteTsPrimitiveTypeToNonAlignedValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1L, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, 2L, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, 3L, new TsPrimitiveType.TsInt(20260717, TSDataType.DATE)}, + {TSDataType.INT64, 4L, new TsPrimitiveType.TsLong(2L)}, + {TSDataType.TIMESTAMP, 5L, new TsPrimitiveType.TsLong(3L)}, + {TSDataType.FLOAT, 6L, new TsPrimitiveType.TsFloat(1.25F)}, + {TSDataType.DOUBLE, 7L, new TsPrimitiveType.TsDouble(2.5D)} + }; + for (Object[] testCase : testCases) { + Type.fromTsDataType((TSDataType) testCase[0]) + .write(writer, (long) testCase[1], (TsPrimitiveType) testCase[2]); + } + + Mockito.verify(writer).write(1L, true); + Mockito.verify(writer).write(2L, 1); + Mockito.verify(writer).write(3L, 20260717); + Mockito.verify(writer).write(4L, 2L); + Mockito.verify(writer).write(5L, 3L); + Mockito.verify(writer).write(6L, 1.25F); + Mockito.verify(writer).write(7L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + TsPrimitiveType binaryValue = new TsPrimitiveType.TsBinary(binary); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Type.fromTsDataType(dataType).write(writer, 8L, binaryValue); + Mockito.verify(writer).write(8L, binary); + Mockito.reset(writer); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(writer, 9L, new TsPrimitiveType.TsLong(1L)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + + @Test + public void testWriteBatchDataToNonAlignedValueChunk() { + ChunkWriterImpl writer = Mockito.mock(ChunkWriterImpl.class); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1L, true}, + {TSDataType.INT32, 2L, 1}, + {TSDataType.DATE, 3L, 20260717}, + {TSDataType.INT64, 4L, 2L}, + {TSDataType.TIMESTAMP, 5L, 3L}, + {TSDataType.FLOAT, 6L, 1.25F}, + {TSDataType.DOUBLE, 7L, 2.5D} + }; + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + BatchData data = new BatchData(dataType); + data.putAnObject(0L, testCase[2]); + Type.fromTsDataType(dataType).write(writer, (long) testCase[1], data); + } + + Mockito.verify(writer).write(1L, true); + Mockito.verify(writer).write(2L, 1); + Mockito.verify(writer).write(3L, 20260717); + Mockito.verify(writer).write(4L, 2L); + Mockito.verify(writer).write(5L, 3L); + Mockito.verify(writer).write(6L, 1.25F); + Mockito.verify(writer).write(7L, 2.5D); + Mockito.reset(writer); + + Binary binary = new Binary("value", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + BatchData data = new BatchData(dataType); + data.putAnObject(0L, binary); + Type.fromTsDataType(dataType).write(writer, 8L, data); + Mockito.verify(writer).write(8L, binary); + Mockito.reset(writer); + } + + BatchData vectorData = new BatchData(TSDataType.VECTOR); + vectorData.putAnObject(0L, new TsPrimitiveType[] {new TsPrimitiveType.TsLong(1L)}); + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).write(writer, 9L, vectorData); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + + @Test + public void testToString() { + Object[][] testCases = { + {TSDataType.BOOLEAN, true, "true"}, + {TSDataType.INT32, 1, "1"}, + {TSDataType.DATE, 20260714, "20260714"}, + {TSDataType.INT64, 2L, "2"}, + {TSDataType.TIMESTAMP, 3L, "3"}, + {TSDataType.FLOAT, 1.25F, "1.25"}, + {TSDataType.DOUBLE, 2.5D, "2.5"}, + {TSDataType.TEXT, new Binary("text", StandardCharsets.UTF_8), "text"}, + {TSDataType.STRING, new Binary("string", StandardCharsets.UTF_8), "string"}, + {TSDataType.BLOB, new Binary(new byte[] {0x01, 0x23, (byte) 0xFF}), "0x0123ff"}, + {TSDataType.OBJECT, new Binary(BytesUtils.longToBytes(1L)), "(Object) 1 B"} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Field field = Field.getField(testCase[1], dataType); + Assert.assertEquals(testCase[2], Type.fromTsDataType(dataType).toString(field)); + Assert.assertEquals(testCase[2], field.getStringValue()); + } + + Assert.assertEquals("null", new Field(null).getStringValue()); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + new Field(dataType).getStringValue(); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + + @Test + public void testGetValueFromField() { + Object[][] testCases = { + {TSDataType.BOOLEAN, true, true}, + {TSDataType.INT32, 1, 1}, + {TSDataType.DATE, 20260714, LocalDate.of(2026, 7, 14)}, + {TSDataType.INT64, 2L, 2L}, + {TSDataType.TIMESTAMP, 3L, 3L}, + {TSDataType.FLOAT, 1.25F, 1.25F}, + {TSDataType.DOUBLE, 2.5D, 2.5D}, + { + TSDataType.TEXT, + new Binary("text", StandardCharsets.UTF_8), + new Binary("text", StandardCharsets.UTF_8) + }, + { + TSDataType.STRING, + new Binary("string", StandardCharsets.UTF_8), + new Binary("string", StandardCharsets.UTF_8) + }, + {TSDataType.BLOB, new Binary(new byte[] {0x01, 0x23}), new Binary(new byte[] {0x01, 0x23})}, + {TSDataType.OBJECT, new Binary(BytesUtils.longToBytes(1L)), "(Object) 1 B"} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Type type = Type.fromTsDataType(dataType); + Field typeField = type.getField(testCase[1]); + Assert.assertEquals(dataType, typeField.getDataType()); + Assert.assertEquals(testCase[2], type.getValue(typeField)); + + Field field = Field.getField(testCase[1], dataType); + Assert.assertEquals(dataType, field.getDataType()); + Assert.assertEquals(testCase[2], type.getValue(field)); + Assert.assertEquals(testCase[2], field.getObjectValue(dataType)); + } + + Assert.assertNull(Field.getField(null, TSDataType.INT32)); + Assert.assertNull(new Field(null).getObjectValue(TSDataType.INT32)); + Assert.assertEquals("1", Field.getField(1, TSDataType.INT32).getObjectValue(TSDataType.OBJECT)); + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + new Field(dataType).getObjectValue(dataType); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + + try { + Type.fromTsDataType(dataType).getField(new Object()); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + + @Test + public void testSetToField() { + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true), true}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1), 1}, + { + TSDataType.DATE, + new TsPrimitiveType.TsInt(20260714, TSDataType.DATE), + LocalDate.of(2026, 7, 14) + }, + {TSDataType.INT64, new TsPrimitiveType.TsLong(2L), 2L}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(3L), 3L}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.25F), 1.25F}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(2.5D), 2.5D}, + { + TSDataType.TEXT, + new TsPrimitiveType.TsBinary(new Binary("text", StandardCharsets.UTF_8)), + new Binary("text", StandardCharsets.UTF_8) + }, + { + TSDataType.STRING, + new TsPrimitiveType.TsBinary(new Binary("string", StandardCharsets.UTF_8)), + new Binary("string", StandardCharsets.UTF_8) + }, + { + TSDataType.BLOB, + new TsPrimitiveType.TsBinary(new Binary(new byte[] {0x01, 0x23})), + new Binary(new byte[] {0x01, 0x23}) + }, + { + TSDataType.OBJECT, + new TsPrimitiveType.TsBinary(new Binary(BytesUtils.longToBytes(1L))), + "(Object) 1 B" + } + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + TsPrimitiveType from = (TsPrimitiveType) testCase[1]; + Type type = Type.fromTsDataType(dataType); + + Field directTarget = new Field(dataType); + type.setTo(from, directTarget); + Assert.assertEquals(testCase[2], type.getValue(directTarget)); + + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(1L, from.getValue()); + Field batchTarget = new Field(dataType); + type.setTo(batchData, batchTarget); + Assert.assertEquals(testCase[2], type.getValue(batchTarget)); + + if (from.getDataType() == dataType) { + Field factoryTarget = new Field(dataType); + Field.setTsPrimitiveValue(from, factoryTarget); + Assert.assertEquals(testCase[2], type.getValue(factoryTarget)); + } + } + + TsPrimitiveType vector = new TsPrimitiveType.TsVector(new TsPrimitiveType[0]); + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).setTo(vector, new Field(dataType)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + + try { + Field.setTsPrimitiveValue(vector, new Field(TSDataType.VECTOR)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + + BatchData vectorBatchData = new BatchData(TSDataType.VECTOR); + vectorBatchData.putVector(1L, new TsPrimitiveType[] {new TsPrimitiveType.TsInt(1)}); + Field vectorTarget = new Field(TSDataType.INT32); + Type.fromTsDataType(TSDataType.VECTOR).setTo(vectorBatchData, vectorTarget); + Assert.assertEquals(1, vectorTarget.getIntV()); + } + + @Test + public void testSetToArray() { + Column column = Mockito.mock(Column.class); + Binary binary = new Binary("value", TSFileConfig.STRING_CHARSET); + Mockito.when(column.getBoolean(1)).thenReturn(true); + Mockito.when(column.getInt(1)).thenReturn(1); + Mockito.when(column.getInt(2)).thenReturn(20260715); + Mockito.when(column.getLong(1)).thenReturn(2L); + Mockito.when(column.getFloat(1)).thenReturn(1.25F); + Mockito.when(column.getDouble(1)).thenReturn(2.5D); + Mockito.when(column.getBinary(1)).thenReturn(binary); + + Object[][] testCases = { + {TSDataType.BOOLEAN, 1, true}, + {TSDataType.INT32, 1, 1}, + {TSDataType.DATE, 2, LocalDate.of(2026, 7, 15)}, + {TSDataType.INT64, 1, 2L}, + {TSDataType.TIMESTAMP, 1, 2L}, + {TSDataType.FLOAT, 1, 1.25F}, + {TSDataType.DOUBLE, 1, 2.5D}, + {TSDataType.TEXT, 1, binary}, + {TSDataType.STRING, 1, binary}, + {TSDataType.BLOB, 1, binary}, + {TSDataType.OBJECT, 1, binary}, + {TSDataType.VECTOR, 1, 2L} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object target = type.createArray(1); + type.setTo(column, (int) testCase[1], target, 0); + Assert.assertEquals(testCase[2], Array.get(target, 0)); + } + + Object[][] columnTestCases = { + {TSDataType.BOOLEAN, 1, true, new boolean[1]}, + {TSDataType.INT32, 1, 1, new int[1]}, + {TSDataType.DATE, 2, 20260715, new int[1]}, + {TSDataType.INT64, 1, 2L, new long[1]}, + {TSDataType.TIMESTAMP, 1, 2L, new long[1]}, + {TSDataType.FLOAT, 1, 1.25F, new float[1]}, + {TSDataType.DOUBLE, 1, 2.5D, new double[1]}, + {TSDataType.TEXT, 1, binary, new Binary[1]}, + {TSDataType.STRING, 1, binary, new Binary[1]}, + {TSDataType.BLOB, 1, binary, new Binary[1]}, + {TSDataType.OBJECT, 1, binary, new Binary[1]} + }; + + for (Object[] testCase : columnTestCases) { + Column targetColumn = Mockito.mock(Column.class); + Object target = testCase[3]; + if (target instanceof boolean[]) { + Mockito.when(targetColumn.getBooleans()).thenReturn((boolean[]) target); + } else if (target instanceof int[]) { + Mockito.when(targetColumn.getInts()).thenReturn((int[]) target); + } else if (target instanceof long[]) { + Mockito.when(targetColumn.getLongs()).thenReturn((long[]) target); + } else if (target instanceof float[]) { + Mockito.when(targetColumn.getFloats()).thenReturn((float[]) target); + } else if (target instanceof double[]) { + Mockito.when(targetColumn.getDoubles()).thenReturn((double[]) target); + } else { + Mockito.when(targetColumn.getBinaries()).thenReturn((Binary[]) target); + } + Type.fromTsDataType((TSDataType) testCase[0]) + .setTo(column, (int) testCase[1], targetColumn, 0); + Assert.assertEquals(testCase[2], Array.get(target, 0)); + } + + try { + Type.fromTsDataType(TSDataType.VECTOR).setTo(column, 1, column, 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).setTo(column, 1, new Object[1], 0); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testSetNullToBinaryColumnRange() { + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Binary[] values = {Binary.EMPTY_VALUE, Binary.EMPTY_VALUE}; + Column column = Mockito.mock(Column.class); + Mockito.when(column.getBinaries()).thenReturn(values); + + Type.fromTsDataType(dataType).setTo(null, column, 0, values.length); + + Assert.assertArrayEquals(new Binary[] {null, null}, values); + } + } + + @Test + public void testDateAddValueWithIntArray() throws IOException { + Type dateType = Type.fromTsDataType(TSDataType.DATE); + int[] values = new int[2]; + + dateType.addValue(0, LocalDate.of(2026, 7, 14), values); + dateType.addValue(1, null, values); + + Assert.assertArrayEquals(new int[] {20260714, DateUtils.EMPTY_DATE_INT}, values); + + dateType.addValue(0, 20260715, values); + Assert.assertEquals(20260715, values[0]); + + Column column = Mockito.mock(Column.class); + Mockito.when(column.getInt(0)).thenReturn(20260715); + dateType.setTo(column, 0, values, 0); + Assert.assertEquals(20260715, values[0]); + + LocalDate[] localDates = new LocalDate[2]; + dateType.copyArrayElement(values, 0, localDates, 0); + dateType.copyArrayElement(localDates, 0, values, 0); + Assert.assertEquals(LocalDate.of(2026, 7, 15), localDates[0]); + Assert.assertEquals(20260715, values[0]); + Assert.assertTrue( + dateType.arrayEquals( + values, new LocalDate[] {LocalDate.of(2026, 7, 15), LocalDate.of(1000, 1, 1)}, 2)); + + Object copied = dateType.arrayCopyOf(values, 3); + Assert.assertTrue(copied instanceof int[]); + Assert.assertArrayEquals(new int[] {20260715, DateUtils.EMPTY_DATE_INT, 0}, (int[]) copied); + Assert.assertEquals(RamUsageEstimator.sizeOf(values), dateType.estimateArraySize(values)); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + dateType.serializeArray(values, values.length, new DataOutputStream(output)); + ByteBuffer buffer = ByteBuffer.wrap(output.toByteArray()); + Assert.assertEquals(20260715, buffer.getInt()); + Assert.assertEquals(DateUtils.EMPTY_DATE_INT, buffer.getInt()); + } + + @Test + public void testAddPoint() { + ResultSet resultSet = Mockito.mock(ResultSet.class); + IDeviceID deviceID = Mockito.mock(IDeviceID.class); + Mockito.when(resultSet.getBoolean("value")).thenReturn(true); + Mockito.when(resultSet.getInt("value")).thenReturn(1); + Mockito.when(resultSet.getLong("value")).thenReturn(2L); + Mockito.when(resultSet.getFloat("value")).thenReturn(1.25F); + Mockito.when(resultSet.getDouble("value")).thenReturn(2.5D); + Mockito.when(resultSet.getString("value")).thenReturn("text"); + Mockito.when(resultSet.getBinary("value")).thenReturn(new byte[] {0x01, 0x23}); + + Object[][] testCases = { + {TSDataType.BOOLEAN, true}, + {TSDataType.INT32, 1}, + {TSDataType.DATE, 1}, + {TSDataType.INT64, 2L}, + {TSDataType.TIMESTAMP, 2L}, + {TSDataType.FLOAT, 1.25F}, + {TSDataType.DOUBLE, 2.5D}, + {TSDataType.TEXT, new Binary("text", StandardCharsets.UTF_8)}, + {TSDataType.STRING, new Binary("text", StandardCharsets.UTF_8)}, + {TSDataType.BLOB, new Binary(new byte[] {0x01, 0x23})}, + {TSDataType.OBJECT, new Binary(new byte[] {0x01, 0x23})} + }; + + for (Object[] testCase : testCases) { + TSRecord record = new TSRecord(deviceID, 1L); + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + type.addPoint(record, "value", resultSet); + Assert.assertEquals(1, record.dataPointList.size()); + Assert.assertEquals(testCase[1], record.dataPointList.get(0).getValue()); + + TSRecord fieldRecord = new TSRecord(deviceID, 1L); + type.addPoint(fieldRecord, "value", type.getField(testCase[1])); + Assert.assertEquals(1, fieldRecord.dataPointList.size()); + Assert.assertEquals(testCase[1], fieldRecord.dataPointList.get(0).getValue()); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + TSRecord record = new TSRecord(deviceID, 1L); + try { + Type.fromTsDataType(dataType).addPoint(record, "value", resultSet); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + + try { + Type.fromTsDataType(dataType).addPoint(record, "value", new Field(dataType)); + Assert.fail("Expected UnSupportedDataTypeException"); + } catch (UnSupportedDataTypeException ignored) { + // Expected. + } + } + } + + @Test + public void testCalcTypeSizeFromObject() { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, true, Byte.BYTES}, + {TSDataType.INT32, 1, Integer.BYTES}, + {TSDataType.DATE, 20260713, Integer.BYTES}, + {TSDataType.INT64, 1L, Long.BYTES}, + {TSDataType.TIMESTAMP, 1L, Long.BYTES}, + {TSDataType.FLOAT, 1.0F, Float.BYTES}, + {TSDataType.DOUBLE, 1.0D, Double.BYTES}, + {TSDataType.TEXT, binary, Integer.BYTES + binary.getLength()}, + {TSDataType.STRING, binary, Integer.BYTES + binary.getLength()}, + {TSDataType.BLOB, binary, Integer.BYTES + binary.getLength()}, + {TSDataType.OBJECT, binary, Integer.BYTES + binary.getLength()} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + Object value = testCase[1]; + int expectedSize = (int) testCase[2]; + Assert.assertEquals(expectedSize, Type.fromTsDataType(dataType).calcTypeSize(value)); + } + } + + @Test + public void testDeserializeColumn() { + boolean[] nullIndicators = {true, false}; + + ByteBuffer booleanBuffer = ByteBuffer.wrap(new byte[] {(byte) 0b1000_0000, 0x01}); + Column booleanColumn = + (Column) + Type.fromTsDataType(TSDataType.BOOLEAN) + .deserializeColumn(booleanBuffer, 2, nullIndicators); + Assert.assertTrue(booleanColumn.isNull(0)); + Assert.assertTrue(booleanColumn.getBoolean(1)); + Assert.assertEquals(1, booleanBuffer.position()); + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT32, TSDataType.DATE}) { + Column column = + (Column) + Type.fromTsDataType(dataType) + .deserializeColumn(intBuffer(20260714), 2, nullIndicators); + Assert.assertEquals(dataType, column.getDataType()); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(20260714, column.getInt(1)); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.INT64, TSDataType.TIMESTAMP}) { + Column column = + (Column) + Type.fromTsDataType(dataType) + .deserializeColumn(longBuffer(123456789L), 2, nullIndicators); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(123456789L, column.getLong(1)); + } + + Column floatColumn = + (Column) + Type.fromTsDataType(TSDataType.FLOAT) + .deserializeColumn(intBuffer(Float.floatToIntBits(1.25F)), 2, nullIndicators); + Assert.assertTrue(floatColumn.isNull(0)); + Assert.assertEquals(1.25F, floatColumn.getFloat(1), 0); + + Column doubleColumn = + (Column) + Type.fromTsDataType(TSDataType.DOUBLE) + .deserializeColumn(longBuffer(Double.doubleToLongBits(2.5D)), 2, nullIndicators); + Assert.assertTrue(doubleColumn.isNull(0)); + Assert.assertEquals(2.5D, doubleColumn.getDouble(1), 0); + + Binary binary = new Binary("test", StandardCharsets.UTF_8); + for (TSDataType dataType : + new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB, TSDataType.OBJECT}) { + Column column = + (Column) + Type.fromTsDataType(dataType) + .deserializeColumn(binaryBuffer(binary), 2, nullIndicators); + Assert.assertTrue(column.isNull(0)); + Assert.assertEquals(binary, column.getBinary(1)); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + Type.fromTsDataType(dataType).deserializeColumn(ByteBuffer.allocate(0), 0, null); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testEstimateArraySize() { + int size = 10; + Object[][] testCases = { + {TSDataType.BOOLEAN, RamUsageEstimator.sizeOfBooleanArray(size)}, + {TSDataType.INT32, RamUsageEstimator.sizeOfIntArray(size)}, + {TSDataType.DATE, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.INT64, RamUsageEstimator.sizeOfLongArray(size)}, + {TSDataType.TIMESTAMP, RamUsageEstimator.sizeOfLongArray(size)}, + {TSDataType.FLOAT, RamUsageEstimator.sizeOfFloatArray(size)}, + {TSDataType.DOUBLE, RamUsageEstimator.sizeOfDoubleArray(size)}, + {TSDataType.TEXT, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.STRING, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.BLOB, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.OBJECT, RamUsageEstimator.sizeOfObjectArray(size)}, + {TSDataType.VECTOR, RamUsageEstimator.sizeOfLongArray(size)} + }; + + for (Object[] testCase : testCases) { + Assert.assertEquals( + testCase[1], Type.fromTsDataType((TSDataType) testCase[0]).estimateArraySize(size)); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).estimateArraySize(size); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testEstimateArraySizeWithArray() { + boolean[] booleans = {false, true}; + int[] ints = {0, 1}; + LocalDate[] dates = {null, LocalDate.of(2026, 7, 15)}; + long[] longs = {0L, 1L}; + float[] floats = {0.0F, 1.0F}; + double[] doubles = {0.0D, 1.0D}; + Binary[] binaries = {null, new Binary("test", StandardCharsets.UTF_8)}; + + assertEstimateArraySize(TSDataType.BOOLEAN, booleans, RamUsageEstimator.sizeOf(booleans)); + assertEstimateArraySize(TSDataType.INT32, ints, RamUsageEstimator.sizeOf(ints)); + assertEstimateArraySize(TSDataType.DATE, dates, RamUsageEstimator.sizeOf(dates)); + assertEstimateArraySize(TSDataType.INT64, longs, RamUsageEstimator.sizeOf(longs)); + assertEstimateArraySize(TSDataType.TIMESTAMP, longs, RamUsageEstimator.sizeOf(longs)); + assertEstimateArraySize(TSDataType.FLOAT, floats, RamUsageEstimator.sizeOf(floats)); + assertEstimateArraySize(TSDataType.DOUBLE, doubles, RamUsageEstimator.sizeOf(doubles)); + assertEstimateArraySize(TSDataType.TEXT, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.STRING, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.BLOB, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.OBJECT, binaries, RamUsageEstimator.sizeOf(binaries)); + assertEstimateArraySize(TSDataType.VECTOR, longs, RamUsageEstimator.sizeOf(longs)); + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).estimateArraySize(new Object[0]); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + private void assertEstimateArraySize(TSDataType dataType, Object array, long expected) { + Assert.assertEquals(expected, Type.fromTsDataType(dataType).estimateArraySize(array)); + } + + @Test + public void testCopyArrayElement() { + LocalDate date = LocalDate.of(2026, 7, 15); + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new boolean[] {false, true}}, + {TSDataType.INT32, new int[] {0, 1}}, + {TSDataType.DATE, new LocalDate[] {null, date}}, + {TSDataType.INT64, new long[] {0L, 1L}}, + {TSDataType.TIMESTAMP, new long[] {0L, 1L}}, + {TSDataType.FLOAT, new float[] {0.0F, 1.0F}}, + {TSDataType.DOUBLE, new double[] {0.0D, 1.0D}}, + {TSDataType.TEXT, new Binary[] {null, binary}}, + {TSDataType.STRING, new Binary[] {null, binary}}, + {TSDataType.BLOB, new Binary[] {null, binary}}, + {TSDataType.OBJECT, new Binary[] {null, binary}}, + {TSDataType.VECTOR, new long[] {0L, 1L}} + }; + + for (Object[] testCase : testCases) { + Type type = Type.fromTsDataType((TSDataType) testCase[0]); + Object source = testCase[1]; + Object target = type.createArray(2); + + type.copyArrayElement(source, 1, target, 0); + type.copyArrayElement(source, 0, target, 1); + + Assert.assertEquals(Array.get(source, 1), Array.get(target, 0)); + Assert.assertEquals(Array.get(source, 0), Array.get(target, 1)); + + Object expanded = type.arrayCopyOf(source, 3); + Assert.assertEquals(3, Array.getLength(expanded)); + Assert.assertEquals(Array.get(source, 0), Array.get(expanded, 0)); + Assert.assertEquals(Array.get(source, 1), Array.get(expanded, 1)); + + Object truncated = type.arrayCopyOf(source, 1); + Assert.assertEquals(1, Array.getLength(truncated)); + Assert.assertEquals(Array.get(source, 0), Array.get(truncated, 0)); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).arrayCopyOf(new Object[0], 1); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testEstimateValueSize() { + Object[][] testCases = { + {TSDataType.BOOLEAN, (long) Byte.BYTES}, + {TSDataType.INT32, (long) Integer.BYTES}, + {TSDataType.DATE, (long) Integer.BYTES}, + {TSDataType.INT64, (long) Long.BYTES}, + {TSDataType.TIMESTAMP, (long) Long.BYTES}, + {TSDataType.FLOAT, (long) Float.BYTES}, + {TSDataType.DOUBLE, (long) Double.BYTES}, + {TSDataType.TEXT, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.STRING, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.BLOB, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.OBJECT, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF}, + {TSDataType.VECTOR, (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF} + }; + + for (Object[] testCase : testCases) { + Assert.assertEquals( + testCase[1], Type.fromTsDataType((TSDataType) testCase[0]).estimateValueSize()); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).estimateValueSize(); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + + @Test + public void testGetOneItemMaxSize() { + Object[][] testCases = { + {TSDataType.BOOLEAN, Byte.BYTES}, + {TSDataType.INT32, Integer.BYTES}, + {TSDataType.DATE, Integer.BYTES}, + {TSDataType.INT64, Long.BYTES}, + {TSDataType.TIMESTAMP, Long.BYTES}, + {TSDataType.FLOAT, Float.BYTES}, + {TSDataType.DOUBLE, Double.BYTES}, + {TSDataType.TEXT, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH}, + {TSDataType.STRING, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH}, + {TSDataType.BLOB, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH}, + {TSDataType.OBJECT, Integer.BYTES + TSFileConfig.BYTE_SIZE_PER_CHAR * VALUE_LENGTH} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + int expectedSize = (int) testCase[1]; + Assert.assertEquals( + expectedSize, Type.fromTsDataType(dataType).getOneItemMaxSize(VALUE_LENGTH)); + Assert.assertEquals( + expectedSize, new PlainEncoder(dataType, VALUE_LENGTH).getOneItemMaxSize()); + } + + for (TSDataType dataType : new TSDataType[] {TSDataType.VECTOR, TSDataType.UNKNOWN}) { + try { + new PlainEncoder(dataType, VALUE_LENGTH).getOneItemMaxSize(); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException ignored) { + // Expected. + } + } + } + + @Test + public void testCreateStatistics() { + for (TSDataType dataType : + new TSDataType[] { + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.DATE, + TSDataType.INT64, + TSDataType.TIMESTAMP, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.OBJECT, + TSDataType.VECTOR + }) { + Type type = Type.fromTsDataType(dataType); + Assert.assertEquals(dataType, type.createStatistics().getType()); + Assert.assertEquals(dataType, Statistics.getStatsByType(dataType).getType()); + Assert.assertTrue(type.getStatisticsInstanceSize() > 0); + Assert.assertEquals(type.getStatisticsInstanceSize(), Statistics.getSizeByType(dataType)); + } + + try { + Type.fromTsDataType(TSDataType.UNKNOWN).createStatistics(); + Assert.fail("Expected UnknownColumnTypeException"); + } catch (UnknownColumnTypeException ignored) { + // Expected. + } + + try { + Statistics.getSizeByType(TSDataType.UNKNOWN); + Assert.fail("Expected UnknownColumnTypeException"); + } catch (UnknownColumnTypeException ignored) { + // Expected. + } + } + + @Test + public void testUpdateStatistics() { + Binary binary = new Binary("test", StandardCharsets.UTF_8); + Object[][] testCases = { + {TSDataType.BOOLEAN, new TsPrimitiveType.TsBoolean(true)}, + {TSDataType.INT32, new TsPrimitiveType.TsInt(1)}, + {TSDataType.DATE, new TsPrimitiveType.TsInt(20260714, TSDataType.DATE)}, + {TSDataType.INT64, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.TIMESTAMP, new TsPrimitiveType.TsLong(1L)}, + {TSDataType.FLOAT, new TsPrimitiveType.TsFloat(1.0F)}, + {TSDataType.DOUBLE, new TsPrimitiveType.TsDouble(1.0D)}, + {TSDataType.TEXT, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.STRING, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.BLOB, new TsPrimitiveType.TsBinary(binary)}, + {TSDataType.OBJECT, new TsPrimitiveType.TsBinary(binary)} + }; + + for (Object[] testCase : testCases) { + TSDataType dataType = (TSDataType) testCase[0]; + TsPrimitiveType value = (TsPrimitiveType) testCase[1]; + Type type = Type.fromTsDataType(dataType); + Statistics statistics = Statistics.getStatsByType(dataType); + type.update(statistics, 100L, value); + Assert.assertEquals(1, statistics.getCount()); + Assert.assertEquals(100L, statistics.getStartTime()); + Assert.assertEquals(100L, statistics.getEndTime()); + Assert.assertFalse(statistics.isEmpty()); + + BatchData batchData = new BatchData(dataType); + batchData.putAnObject(200L, value.getValue()); + Assert.assertEquals(value.getValue(), type.getCurrentValue(batchData)); + Assert.assertEquals(value.getValue(), batchData.currentValue()); + Statistics batchStatistics = Statistics.getStatsByType(dataType); + type.update(batchStatistics, batchData); + Assert.assertEquals(1, batchStatistics.getCount()); + Assert.assertEquals(200L, batchStatistics.getStartTime()); + Assert.assertEquals(200L, batchStatistics.getEndTime()); + Assert.assertFalse(batchStatistics.isEmpty()); + + TsBlockBuilder blockBuilder = new TsBlockBuilder(Arrays.asList(dataType, dataType)); + blockBuilder.getTimeColumnBuilder().writeLong(250L); + blockBuilder.getColumnBuilder(0).writeObject(value.getValue()); + blockBuilder.getColumnBuilder(1).writeObject(value.getValue()); + blockBuilder.declarePosition(); + blockBuilder.getTimeColumnBuilder().writeLong(300L); + blockBuilder.getColumnBuilder(0).writeObject(value.getValue()); + blockBuilder.getColumnBuilder(1).writeObject(value.getValue()); + blockBuilder.declarePosition(); + TsBlock block = blockBuilder.build(); + Statistics blockStatistics = Statistics.getStatsByType(dataType); + type.update(blockStatistics, block, 1, 1); + Assert.assertEquals(1, blockStatistics.getCount()); + Assert.assertEquals(300L, blockStatistics.getStartTime()); + Assert.assertEquals(300L, blockStatistics.getEndTime()); + Assert.assertFalse(blockStatistics.isEmpty()); + } + } + + @Test + public void testGetCurrentValueForVectorAndUnknown() { + TsPrimitiveType[] vector = {new TsPrimitiveType.TsLong(1L)}; + BatchData batchData = new BatchData(TSDataType.VECTOR); + batchData.putAnObject(1L, vector); + Assert.assertSame(vector, Type.fromTsDataType(TSDataType.VECTOR).getCurrentValue(batchData)); + Assert.assertSame(vector, batchData.currentValue()); + Assert.assertNull(Type.fromTsDataType(TSDataType.UNKNOWN).getCurrentValue(null)); + } + + private static ByteBuffer intBuffer(int value) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.putInt(value); + return buffer.flip(); + } + + private static ByteBuffer longBuffer(long value) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(value); + return buffer.flip(); + } + + private static ByteBuffer binaryBuffer(Binary value) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES + value.getLength()); + buffer.putInt(value.getLength()); + buffer.put(value.getValues()); + return buffer.flip(); + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java index a4b7f3c8d..a6c6e0599 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/filter/TsBlockFilterTest.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.common.block.column.IntColumn; import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.IntTypeService; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.ValueFilterApi; import org.apache.tsfile.utils.Binary; @@ -41,6 +43,21 @@ public class TsBlockFilterTest { private static final int COL_NUM = 3; private static final int ROW_NUM = 7; + private static final IntTypeService GET_MEASUREMENT_ID_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> 0; + case INT32 -> 1; + case INT64 -> 2; + case FLOAT -> 3; + case DOUBLE -> 4; + case TEXT -> 5; + case BLOB, DATE, OBJECT, ROW, STRING, TIMESTAMP, UNKNOWN, VECTOR -> -1; + }; + + static { + GET_MEASUREMENT_ID_SERVICE.check(); + } private TsBlock tsBlock; @@ -49,22 +66,7 @@ public class TsBlockFilterTest { private boolean[] selection = new boolean[COL_NUM]; public int getMeasurementId(TSDataType type) { - switch (type) { - case BOOLEAN: - return 0; - case INT32: - return 1; - case INT64: - return 2; - case FLOAT: - return 3; - case DOUBLE: - return 4; - case TEXT: - return 5; - default: - return -1; - } + return GET_MEASUREMENT_ID_SERVICE.call(Type.fromTsDataType(type)); } @Before diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java index 7b0258f2d..7c9a38bd8 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/PageReaderTest.java @@ -36,6 +36,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.reader.page.PageReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.page.PageWriter; @@ -51,6 +53,22 @@ public class PageReaderTest { private static final int POINTS_COUNT_IN_ONE_PAGE = 1000000; + private static final TypeService WRITE_PAGE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (writer, time, value) -> writer.write(time, (Boolean) value); + case INT32 -> (writer, time, value) -> writer.write(time, (Integer) value); + case INT64 -> (writer, time, value) -> writer.write(time, (Long) value); + case FLOAT -> (writer, time, value) -> writer.write(time, (Float) value); + case DOUBLE -> (writer, time, value) -> writer.write(time, (Double) value); + case TEXT -> (writer, time, value) -> writer.write(time, (Binary) value); + case BLOB, DATE, OBJECT, ROW, STRING, TIMESTAMP, UNKNOWN, VECTOR -> + (writer, time, value) -> {}; + }; + + static { + WRITE_PAGE_VALUE_SERVICE.check(); + } @Test public void testLong() { @@ -272,33 +290,21 @@ public void testDelete(TSDataType dataType) { } private void writeData() { + PageValueWriter valueWriter = WRITE_PAGE_VALUE_SERVICE.call(Type.fromTsDataType(dataType)); for (int i = 0; i < count; i++) { - switch (dataType) { - case BOOLEAN: - pageWriter.write(i, (Boolean) generateValueByIndex(i)); - break; - case INT32: - pageWriter.write(i, (Integer) generateValueByIndex(i)); - break; - case INT64: - pageWriter.write(i, (Long) generateValueByIndex(i)); - break; - case FLOAT: - pageWriter.write(i, (Float) generateValueByIndex(i)); - break; - case DOUBLE: - pageWriter.write(i, (Double) generateValueByIndex(i)); - break; - case TEXT: - pageWriter.write(i, (Binary) generateValueByIndex(i)); - break; - } + valueWriter.write(pageWriter, i, generateValueByIndex(i)); } } public abstract Object generateValueByIndex(int i); } + @FunctionalInterface + private interface PageValueWriter { + + void write(PageWriter writer, long time, Object value); + } + @Test public void testPageDelete() { LoopWriteReadTest test = diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java index bfc55868d..efa9871b4 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/reader/TsFileLastReaderTest.java @@ -24,6 +24,8 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; @@ -61,6 +63,34 @@ public class TsFileLastReaderTest { Arrays.asList(TSDataType.INT64, TSDataType.BLOB); private static final Map typeAddValueFunctions = new HashMap<>(); + private static final TypeService ADD_OFFSET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT64 -> + (tablet, row, column, offset) -> tablet.addValue(row, column, (long) row + offset); + case BLOB -> + (tablet, row, column, offset) -> + tablet.addValue( + row, + column, + Long.toBinaryString(row + offset).getBytes(StandardCharsets.UTF_8)); + case BOOLEAN, + DATE, + DOUBLE, + FLOAT, + INT32, + OBJECT, + ROW, + STRING, + TEXT, + TIMESTAMP, + UNKNOWN, + VECTOR -> + (tablet, row, column, offset) -> { + throw new IllegalArgumentException( + "Unsupported TSDataType " + type.getTypeEnum()); + }; + }; static { typeAddValueFunctions.put( @@ -70,6 +100,7 @@ public class TsFileLastReaderTest { ((tablet, row, column) -> tablet.addValue( row, column, Long.toBinaryString(row).getBytes(StandardCharsets.UTF_8)))); + ADD_OFFSET_VALUE_SERVICE.check(); } private final String filePath = "target/test.tsfile"; @@ -142,20 +173,10 @@ private void createFileWithLastEmptyChunks(int deviceNum, int measurementNum, in } for (int j = 0; j < measurementNum / 2; j++) { TSDataType tsDataType = dataTypes.get(j % dataTypes.size()); + OffsetTabletAddValueFunction addValueFunction = + ADD_OFFSET_VALUE_SERVICE.call(Type.fromTsDataType(tsDataType)); for (int k = 0; k < seriesPointNum; k++) { - switch (tsDataType) { - case INT64: - tablet.addValue(k, j, (long) k + seriesPointNum / 2); - break; - case BLOB: - tablet.addValue( - k, - j, - Long.toBinaryString(k + seriesPointNum / 2).getBytes(StandardCharsets.UTF_8)); - break; - default: - throw new IllegalArgumentException("Unsupported TSDataType " + tsDataType); - } + addValueFunction.addValue(tablet, k, j, seriesPointNum / 2); } } writer.writeTree(tablet); @@ -163,6 +184,12 @@ private void createFileWithLastEmptyChunks(int deviceNum, int measurementNum, in } } + @FunctionalInterface + private interface OffsetTabletAddValueFunction { + + void addValue(Tablet tablet, int row, int column, int offset); + } + private void createAlignedFileWithObject(int seriesPointNum) throws IOException, WriteProcessException { try (TsFileWriter writer = new TsFileWriter(file)) { @@ -332,13 +359,16 @@ public void testLastEmptyPage() throws Exception { alignedChunkWriter.write( 0, new TsPrimitiveType[] { - TsPrimitiveType.getByType(TSDataType.INT64, 0L), - TsPrimitiveType.getByType( - TSDataType.BLOB, new Binary("0".getBytes(StandardCharsets.UTF_8))) + Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(0L), + Type.fromTsDataType(TSDataType.BLOB) + .getTsPrimitiveType(new Binary("0".getBytes(StandardCharsets.UTF_8))) }); alignedChunkWriter.sealCurrentPage(); alignedChunkWriter.write( - 1, new TsPrimitiveType[] {TsPrimitiveType.getByType(TSDataType.INT64, 1L), null}); + 1, + new TsPrimitiveType[] { + Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(1L), null + }); alignedChunkWriter.writeToFileWriter(ioWriter); ioWriter.endChunkGroup(); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java index b55473681..434fd50cf 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java @@ -32,6 +32,8 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.tsfile.read.expression.QueryExpression; @@ -68,6 +70,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.IntFunction; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -81,6 +84,29 @@ public class TableViewTest { private static final int measurementSchemaNum = 5; private static TableSchema testTableSchema; private static int numTimestampPerDevice = 10; + private static final TypeService> GET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT64 -> i -> (long) i; + case TEXT -> i -> new Binary(String.valueOf(i), StandardCharsets.UTF_8); + case BLOB, + BOOLEAN, + DATE, + DOUBLE, + FLOAT, + INT32, + OBJECT, + ROW, + STRING, + TIMESTAMP, + UNKNOWN, + VECTOR -> + i -> i; + }; + + static { + GET_VALUE_SERVICE.check(); + } @Before public void setUp() throws Exception { @@ -505,14 +531,7 @@ public static Tablet genTablet(TableSchema tableSchema, int offset, int deviceNu } public static Object getValue(TSDataType dataType, int i) { - switch (dataType) { - case INT64: - return (long) i; - case TEXT: - return new Binary(String.valueOf(i), StandardCharsets.UTF_8); - default: - return i; - } + return GET_VALUE_SERVICE.call(Type.fromTsDataType(dataType)).apply(i); } public static TableSchema genTableSchema(int tableNum) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java index 7e623590b..f67f82a0b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java @@ -21,13 +21,10 @@ import org.apache.tsfile.common.constant.JsonFormatConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.TSRecord; -import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; -import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; -import org.apache.tsfile.write.record.datapoint.FloatDataPoint; -import org.apache.tsfile.write.record.datapoint.IntDataPoint; -import org.apache.tsfile.write.record.datapoint.LongDataPoint; -import org.apache.tsfile.write.record.datapoint.StringDataPoint; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.Schema; @@ -56,7 +53,7 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { try { timestamp = Long.parseLong(items[1].trim()); } catch (NumberFormatException e) { - LOG.warn("given timestamp is illegal:{}", str); + LOG.warn(Messages.get("log.utils.record_timestamp_illegal"), str); // return a TSRecord without any data points return new TSRecord(deviceId, -1); } @@ -75,7 +72,7 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { ? null : measurementGroup.getMeasurementSchemaMap().get(measurementId); if (measurementSchema == null) { - LOG.warn("measurementId:{},type not found, pass", measurementId); + LOG.warn(Messages.get("log.utils.record_measurement_type_not_found"), measurementId); continue; } type = measurementSchema.getType(); @@ -84,35 +81,11 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { // TSRecord if (!"".equals(value)) { try { - switch (type) { - case INT32: - case DATE: - ret.addTuple(new IntDataPoint(measurementId, Integer.parseInt(value))); - break; - case INT64: - case TIMESTAMP: - ret.addTuple(new LongDataPoint(measurementId, Long.parseLong(value))); - break; - case FLOAT: - ret.addTuple(new FloatDataPoint(measurementId, Float.parseFloat(value))); - break; - case DOUBLE: - ret.addTuple(new DoubleDataPoint(measurementId, Double.parseDouble(value))); - break; - case BOOLEAN: - ret.addTuple(new BooleanDataPoint(measurementId, Boolean.parseBoolean(value))); - break; - case TEXT: - case BLOB: - case STRING: - ret.addTuple(new StringDataPoint(measurementId, BytesUtils.valueOf(items[i + 1]))); - break; - default: - LOG.warn("unsupported data type:{}", type); - break; - } + ret.addTuple(Type.fromTsDataType(type).getDataPoint(measurementId, value)); } catch (NumberFormatException e) { - LOG.warn("parsing measurement meets error, omit it", e); + LOG.warn(Messages.get("log.utils.record_parse_measurement_error"), e); + } catch (UnSupportedDataTypeException e) { + LOG.warn(Messages.get("log.utils.record_unsupported_data_type"), type); } } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java index d74847b38..7c55269c7 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsPrimitiveTypeTest.java @@ -21,12 +21,15 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.TsPrimitiveType.TsBinary; import org.apache.tsfile.utils.TsPrimitiveType.TsBoolean; import org.apache.tsfile.utils.TsPrimitiveType.TsDouble; import org.apache.tsfile.utils.TsPrimitiveType.TsFloat; import org.apache.tsfile.utils.TsPrimitiveType.TsInt; import org.apache.tsfile.utils.TsPrimitiveType.TsLong; +import org.apache.tsfile.utils.TsPrimitiveType.TsVector; import org.junit.Assert; import org.junit.Test; @@ -35,32 +38,48 @@ public class TsPrimitiveTypeTest { @Test public void testNewAndGet() { - TsPrimitiveType intValue = TsPrimitiveType.getByType(TSDataType.INT32, 123); + TsPrimitiveType intValue = Type.fromTsDataType(TSDataType.INT32).getTsPrimitiveType(123); Assert.assertEquals(new TsInt(123), intValue); Assert.assertEquals(123, intValue.getInt()); - TsPrimitiveType longValue = TsPrimitiveType.getByType(TSDataType.INT64, 456L); + TsPrimitiveType longValue = Type.fromTsDataType(TSDataType.INT64).getTsPrimitiveType(456L); Assert.assertEquals(new TsLong(456), longValue); Assert.assertEquals(456L, longValue.getLong()); - TsPrimitiveType floatValue = TsPrimitiveType.getByType(TSDataType.FLOAT, 123f); + TsPrimitiveType floatValue = Type.fromTsDataType(TSDataType.FLOAT).getTsPrimitiveType(123f); Assert.assertEquals(new TsFloat(123), floatValue); Assert.assertEquals(123f, floatValue.getFloat(), 0.01); - TsPrimitiveType doubleValue = TsPrimitiveType.getByType(TSDataType.DOUBLE, 456d); + TsPrimitiveType doubleValue = Type.fromTsDataType(TSDataType.DOUBLE).getTsPrimitiveType(456d); Assert.assertEquals(new TsDouble(456), doubleValue); Assert.assertEquals(456d, doubleValue.getDouble(), 0.01); TsPrimitiveType textValue = - TsPrimitiveType.getByType(TSDataType.TEXT, new Binary("123", TSFileConfig.STRING_CHARSET)); + Type.fromTsDataType(TSDataType.TEXT) + .getTsPrimitiveType(new Binary("123", TSFileConfig.STRING_CHARSET)); Assert.assertEquals(new TsBinary(new Binary("123", TSFileConfig.STRING_CHARSET)), textValue); Assert.assertEquals(new Binary("123", TSFileConfig.STRING_CHARSET), textValue.getBinary()); - TsPrimitiveType booleanValue = TsPrimitiveType.getByType(TSDataType.BOOLEAN, true); + TsPrimitiveType booleanValue = Type.fromTsDataType(TSDataType.BOOLEAN).getTsPrimitiveType(true); Assert.assertEquals(new TsBoolean(true), booleanValue); Assert.assertTrue(booleanValue.getBoolean()); } + @Test + public void testTypeSpecificCreation() { + TsPrimitiveType emptyDate = Type.fromTsDataType(TSDataType.DATE).getTsPrimitiveType(); + Assert.assertEquals(TSDataType.DATE, emptyDate.getDataType()); + + TsPrimitiveType date = Type.fromTsDataType(TSDataType.DATE).getTsPrimitiveType(20260713); + Assert.assertEquals(20260713, date.getInt()); + Assert.assertEquals(TSDataType.DATE, date.getDataType()); + + TsPrimitiveType[] values = {date, null}; + TsPrimitiveType vector = RowType.anonymousRow().getTsPrimitiveType(values); + Assert.assertSame(values, vector.getVector()); + Assert.assertNull(RowType.anonymousRow().getTsPrimitiveType().getVector()); + } + @Test public void testCompareWithNullValue() { TimeValuePair timeValuePair1 = @@ -75,4 +94,66 @@ public void testCompareWithNullValue() { new TsPrimitiveType[] {new TsBoolean(true), null, new TsInt(1), null})); Assert.assertFalse(timeValuePair1.equals(timeValuePair2)); } + + @Test + public void testCopy() { + TsBoolean booleanValue = new TsBoolean(true); + TsBoolean booleanCopy = new TsBoolean(); + booleanCopy.copy(booleanValue); + booleanValue.setBoolean(false); + Assert.assertTrue(booleanCopy.getBoolean()); + + TsInt dateValue = new TsInt(20260722, TSDataType.DATE); + TsInt dateCopy = new TsInt(); + dateCopy.copy(dateValue); + dateValue.setInt(20260723); + Assert.assertEquals(20260722, dateCopy.getInt()); + Assert.assertEquals(TSDataType.DATE, dateCopy.getDataType()); + + TsLong longValue = new TsLong(1L); + TsLong longCopy = new TsLong(); + longCopy.copy(longValue); + longValue.setLong(2L); + Assert.assertEquals(1L, longCopy.getLong()); + + TsFloat floatValue = new TsFloat(1.0F); + TsFloat floatCopy = new TsFloat(); + floatCopy.copy(floatValue); + floatValue.setFloat(2.0F); + Assert.assertEquals(1.0F, floatCopy.getFloat(), 0.0F); + + TsDouble doubleValue = new TsDouble(1.0D); + TsDouble doubleCopy = new TsDouble(); + doubleCopy.copy(doubleValue); + doubleValue.setDouble(2.0D); + Assert.assertEquals(1.0D, doubleCopy.getDouble(), 0.0D); + + Binary binary = new Binary("original", TSFileConfig.STRING_CHARSET); + TsBinary binaryValue = new TsBinary(binary); + TsBinary binaryCopy = new TsBinary(); + binaryCopy.copy(binaryValue); + binary.getValues()[0] = 'O'; + Assert.assertEquals( + "original", binaryCopy.getBinary().getStringValue(TSFileConfig.STRING_CHARSET)); + + TsVector vectorValue = + new TsVector(new TsPrimitiveType[] {dateCopy, binaryCopy, null, new TsBoolean(true)}); + TsVector vectorCopy = new TsVector(); + vectorCopy.copy(vectorValue); + vectorValue.getVector()[0].setInt(0); + vectorValue.getVector()[1].getBinary().getValues()[0] = 'O'; + vectorValue.getVector()[3].setBoolean(false); + + Assert.assertNotSame(vectorValue.getVector(), vectorCopy.getVector()); + Assert.assertEquals(20260722, vectorCopy.getVector()[0].getInt()); + Assert.assertEquals(TSDataType.DATE, vectorCopy.getVector()[0].getDataType()); + Assert.assertEquals( + "original", + vectorCopy.getVector()[1].getBinary().getStringValue(TSFileConfig.STRING_CHARSET)); + Assert.assertNull(vectorCopy.getVector()[2]); + Assert.assertTrue(vectorCopy.getVector()[3].getBoolean()); + + booleanCopy.copy(null); + Assert.assertFalse(booleanCopy.getBoolean()); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java index 6398eac75..68384314d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java @@ -20,6 +20,8 @@ package org.apache.tsfile.utils; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.junit.Test; @@ -28,6 +30,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.function.IntFunction; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -36,6 +39,129 @@ public class TypeCastTest { + private static final TypeService CHECK_ARRAY_CAST_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> + (from, to, array) -> + assertArrayEquals( + (int[]) genValueArray(to), (int[]) to.castFromArray(from, array)); + case INT64, TIMESTAMP -> + (from, to, array) -> + assertArrayEquals( + (long[]) genValueArray(to), (long[]) to.castFromArray(from, array)); + case BOOLEAN -> + (from, to, array) -> + assertArrayEquals( + (boolean[]) genValueArray(to), (boolean[]) to.castFromArray(from, array)); + case BLOB, STRING, TEXT -> + (from, to, array) -> + assertArrayEquals( + TypeCastTest.EXPECTED_BINARY_ARRAY_SERVICE + .call(Type.fromTsDataType(from)) + .generate(array), + (Binary[]) to.castFromArray(from, array)); + case FLOAT -> + (from, to, array) -> + assertArrayEquals( + (float[]) genValueArray(to), (float[]) to.castFromArray(from, array), 0.1f); + case DOUBLE -> + (from, to, array) -> + assertArrayEquals( + (double[]) genValueArray(to), + (double[]) to.castFromArray(from, array), + 0.1); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (from, to, array) -> fail("Unexpected type: " + to); + }; + + private static final TypeService EXPECTED_BINARY_ARRAY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> + array -> { + int[] values = (int[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case DATE -> + array -> { + int[] values = (int[]) array; + return toBinaryArray( + values.length, i -> TSDataType.getDateStringValue(values[i])); + }; + case INT64, TIMESTAMP -> + array -> { + long[] values = (long[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case FLOAT -> + array -> { + float[] values = (float[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case DOUBLE -> + array -> { + double[] values = (double[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case BOOLEAN -> + array -> { + boolean[] values = (boolean[]) array; + return toBinaryArray(values.length, i -> String.valueOf(values[i])); + }; + case BLOB, STRING, TEXT -> array -> (Binary[]) array; + case OBJECT, ROW, UNKNOWN, VECTOR -> + array -> { + throw new IllegalArgumentException( + "Unsupported data type: " + type.getTypeEnum()); + }; + }; + + private static final TypeService GENERATE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> () -> 1; + case INT64, TIMESTAMP -> () -> 1L; + case BOOLEAN -> () -> false; + case FLOAT -> () -> 1.0f; + case DOUBLE -> () -> 1.0; + case BLOB, OBJECT, STRING, TEXT -> () -> new Binary("1", StandardCharsets.UTF_8); + case ROW, UNKNOWN, VECTOR -> + () -> { + throw new IllegalArgumentException( + "Unsupported data type: " + type.getTypeEnum()); + }; + }; + + private static final TypeService GENERATE_VALUE_ARRAY_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32, DATE -> () -> new int[] {1, 2, 3}; + case INT64, TIMESTAMP -> () -> new long[] {1, 2, 3}; + case BOOLEAN -> () -> new boolean[] {true, false}; + case FLOAT -> () -> new float[] {1.0f, 2.0f, 3.0f}; + case DOUBLE -> () -> new double[] {1.0, 2.0, 3.0}; + case BLOB, OBJECT, STRING, TEXT -> + () -> + new Binary[] { + new Binary("1", StandardCharsets.UTF_8), + new Binary("2", StandardCharsets.UTF_8), + new Binary("3", StandardCharsets.UTF_8) + }; + case ROW, UNKNOWN, VECTOR -> + () -> { + throw new IllegalArgumentException( + "Unsupported data type: " + type.getTypeEnum()); + }; + }; + + static { + CHECK_ARRAY_CAST_SERVICE.check(); + EXPECTED_BINARY_ARRAY_SERVICE.check(); + GENERATE_VALUE_SERVICE.check(); + GENERATE_VALUE_ARRAY_SERVICE.check(); + } + @Test public void testSingleCast() { Set dataTypes = new HashSet<>(); @@ -85,154 +211,42 @@ public void testArrayCast() { assertThrows(ClassCastException.class, () -> to.castFromArray(from, array)); continue; } - switch (to) { - case INT32: - case DATE: - assertArrayEquals((int[]) genValueArray(to), (int[]) to.castFromArray(from, array)); - break; - case INT64: - case TIMESTAMP: - assertArrayEquals((long[]) genValueArray(to), (long[]) to.castFromArray(from, array)); - break; - case BOOLEAN: - assertArrayEquals( - (boolean[]) genValueArray(to), (boolean[]) to.castFromArray(from, array)); - break; - case STRING: - case BLOB: - case TEXT: - switch (from) { - case BLOB: - case STRING: - assertArrayEquals((Binary[]) array, (Binary[]) to.castFromArray(from, array)); - break; - case INT32: - int[] tmpInt = (int[]) array; - Binary[] intResult = new Binary[tmpInt.length]; - for (int i = 0; i < tmpInt.length; i++) { - intResult[i] = new Binary(String.valueOf(tmpInt[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(intResult, (Binary[]) to.castFromArray(from, array)); - break; - case DATE: - int[] tmpDate = (int[]) array; - Binary[] dateResult = new Binary[tmpDate.length]; - for (int i = 0; i < tmpDate.length; i++) { - dateResult[i] = - new Binary(TSDataType.getDateStringValue(tmpDate[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(dateResult, (Binary[]) to.castFromArray(from, array)); - break; - case INT64: - case TIMESTAMP: - long[] tmpLong = (long[]) array; - Binary[] longResult = new Binary[tmpLong.length]; - for (int i = 0; i < tmpLong.length; i++) { - longResult[i] = new Binary(String.valueOf(tmpLong[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(longResult, (Binary[]) to.castFromArray(from, array)); - break; - case FLOAT: - float[] tmpFloat = (float[]) array; - Binary[] floatResult = new Binary[tmpFloat.length]; - for (int i = 0; i < tmpFloat.length; i++) { - floatResult[i] = new Binary(String.valueOf(tmpFloat[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(floatResult, (Binary[]) to.castFromArray(from, array)); - break; - case DOUBLE: - double[] tmpDouble = (double[]) array; - Binary[] doubleResult = new Binary[tmpDouble.length]; - for (int i = 0; i < tmpDouble.length; i++) { - doubleResult[i] = - new Binary(String.valueOf(tmpDouble[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(doubleResult, (Binary[]) to.castFromArray(from, array)); - break; - case BOOLEAN: - boolean[] tmpBoolean = (boolean[]) array; - Binary[] booleanResult = new Binary[tmpBoolean.length]; - for (int i = 0; i < tmpBoolean.length; i++) { - booleanResult[i] = - new Binary(String.valueOf(tmpBoolean[i]), StandardCharsets.UTF_8); - } - assertArrayEquals(booleanResult, (Binary[]) to.castFromArray(from, array)); - break; - default: - break; - } - break; - case FLOAT: - assertArrayEquals( - (float[]) genValueArray(to), (float[]) to.castFromArray(from, array), 0.1f); - break; - case DOUBLE: - assertArrayEquals( - (double[]) genValueArray(to), (double[]) to.castFromArray(from, array), 0.1); - break; - case UNKNOWN: - case VECTOR: - default: - fail("Unexpected type: " + to); - } + CHECK_ARRAY_CAST_SERVICE.call(Type.fromTsDataType(to)).check(from, to, array); } } } - private Object genValue(TSDataType dataType) { - int i = 1; - switch (dataType) { - case INT32: - case DATE: - return i; - case TIMESTAMP: - case INT64: - return (long) i; - case BOOLEAN: - return false; - case FLOAT: - return i * 1.0f; - case DOUBLE: - return i * 1.0; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - return new Binary(Integer.toString(i), StandardCharsets.UTF_8); - case UNKNOWN: - case VECTOR: - default: - throw new IllegalArgumentException("Unsupported data type: " + dataType); - } + private static Object genValue(TSDataType dataType) { + return GENERATE_VALUE_SERVICE.call(Type.fromTsDataType(dataType)).generate(); + } + + private static Object genValueArray(TSDataType dataType) { + return GENERATE_VALUE_ARRAY_SERVICE.call(Type.fromTsDataType(dataType)).generate(); } - private Object genValueArray(TSDataType dataType) { - switch (dataType) { - case INT32: - case DATE: - return new int[] {1, 2, 3}; - case TIMESTAMP: - case INT64: - return new long[] {1, 2, 3}; - case BOOLEAN: - return new boolean[] {true, false}; - case FLOAT: - return new float[] {1.0f, 2.0f, 3.0f}; - case DOUBLE: - return new double[] {1.0, 2.0, 3.0}; - case STRING: - case TEXT: - case BLOB: - case OBJECT: - return new Binary[] { - new Binary(Integer.toString(1), StandardCharsets.UTF_8), - new Binary(Integer.toString(2), StandardCharsets.UTF_8), - new Binary(Integer.toString(3), StandardCharsets.UTF_8) - }; - case UNKNOWN: - case VECTOR: - default: - throw new IllegalArgumentException("Unsupported data type: " + dataType); + private static Binary[] toBinaryArray(int length, IntFunction valueProvider) { + Binary[] result = new Binary[length]; + for (int i = 0; i < length; i++) { + result[i] = new Binary(valueProvider.apply(i), StandardCharsets.UTF_8); } + return result; + } + + @FunctionalInterface + private interface ArrayCastChecker { + + void check(TSDataType from, TSDataType to, Object array); + } + + @FunctionalInterface + private interface BinaryArrayGenerator { + + Binary[] generate(Object array); + } + + @FunctionalInterface + private interface ValueGenerator { + + Object generate(); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java index ab4bf377b..660fe8534 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java @@ -24,6 +24,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.service.TypeService; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; @@ -59,6 +61,98 @@ public class TabletTest { + private static final TypeService SAMPLE_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case BOOLEAN -> (row, col, variableBinaryLength) -> (row + col) % 2 == 0; + case INT32 -> (row, col, variableBinaryLength) -> row + col * 100; + case INT64, TIMESTAMP -> + (row, col, variableBinaryLength) -> (long) (valueOffset(row, col) * 1_000_000L); + case FLOAT -> (row, col, variableBinaryLength) -> (row + col) * 1.5f; + case DOUBLE -> (row, col, variableBinaryLength) -> (row + col) * 2.5; + case TEXT, STRING -> + (row, col, variableBinaryLength) -> + stringOfLength(binaryPayloadLength(variableBinaryLength, row, col)); + case BLOB -> + (row, col, variableBinaryLength) -> + binaryOfLength(binaryPayloadLength(variableBinaryLength, row, col)); + case DATE -> + (row, col, variableBinaryLength) -> + LocalDate.of(2000 + (row % 20), (col % 12) + 1, (row % 28) + 1); + case OBJECT, ROW, UNKNOWN, VECTOR -> + (row, col, variableBinaryLength) -> { + throw new IllegalArgumentException( + "Unsupported type in test: " + type.getTypeEnum()); + }; + }; + + private static final TypeService WRITE_TABLET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (tablet, row, col, value) -> tablet.addValue(row, col, value); + case INT64, TIMESTAMP -> + (tablet, row, col, value) -> tablet.addValue(row, col, (long) value); + case FLOAT -> (tablet, row, col, value) -> tablet.addValue(row, col, value * 1.0f); + case DOUBLE -> (tablet, row, col, value) -> tablet.addValue(row, col, value * 1.0); + case BOOLEAN -> (tablet, row, col, value) -> tablet.addValue(row, col, value % 2 == 0); + case BLOB, STRING, TEXT -> + (tablet, row, col, value) -> tablet.addValue(row, col, String.valueOf(value)); + case OBJECT -> + (tablet, row, col, value) -> + tablet.addValue(row, col, value % 2 == 0, value, new byte[] {(byte) row}); + case DATE -> + (tablet, row, col, value) -> tablet.addValue(row, col, LocalDate.of(value, 1, 1)); + case ROW, UNKNOWN, VECTOR -> + (tablet, row, col, value) -> { + throw new IllegalArgumentException( + "Unsupported type in test: " + type.getTypeEnum()); + }; + }; + + private static final TypeService CHECK_TABLET_VALUE_SERVICE = + type -> + switch (type.getTypeEnum()) { + case INT32 -> (tablet, row, col) -> assertEquals(row, tablet.getValue(row, col)); + case INT64, TIMESTAMP -> + (tablet, row, col) -> assertEquals((long) row, tablet.getValue(row, col)); + case FLOAT -> + (tablet, row, col) -> + assertEquals(row * 1.0f, (float) tablet.getValue(row, col), 0.0001f); + case DOUBLE -> + (tablet, row, col) -> + assertEquals(row * 1.0, (double) tablet.getValue(row, col), 0.0001); + case BOOLEAN -> + (tablet, row, col) -> assertEquals(row % 2 == 0, tablet.getValue(row, col)); + case BLOB, STRING, TEXT -> + (tablet, row, col) -> + assertEquals( + new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8)), + tablet.getValue(row, col)); + case OBJECT -> + (tablet, row, col) -> { + byte[] content = new byte[] {(byte) row}; + byte[] expected = new byte[content.length + 9]; + expected[0] = (byte) (row % 2); + System.arraycopy(BytesUtils.longToBytes(row), 0, expected, 1, 8); + System.arraycopy(content, 0, expected, 9, content.length); + assertEquals(new Binary(expected), tablet.getValue(row, col)); + }; + case DATE -> + (tablet, row, col) -> + assertEquals(LocalDate.of(row, 1, 1), tablet.getValue(row, col)); + case ROW, UNKNOWN, VECTOR -> + (tablet, row, col) -> { + throw new IllegalArgumentException( + "Unsupported type in test: " + type.getTypeEnum()); + }; + }; + + static { + SAMPLE_VALUE_SERVICE.check(); + WRITE_TABLET_VALUE_SERVICE.check(); + CHECK_TABLET_VALUE_SERVICE.check(); + } + @Test public void testAddValue() { Tablet tablet = @@ -694,28 +788,9 @@ private static int binaryPayloadLength(boolean variableBinaryLength, int row, in } private Object sampleValue(TSDataType type, int row, int col, boolean variableBinaryLength) { - switch (type) { - case BOOLEAN: - return (row + col) % 2 == 0; - case INT32: - return row + col * 100; - case INT64: - case TIMESTAMP: - return (long) (valueOffset(row, col) * 1_000_000L); - case FLOAT: - return (row + col) * 1.5f; - case DOUBLE: - return (row + col) * 2.5; - case TEXT: - case STRING: - return stringOfLength(binaryPayloadLength(variableBinaryLength, row, col)); - case BLOB: - return binaryOfLength(binaryPayloadLength(variableBinaryLength, row, col)); - case DATE: - return LocalDate.of(2000 + (row % 20), (col % 12) + 1, (row % 28) + 1); - default: - throw new IllegalArgumentException("Unsupported type in test: " + type); - } + return SAMPLE_VALUE_SERVICE + .call(Type.fromTsDataType(type)) + .generate(row, col, variableBinaryLength); } private static int valueOffset(int row, int col) { @@ -799,35 +874,9 @@ private void fillTablet(Tablet t, int valueOffset, int length) { for (int i = 0; i < length; i++) { t.addTimestamp(i, i + valueOffset); for (int j = 0; j < t.getSchemas().size(); j++) { - switch (t.getSchemas().get(j).getType()) { - case INT32: - t.addValue(i, j, i + valueOffset); - break; - case TIMESTAMP: - case INT64: - t.addValue(i, j, (long) (i + valueOffset)); - break; - case FLOAT: - t.addValue(i, j, (i + valueOffset) * 1.0f); - break; - case DOUBLE: - t.addValue(i, j, (i + valueOffset) * 1.0); - break; - case BOOLEAN: - t.addValue(i, j, (i + valueOffset) % 2 == 0); - break; - case TEXT: - case STRING: - case BLOB: - t.addValue(i, j, String.valueOf(i + valueOffset)); - break; - case OBJECT: - t.addValue(i, j, (i + valueOffset) % 2 == 0, i + valueOffset, new byte[] {(byte) i}); - break; - case DATE: - t.addValue(i, j, LocalDate.of(i + valueOffset, 1, 1)); - break; - } + WRITE_TABLET_VALUE_SERVICE + .call(Type.fromTsDataType(t.getSchemas().get(j).getType())) + .write(t, i, j, i + valueOffset); } } } @@ -1030,45 +1079,28 @@ private void checkAppendedTablet( } assertFalse(result.isNull(i, j)); - switch (result.getSchemas().get(j).getType()) { - case INT32: - assertEquals(i, result.getValue(i, j)); - break; - case TIMESTAMP: - case INT64: - assertEquals((long) i, result.getValue(i, j)); - break; - case FLOAT: - assertEquals(i * 1.0f, (float) result.getValue(i, j), 0.0001f); - break; - case DOUBLE: - assertEquals(i * 1.0, (double) result.getValue(i, j), 0.0001); - break; - case BOOLEAN: - assertEquals(i % 2 == 0, result.getValue(i, j)); - break; - case TEXT: - case BLOB: - case STRING: - assertEquals( - new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)), - result.getValue(i, j)); - break; - case OBJECT: - { - byte[] content = new byte[] {(byte) i}; - byte[] expected = new byte[content.length + 9]; - expected[0] = (byte) (i % 2); - System.arraycopy(BytesUtils.longToBytes(i), 0, expected, 1, 8); - System.arraycopy(content, 0, expected, 9, content.length); - assertEquals(new Binary(expected), result.getValue(i, j)); - } - break; - case DATE: - assertEquals(LocalDate.of(i, 1, 1), result.getValue(i, j)); - break; - } + CHECK_TABLET_VALUE_SERVICE + .call(Type.fromTsDataType(result.getSchemas().get(j).getType())) + .check(result, i, j); } } } + + @FunctionalInterface + private interface SampleValueGenerator { + + Object generate(int row, int col, boolean variableBinaryLength); + } + + @FunctionalInterface + private interface TabletValueWriter { + + void write(Tablet tablet, int row, int col, int value); + } + + @FunctionalInterface + private interface TabletValueChecker { + + void check(Tablet tablet, int row, int col); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java index 749141235..5a8dd5a99 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/AlignedChunkWriterImplTest.java @@ -22,6 +22,9 @@ import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,12 +34,37 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; public class AlignedChunkWriterImplTest { + @Test + public void testBatchWrite() { + AlignedChunkWriterImpl chunkWriter = + new AlignedChunkWriterImpl(new VectorMeasurementSchemaStub()); + TsBlockBuilder builder = + new TsBlockBuilder(Arrays.asList(TSDataType.FLOAT, TSDataType.INT32, TSDataType.DOUBLE)); + for (int time = 1; time <= 20; time++) { + builder.getTimeColumnBuilder().writeLong(time); + builder.getValueColumnBuilders()[0].writeFloat(time); + builder.getValueColumnBuilders()[1].writeInt(time); + builder.getValueColumnBuilders()[2].writeDouble(time); + builder.declarePosition(); + } + TsBlock tsBlock = builder.build(); + + chunkWriter.write( + (TimeColumn) tsBlock.getTimeColumn(), + tsBlock.getValueColumns(), + tsBlock.getPositionCount()); + chunkWriter.sealCurrentPage(); + + assertEquals(485, chunkWriter.getSerializedChunkSize()); + } + @Test public void testWrite1() { VectorMeasurementSchemaStub measurementSchema = new VectorMeasurementSchemaStub(); diff --git a/pom.xml b/pom.xml index eb9380fb9..68eef8e04 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 17 17 17 - + false 3.30.2-b1 2.44.3 @@ -263,7 +263,7 @@ validate - +