diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt index d1e897ce3a..eb9926276b 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt @@ -1,12 +1,207 @@ package org.jetbrains.kotlinx.dataframe.io.db +import kotlinx.datetime.LocalDate +import kotlinx.datetime.LocalDateTime +import kotlinx.datetime.LocalTime +import kotlinx.datetime.toKotlinLocalDate +import kotlinx.datetime.toKotlinLocalDateTime +import kotlinx.datetime.toKotlinLocalTime +import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig import org.sqlite.SQLiteConfig import java.sql.Connection import java.sql.DriverManager import java.sql.ResultSet +import java.sql.Types +import java.time.ZoneOffset import kotlin.reflect.KType import kotlin.reflect.full.withNullability +import kotlin.reflect.typeOf +import kotlin.time.Instant +import kotlin.time.toKotlinInstant +import java.time.LocalDate as JavaLocalDate +import java.time.LocalDateTime as JavaLocalDateTime +import java.time.LocalTime as JavaLocalTime + +/** + * A user-provided converter from an SQLite declared column type name to + * the [DataFrame] column type and a lambda thatн converts each stored value. + * + * The type parameter [T] is the **storage-class type** of the column — the actual class of + * values returned by [java.sql.ResultSet.getObject][ResultSet.getObject] for that column. + * The Xerial SQLite JDBC driver's `getObject(int)` returns exactly one of the following, chosen + * from the runtime storage class of the value (not the declared column type): + * + * - [java.lang.Integer] ([Int]) — `INTEGER` values that fit in `Int` (`-2^31 .. 2^31 - 1`) + * - [java.lang.Long] ([Long]) — `INTEGER` values outside `Int` range + * - [java.lang.Double] ([Double]) — `REAL` values + * - [java.lang.String] ([String]) — `TEXT` values + * - `byte[]` ([ByteArray]) — `BLOB` values + * - `null` — NULL + * + * The driver never produces any other type — in particular, declared `DATE` / `TIME` / `TIMESTAMP` / + * `BOOLEAN` columns still surface as one of the six types above, not as their Java-time / Boolean + * equivalent + * + * The mapping consists of a pair of: + * 1. The [KType] of the resulting [DataFrame] column; + * 2. The lambda that returns a converted value (which must fit the [KType] from the first component). + * + * ### Example + * ``` + * val format = LocalDateTime.Format { + * year(); char('-'); monthNumber(); char('-'); day() + * char(' ') + * hour(); char(':'); minute(); char(':'); second() + * chars(" UTC") + * } + * + * val sqliteCustom = Sqlite.withCustomConverters { + * // Every column declared with "MY_DATETIME" type is parsed from custom text into Instant. + * forType("MY_DATETIME") { raw: String? -> + * raw?.let { LocalDateTime.parse(it, format).toInstant(TimeZone.UTC) } + * } + * // Identity shortcut — pin the declared type LONGVARCHAR to `String?` regardless of + * // how SQLite's type affinity would classify it. + * forType("LONGVARCHAR") + * // The "ratio" column overrides its declared type — read as Double. + * forColumn("ratio") { raw: String -> raw.toDouble() } + * } + * + * val df = DataFrame.readSqlTable(connection, "events", dbType = sqliteCustom) + * ``` + */ +public typealias SqliteCustomTypeConverter = Pair Any?)> + +/** + * DSL builder collected by [Sqlite.withCustomConverters]. Register converters via + * [forType] (matches by declared SQL type name) or [forColumn] (matches by column name; + * takes precedence over [forType]). + * + * Two overloads are available for each column or type converter: + * - **Converter form** — `forType(name) { raw -> ... }` / `forColumn(name) { raw -> ... }`. + * Accepts a lambda that converts each raw stored value; the [column's type][org.jetbrains.kotlinx.dataframe.DataColumn.type] is derived from + * the reified `R` via `typeOf()`. + * - **Identity form** — `forType(name)` / `forColumn(name)`. Same shape but with no + * conversion — values pass through as `T`. Handy when SQLite's + * [type affinity](https://www.sqlite.org/datatype3.html#type_affinity) misclassifies a + * declared type and the built-in mapping picks the wrong Kotlin type. + * + * ### Example + * ``` + * val format = LocalDateTime.Format { + * year(); char('-'); monthNumber(); char('-'); day() + * char(' ') + * hour(); char(':'); minute(); char(':'); second() + * chars(" UTC") + * } + * + * val sqliteCustom = Sqlite.withCustomConverters { + * // Every column declared with "MY_DATETIME" type is parsed from custom text into Instant. + * forType("MY_DATETIME") { raw: String? -> + * raw?.let { LocalDateTime.parse(it, format).toInstant(TimeZone.UTC) } + * } + * // Identity shortcut — pin the declared type LONGVARCHAR to `String?` regardless of + * // how SQLite's type affinity would classify it. + * forType("LONGVARCHAR") + * // The "ratio" column overrides its declared type — read as Double. + * forColumn("ratio") { raw: String -> raw.toDouble() } + * } + * + * val df = DataFrame.readSqlTable(connection, "events", dbType = sqliteCustom) + * ``` + */ +public class SqliteCustomConvertersBuilder + @PublishedApi + internal constructor() { + @PublishedApi + internal val typeMappings: MutableMap> = mutableMapOf() + + @PublishedApi + internal val columnMappings: MutableMap> = mutableMapOf() + + /** + * Register a converter for every column with the given declared SQL type name. + * + * @param T the storage class of the raw stored value ([String], [Int], [Long], [Double], + * or [ByteArray] or their nullable variants). + * @param R the target Kotlin type for the resulting DataFrame column. Resolved via + * `typeOf()` — must be a non-abstract type reachable by reflection. + * + * @param [sqlTypeName] name of the declared SQL type (as written in `CREATE TABLE`). + * @param [convert] lambda to convert the raw stored value to the target Kotlin type. + */ + public inline fun forType(sqlTypeName: String, crossinline convert: (T) -> R) { + val mapping: SqliteCustomTypeConverter = typeOf() to { raw -> convert(raw) } + typeMappings[sqlTypeName] = mapping + } + + /** + * Register an **identity converter** for every column with the given declared SQL type name: + * values pass through unchanged, but the [DataFrame] [column type][org.jetbrains.kotlinx.dataframe.DataColumn.type] + * is fixed to `T` (resolved via `typeOf()`). + * + * Useful when SQLite's [type affinity](https://www.sqlite.org/datatype3.html#type_affinity) + * misclassifies your column and the built-in mapping resolves the wrong Kotlin type. Example: + * a column declared `MY_ID` has NUMERIC affinity (no `INT`/`CHAR`/`BLOB`/`REAL`/`FLOA`/`DOUB` + * substring), so SQLite will happily convert `'42'` to an integer on insert; declaring + * `forType("MY_ID")` pins the column to `String` regardless. + * + * @param T the target Kotlin type for the resulting DataFrame column — must be one of the + * storage-class types the driver actually returns ([String], [Int], [Long], [Double], + * or [ByteArray] or their nullable variants). + * + * ### Example + * ``` + * Sqlite.withCustomConverters { + * forType("LONGVARCHAR") // read as `String?` even though affinity says NUMERIC + * forType("BINARY_ID") // read as raw bytes + * } + * ``` + */ + public inline fun forType(sqlTypeName: String) { + val mapping: SqliteCustomTypeConverter = typeOf() to { it } + typeMappings[sqlTypeName] = mapping + } + + /** + * Register a converter for a specific column by name. Column-name overrides take + * precedence over type-name overrides registered via [forType]. + * + * @param T the storage class of the raw stored value. + * @param R the target Kotlin type for the resulting DataFrame column. + */ + public inline fun forColumn(columnName: String, crossinline convert: (T) -> R) { + val mapping: SqliteCustomTypeConverter = typeOf() to { raw -> convert(raw) } + columnMappings[columnName] = mapping + } + + /** + * Register an **identity converter** for a specific column by name: values pass through + * unchanged, but the DataFrame column type is fixed to `T` (resolved via `typeOf()`). + * Column-name overrides take precedence over type-name overrides. + * + * Useful for a single column whose declared SQL type is misleading — either because SQLite's + * type affinity picks the wrong bucket or because you want a stricter type than the shared + * declared type would give. + * + * @param T the target Kotlin type for the resulting DataFrame column — must be one of the + * storage-class types the driver actually returns ([String], [Int], [Long], [Double], + * or [ByteArray] or their nullable variants). + * + * ### Example + * ``` + * Sqlite.withCustomConverters { + * forColumn("uuid") // treat the `uuid` column as raw text + * forColumn("payload") + * } + * ``` + */ + public inline fun forColumn(columnName: String) { + val mapping: SqliteCustomTypeConverter = typeOf() to { it } + columnMappings[columnName] = mapping + } + } /** * Represents the Sqlite database type. @@ -14,18 +209,308 @@ import kotlin.reflect.full.withNullability * This class provides methods to convert data from a ResultSet to the appropriate type for Sqlite, * and to generate the corresponding column schema. * - * Use [customTypesMap] to register custom types and provide the corresponding [KType] for each one. - * [KType] must correspond to JDBC actual type. - * Even for default Sqlite types, you can override the actual [KType] after reading the column - * by providing a custom type in [customTypesMap]. + * Two levels of custom overrides are supported, with the following resolution order (first match + * wins): + * 1. [customColumnsMap] — keyed by the **column name**. Use this to override a specific column, + * regardless of its declared type. + * 2. [customTypesMap] — keyed by the declared **SQL type name** (as written in `CREATE TABLE`). + * Use this to override every column that shares a declared type. + * 3. The built-in SQLite conversion for BOOLEAN, DATE, DATETIME, TIME, TIMESTAMP, DECIMAL, + * NUMERIC. + * 4. The base `DbType` mapping. + * + * Both maps take a [SqliteCustomTypeConverter] lambda that returns both the target [KType] and the + * converted value. */ -public class Sqlite(public val customTypesMap: Map = mapOf()) : DbType("sqlite") { +public class Sqlite( + public val customTypesMap: Map> = emptyMap(), + public val customColumnsMap: Map> = emptyMap(), +) : DbType("sqlite") { override val driverClassName: String get() = "org.sqlite.JDBC" - override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType = - customTypesMap[tableColumnMetadata.sqlTypeName]?.withNullability(tableColumnMetadata.isNullable) - ?: super.getExpectedJdbcType(tableColumnMetadata) + // SQLite is dynamically typed with only five storage classes (NULL, INTEGER, REAL, TEXT, BLOB). + // The declared column type is a hint (type affinity), so a column declared DATE/DATETIME/ + // TIMESTAMP/DECIMAL/NUMERIC can actually hold a String, Integer, or Double at runtime. + // + // - For DATE / DATETIME / TIME / TIMESTAMP we detect the declared type by name (Xerial changes + // the reported `jdbcType` based on the stored value's storage class — e.g. a DATE column + // with a REAL value is reported as `Types.FLOAT`) and return an idiomatic Kotlin date-time + // type (`kotlinx.datetime.LocalDate` / `LocalDateTime` / `LocalTime` / `kotlin.time.Instant`). + // The raw storage value is converted in `preprocessValue`. + // - For DECIMAL and NUMERIC, we trust the driver-reported `javaClassName` (the actual stored + // value's class): a NUMERIC column can hold a genuinely mixed set of ints and doubles, and + // there's no natural "canonical" numeric type to promote them to. + override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType { + val nullable = tableColumnMetadata.isNullable + // Column-name override wins over type-name override. Column nullability from the + // schema is always applied on top of the KType the user declared. + customMappingFor(tableColumnMetadata)?.let { (kType, _) -> + return kType + } + val declaredUpper = tableColumnMetadata.sqlTypeName.uppercase() + + // Date/time detection by declared type name substring matching + when { + "DATETIME" in declaredUpper -> + return typeOf().withNullability(nullable) + + "TIMESTAMP" in declaredUpper -> + return typeOf().withNullability(nullable) + + "DATE" in declaredUpper -> + return typeOf().withNullability(nullable) + + "TIME" in declaredUpper -> + return typeOf().withNullability(nullable) + } + + // Numeric ambiguity: trust storage class. + when (tableColumnMetadata.jdbcType) { + Types.DECIMAL, Types.NUMERIC -> + javaClassNameToKType(tableColumnMetadata.javaClassName)?.let { + return it.withNullability(nullable) + } + } + + return super.getExpectedJdbcType(tableColumnMetadata) + } + + // For DECIMAL/NUMERIC we already resolved the DataFrame type from the storage class in + // getExpectedJdbcType, so we keep that as-is. For other types we let the base decide + // (base maps TIMESTAMP → Instant, BINARY(UUID) → Uuid, etc.). + override fun getPreprocessedValueType(tableColumnMetadata: TableColumnMetadata, expectedJdbcType: KType): KType = + when (tableColumnMetadata.jdbcType) { + Types.DECIMAL, Types.NUMERIC -> expectedJdbcType + else -> super.getPreprocessedValueType(tableColumnMetadata, expectedJdbcType) + } + + // Converts the raw stored value into the type the DataFrame column expects. Dispatched by + // the target Kotlin type — this uniformly handles the SQLite "declared type ≠ storage class" + // mismatch for BOOLEAN, DATE, DATETIME, TIME, and TIMESTAMP. Custom mappings from + // `customTypesMap` take precedence and completely replace the built-in conversion. + override fun preprocessValue( + value: J, + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + expectedPreprocessedValueType: KType, + ): D { + customMappingFor(tableColumnMetadata)?.let { (_, convert) -> + @Suppress("UNCHECKED_CAST") + return convert(value) as D + } + val target = expectedPreprocessedValueType.classifier + @Suppress("UNCHECKED_CAST") + return when (target) { + Boolean::class -> convertToBoolean(value, tableColumnMetadata) as D + + Instant::class -> convertToInstant(value, tableColumnMetadata) as D + + LocalDate::class -> convertToLocalDate(value, tableColumnMetadata) as D + + LocalDateTime::class -> convertToLocalDateTime(value, tableColumnMetadata) as D + + LocalTime::class -> convertToLocalTime(value, tableColumnMetadata) as D + + // DECIMAL / NUMERIC (or any other type resolved via storage class): return as-is. + else -> { + if (tableColumnMetadata.jdbcType == Types.DECIMAL || + tableColumnMetadata.jdbcType == Types.NUMERIC + ) { + return value as D + } + super.preprocessValue( + value = value, + tableColumnMetadata = tableColumnMetadata, + expectedJdbcType = expectedJdbcType, + expectedPreprocessedValueType = expectedPreprocessedValueType, + ) + } + } + } + + /** + * Resolves the effective custom mapping for a column: by-name takes precedence over by-type. + * The returned mapping is cast so the converter accepts `Any?` — at runtime the user's + * declared `T` is erased and the raw value is passed straight through. + */ + @Suppress("UNCHECKED_CAST") + private fun customMappingFor(tableColumnMetadata: TableColumnMetadata): Pair Any?>? = + ( + customColumnsMap[tableColumnMetadata.name] + ?: customTypesMap[tableColumnMetadata.sqlTypeName] + ) as Pair Any?>? + + private fun javaClassNameToKType(className: String): KType? = + when (className) { + "java.lang.String" -> typeOf() + "java.lang.Integer" -> typeOf() + "java.lang.Long" -> typeOf() + "java.lang.Double" -> typeOf() + "[B" -> typeOf() + else -> null + } + + // ---------- storage class → target conversions ---------- + // + // The Xerial SQLite JDBC driver's `ResultSet.getObject(int)` inspects the runtime storage + // class (never the declared column type) and returns exactly one of: + // + // INTEGER → java.lang.Integer (values in Int range) + // INTEGER → java.lang.Long (values outside Int range) + // REAL → java.lang.Double + // TEXT → java.lang.String + // BLOB → byte[] (i.e. `ByteArray`) + // NULL → null + // + // No other type ever arrives here. + // Consequently, each `convertToX` below only branches on `null`, `Int`, `Long`, `Double`, + // `String`, `ByteArray` where they make sense. + + private fun convertToBoolean(value: Any?, meta: TableColumnMetadata): Boolean? = + when (value) { + null -> null + + // SQLite convention: booleans are stored as INTEGER (0/1). Any non-zero → true. + is Int -> value != 0 + + is Long -> value != 0L + + // Some users may store booleans as REAL. Any non-zero → true. + is Double -> value != 0.0 + + is String -> when (value.trim().lowercase()) { + "true", "1", "yes", "y", "t" -> true + "false", "0", "no", "n", "f" -> false + else -> parseFailure(value, "Boolean", meta) + } + + else -> unsupportedConversion(value, "Boolean", meta) + } + + private fun convertToInstant(value: Any?, meta: TableColumnMetadata): Instant? = + when (value) { + null -> null + + // SQLite convention: INTEGER = Unix seconds since 1970-01-01 UTC. + is Int -> Instant.fromEpochSeconds(value.toLong()) + + is Long -> Instant.fromEpochSeconds(value) + + // SQLite convention: REAL = Julian day (days since -4713-11-24 12:00 UTC). + is Double -> julianDayToInstant(value) + + is String -> parseStringAsInstant(value, meta) + + else -> unsupportedConversion(value, "kotlin.time.Instant", meta) + } + + private fun convertToLocalDate(value: Any?, meta: TableColumnMetadata): LocalDate? = + when (value) { + null -> null + + is Int -> instantToLocalDate(Instant.fromEpochSeconds(value.toLong())) + + is Long -> instantToLocalDate(Instant.fromEpochSeconds(value)) + + // SQLite convention: REAL = Julian day (days since -4713-11-24 12:00 UTC). + is Double -> instantToLocalDate(julianDayToInstant(value)) + + is String -> parseStringAsLocalDate(value, meta) + + else -> unsupportedConversion(value, "kotlinx.datetime.LocalDate", meta) + } + + private fun convertToLocalDateTime(value: Any?, meta: TableColumnMetadata): LocalDateTime? = + when (value) { + null -> null + is Int -> instantToLocalDateTime(Instant.fromEpochSeconds(value.toLong())) + is Long -> instantToLocalDateTime(Instant.fromEpochSeconds(value)) + is Double -> instantToLocalDateTime(julianDayToInstant(value)) + is String -> parseStringAsLocalDateTime(value, meta) + else -> unsupportedConversion(value, "kotlinx.datetime.LocalDateTime", meta) + } + + private fun convertToLocalTime(value: Any?, meta: TableColumnMetadata): LocalTime? = + when (value) { + null -> null + + // Interpret INTEGER as seconds since midnight. + is Int -> JavaLocalTime.ofSecondOfDay(value.toLong()).toKotlinLocalTime() + + is Long -> JavaLocalTime.ofSecondOfDay(value).toKotlinLocalTime() + + is String -> parseStringAsLocalTime(value, meta) + + else -> unsupportedConversion(value, "kotlinx.datetime.LocalTime", meta) + } + + private fun parseStringAsInstant(s: String, meta: TableColumnMetadata): Instant { + // Try, in order: full ISO instant, LocalDateTime (T or space separator), LocalDate. + runCatching { return Instant.parse(s) } + val normalised = s.replace(' ', 'T') + runCatching { + return JavaLocalDateTime.parse(normalised).toInstant(ZoneOffset.UTC).toKotlinInstant() + } + runCatching { + return JavaLocalDate.parse(s).atStartOfDay(ZoneOffset.UTC).toInstant().toKotlinInstant() + } + parseFailure(s, "an ISO 8601 date/time", meta) + } + + private fun parseStringAsLocalDate(s: String, meta: TableColumnMetadata): LocalDate { + runCatching { return JavaLocalDate.parse(s).toKotlinLocalDate() } + // Also accept full date-time / instant strings — truncate to the date portion. + runCatching { return instantToLocalDate(parseStringAsInstant(s, meta)) } + parseFailure(s, "an ISO 8601 date", meta) + } + + private fun parseStringAsLocalDateTime(s: String, meta: TableColumnMetadata): LocalDateTime { + val normalised = s.replace(' ', 'T') + runCatching { return JavaLocalDateTime.parse(normalised).toKotlinLocalDateTime() } + runCatching { return JavaLocalDate.parse(s).atStartOfDay().toKotlinLocalDateTime() } + // As a last resort, accept ISO instant strings and convert to LocalDateTime at UTC. + runCatching { return instantToLocalDateTime(Instant.parse(s)) } + parseFailure(s, "an ISO 8601 date-time", meta) + } + + private fun parseStringAsLocalTime(s: String, meta: TableColumnMetadata): LocalTime { + runCatching { return JavaLocalTime.parse(s).toKotlinLocalTime() } + parseFailure(s, "an ISO 8601 time", meta) + } + + private fun instantToLocalDate(instant: Instant): LocalDate = + java.time.Instant.ofEpochSecond(instant.epochSeconds) + .atZone(ZoneOffset.UTC) + .toLocalDate() + .toKotlinLocalDate() + + private fun instantToLocalDateTime(instant: Instant): LocalDateTime = + java.time.Instant.ofEpochSecond(instant.epochSeconds) + .atZone(ZoneOffset.UTC) + .toLocalDateTime() + .toKotlinLocalDateTime() + + private fun julianDayToInstant(julianDay: Double): Instant { + val epochSeconds = ((julianDay - JULIAN_DAY_UNIX_EPOCH) * SECONDS_PER_DAY).toLong() + return Instant.fromEpochSeconds(epochSeconds) + } + + private fun unsupportedConversion(value: Any?, target: String, meta: TableColumnMetadata): Nothing = + conversionError( + "cannot convert value of type ${value?.javaClass?.name} to $target", + meta, + ) + + private fun parseFailure(value: Any?, target: String, meta: TableColumnMetadata): Nothing = + conversionError("cannot parse '$value' as $target", meta) + + private fun conversionError(problem: String, meta: TableColumnMetadata): Nothing = + error( + "SQLite: $problem from column '${meta.name}' (declared '${meta.sqlTypeName}'). " + + "Register a custom converter for this type or column via " + + "`Sqlite.withCustomConverters { } to override the built-in mapping.", + ) override fun isSystemTable(tableMetadata: TableMetadata): Boolean = tableMetadata.name.startsWith("sqlite_") @@ -46,8 +531,59 @@ public class Sqlite(public val customTypesMap: Map = mapOf()) : D } public companion object { + /** + * Default [Sqlite] instance with no custom overrides. + * + * Uses built-in SQLite conversions for most common types (`BOOLEAN`, `DATE`, `DATETIME`, `TIME`, `TIMESTAMP`, + * `DECIMAL`, `NUMERIC`). + * + * See [Sqlite.withCustomConverters] to register custom type converters. + */ public val default: Sqlite = Sqlite() - public fun withCustomTypes(customTypesMap: Map): Sqlite = Sqlite(customTypesMap) + /** + * Builds a [Sqlite] with custom type converters registered via a [SqliteCustomConvertersBuilder] DSL block. + * + * * use [forType][SqliteCustomConvertersBuilder.forType] to register a converter + * keyed by the declared SQL type name (as written in `CREATE TABLE`); + * * use [forColumn][SqliteCustomConvertersBuilder.forColumn] to register a converter keyed by column name + * (takes precedence over [forType][SqliteCustomConvertersBuilder.forType] for the named column). + * + * Both DSL functions accept two generic type parameters: + * - `T` — the storage class of the raw stored value [String], [Int], [Long], [Double], + * or [ByteArray] or their nullable variants). + * - `R` — the target Kotlin type of the resulting DataFrame column; must be reified. + * + * The converting lambda receives the raw value and returns the converted result; + * + * ### Example + * ``` + * val format = LocalDateTime.Format { + * year(); char('-'); monthNumber(); char('-'); day() + * char(' ') + * hour(); char(':'); minute(); char(':'); second() + * chars(" UTC") + * } + * + * val sqliteCustom = Sqlite.withCustomConverters { + * // Every column declared with "MY_DATETIME" type is parsed from custom text into Instant. + * forType("MY_DATETIME") { raw: String? -> + * raw?.let { LocalDateTime.parse(it, format).toInstant(TimeZone.UTC) } + * } + * // The "ratio" column overrides its declared type — read as Double. + * forColumn("ratio") { raw: String -> raw.toDouble() } + * } + * + * val df = DataFrame.readSqlTable(connection, "events", dbType = sqliteCustom) + * ``` + */ + public fun withCustomConverters(block: SqliteCustomConvertersBuilder.() -> Unit): Sqlite { + val builder = SqliteCustomConvertersBuilder().also { it.block() } + return Sqlite(builder.typeMappings, builder.columnMappings) + } + + // Julian day number at Unix epoch (1970-01-01 00:00 UTC). + private const val JULIAN_DAY_UNIX_EPOCH = 2440587.5 + private const val SECONDS_PER_DAY = 86_400 } } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt index 56758ec27b..e483216da3 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt @@ -3,113 +3,585 @@ package org.jetbrains.kotlinx.dataframe.io.db import io.kotest.matchers.shouldBe -import org.jetbrains.kotlinx.dataframe.io.db.JdbcTypesTest.MySqlDBTypes.BIGINT_UNSIGNED import org.junit.Test import org.junit.experimental.runners.Enclosed import org.junit.runner.RunWith +import org.postgresql.geometric.PGbox +import org.postgresql.geometric.PGcircle +import org.postgresql.geometric.PGline +import org.postgresql.geometric.PGlseg +import org.postgresql.geometric.PGpath +import org.postgresql.geometric.PGpoint +import org.postgresql.geometric.PGpolygon +import org.postgresql.util.PGInterval +import org.postgresql.util.PGmoney +import java.math.BigDecimal import java.math.BigInteger +import java.sql.Blob +import java.sql.Clob +import java.sql.NClob +import java.sql.Ref +import java.sql.RowId +import java.sql.SQLXML +import java.sql.Time +import java.sql.Types +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.util.Date import kotlin.reflect.KType +import kotlin.reflect.full.withNullability import kotlin.reflect.typeOf +import kotlin.time.Instant +import kotlin.uuid.Uuid +import kotlinx.datetime.LocalDate as KotlinLocalDate +import kotlinx.datetime.LocalDateTime as KotlinLocalDateTime +import kotlinx.datetime.LocalTime as KotlinLocalTime -// TODO: complete and enhance (#1736) +/** + * Non-integration tests for [DbType.getExpectedJdbcType] and related type-mapping logic. + * + * Each DB owns a [TypeMapping] list that acts as the source of truth for its SQL → Kotlin type + * mapping. The list is exercised for both nullable and non-nullable columns. + * + * See https://github.com/Kotlin/dataframe/issues/1736. + */ @RunWith(Enclosed::class) class JdbcTypesTest { - abstract class ColumnType( - val sqlTypeName: String, - val jdbcType: Int, - val javaClassName: String, - val isNullable: Boolean, - val expectedKotlinType: KType, - ) { - fun mockkColMetaData() = - TableColumnMetadata( - "name", - sqlTypeName, - jdbcType, - 10, - javaClassName, - isNullable, - ) + class DefaultDbTypeTypes { + + // A concrete DbType whose behavior is exactly the default one from the base class. + private object DefaultDbType : DbType("default") { + override val driverClassName: String get() = "does.not.matter" + + override fun isSystemTable(tableMetadata: TableMetadata): Boolean = false + + override fun buildTableMetadata(tables: java.sql.ResultSet): TableMetadata = + TableMetadata("t", null, null) + } + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(DefaultDbType, commonJdbcTypeMappings) + } + + @Test + fun `TIMESTAMP with LocalDateTime driver class maps to java_time_LocalDateTime`() { + assertMappings(DefaultDbType, listOf(timestampAsLocalDateTime)) + } + + @Test + fun `BINARY with UUID driver class maps to UUID`() { + assertMappings(DefaultDbType, listOf(binaryAsUuid)) + } + + @Test + fun `Types_OTHER with byte array javaClassName maps to ByteArray`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "[B", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `Types_OTHER with generic Object maps to Any`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + } } - class MariaDBTypes { + class MariaDbTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MariaDb, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MariaDB-specific overrides`() { + assertMappings(MariaDb, mariaDbSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MariaDb.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MariaDb) } } - class MySqlDBTypes { + class MySqlTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MySql, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MySQL-specific overrides`() { + assertMappings(MySql, mySqlSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MySql.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MySql) } } + /** + * SQLite is dynamically typed: it has only 5 storage classes (NULL, INTEGER, REAL, TEXT, BLOB) + * and picks one per row based on the value, guided by "type affinity" derived from the + * declared column type. The Xerial JDBC driver reports metadata based on the actual stored + * value, so `getExpectedJdbcType` sees driver-specific `jdbcType` and `javaClassName` combos + * that differ from other databases. The tests below reflect that. + */ class SqliteTypes { - // Taken from #964 - - object LONGVARCHAR_1 : ColumnType( - "LONGVARCHAR", - -2, - "java.lang.Object", - false, - typeOf(), - ) - - object LONGVARCHAR_2 : ColumnType( - "LONGVARCHAR", - 12, - "java.lang.String", - true, - typeOf(), - ) - - val customTypes: List = listOf( - LONGVARCHAR_1, - LONGVARCHAR_2, - ) - - @Test - fun `SQLite custom types`() { - val sqliteCustom = Sqlite( - mapOf("LONGVARCHAR" to typeOf()), - ) - customTypes.forEach { type -> - sqliteCustom.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType + @Test + fun `INTEGER affinity — declared int-like types map to Int or Long`() { + assertMappings(Sqlite.default, sqliteIntegerAffinityMappings) + } + + @Test + fun `REAL affinity — declared real-like types map to Double`() { + assertMappings(Sqlite.default, sqliteRealAffinityMappings) + } + + @Test + fun `TEXT affinity — declared text-like types map to String`() { + assertMappings(Sqlite.default, sqliteTextAffinityMappings) + } + + @Test + fun `BLOB affinity — declared BLOB maps to ByteArray`() { + assertMappings(Sqlite.default, sqliteBlobAffinityMappings) + } + + @Test + fun `NUMERIC affinity — declared numeric-like types map by declared type`() { + assertMappings(Sqlite.default, sqliteNumericAffinityMappings) + } + + @Test + fun `BOOLEAN declared type resolves to Boolean`() { + // Xerial reports Types.BOOLEAN metadata even though values are stored as INTEGER; + // the schema type is Boolean, and `getValueFromResultSet` converts each row via + // `rs.getBoolean` (see Sqlite.getValueFromResultSet). + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "BOOLEAN", + jdbcType = Types.BOOLEAN, + javaClassName = "java.lang.Integer", + isNullable = false, + ), + ) shouldBe typeOf() + + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "BOOLEAN", + jdbcType = Types.BOOLEAN, + javaClassName = "java.lang.Integer", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unrecognised declared type is treated by NUMERIC affinity`() { + // For an unknown declared type Xerial applies NUMERIC affinity and reports the + // jdbcType of the actual stored value; for a text sample that is VARCHAR. + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "CUSTOM_TYPE", + jdbcType = Types.VARCHAR, + javaClassName = "java.lang.String", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `customTypesMap overrides the default mapping by SQL type name`() { + val custom = Sqlite.withCustomConverters { + forType("INTEGER") + forType("MY_TYPE") } + // INTEGER is normally Int, but is overridden to Long + custom.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "INTEGER", + jdbcType = Types.INTEGER, + javaClassName = "java.lang.Integer", + isNullable = true, + ), + ) shouldBe typeOf() + // Custom type name is respected as-is + custom.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MY_TYPE", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(Sqlite.default) } } + + class PostgreSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(PostgreSql, commonJdbcTypeMappings) + } + + @Test + fun `PGobject types map to their PGobject Kotlin types`() { + assertMappings(PostgreSql, postgreSqlSpecificMappings) + } + + @Test + fun `PGobject lookup is case-insensitive`() { + PostgreSql.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "POINT", + jdbcType = Types.OTHER, + javaClassName = "org.postgresql.geometric.PGpoint", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(PostgreSql) + } + } + + class MsSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MsSql, commonJdbcTypeMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MsSql) + } + } + + class H2Types { + + @Test + fun `Regular mode uses default type mappings`() { + assertMappings(H2(H2.Mode.Regular), commonJdbcTypeMappings) + } + + @Test + fun `MySql mode delegates to MySQL-specific overrides`() { + assertMappings(H2(H2.Mode.MySql), mySqlSpecificMappings) + } + + @Test + fun `MariaDb mode delegates to MariaDB-specific overrides`() { + assertMappings(H2(H2.Mode.MariaDb), mariaDbSpecificMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(H2(H2.Mode.Regular)) + } + } +} + +// -------------------- Type mapping model & helpers -------------------- + +/** + * A single row in the JDBC → Kotlin type mapping table. + * + * @property sqlTypeName the human-readable SQL type name (e.g. "BIGINT") + * @property jdbcType a constant from [java.sql.Types] + * @property javaClassName the JDBC-reported class name for this column (as returned by + * [java.sql.ResultSetMetaData.getColumnClassName]) + * @property expectedType the expected non-nullable Kotlin type + */ +internal data class TypeMapping( + val sqlTypeName: String, + val jdbcType: Int, + val javaClassName: String, + val expectedType: KType, +) + +internal const val UNKNOWN_JDBC_TYPE: Int = -9999 + +/** + * Test helper that constructs a [TableColumnMetadata] with sensible defaults. + * Not a mock — a lightweight factory to keep test call sites readable. + */ +internal fun createColumnMetadata( + name: String = "col", + sqlTypeName: String, + jdbcType: Int, + size: Int = 10, + javaClassName: String, + isNullable: Boolean, +): TableColumnMetadata = + TableColumnMetadata( + name = name, + sqlTypeName = sqlTypeName, + jdbcType = jdbcType, + size = size, + javaClassName = javaClassName, + isNullable = isNullable, + ) + +/** + * Verifies each mapping resolves correctly for both nullable and non-nullable columns. + * Runs the full type-resolution pipeline (`getExpectedJdbcType` → `getPreprocessedValueType`) + * and compares against the **final DataFrame column type**, which is what the reference + * documentation describes. + */ +internal fun assertMappings(dbType: DbType, mappings: List) { + mappings.forEach { m -> + listOf(false, true).forEach { isNullable -> + val meta = createColumnMetadata( + sqlTypeName = m.sqlTypeName, + jdbcType = m.jdbcType, + javaClassName = m.javaClassName, + isNullable = isNullable, + ) + val jdbcType = dbType.getExpectedJdbcType(meta) + val finalType = dbType.getPreprocessedValueType(meta, jdbcType) + finalType shouldBe m.expectedType.withNullability(isNullable) + } + } +} + +internal fun assertUnknownMapsToString(dbType: DbType) { + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() } + +// -------------------- Type mapping tables -------------------- + +/** + * The default SQL → Kotlin type mapping applied by [DbType]. + * Every DB that does not override the given entry falls through to this table. + */ +internal val commonJdbcTypeMappings: List = listOf( + TypeMapping("BIT", Types.BIT, "java.lang.Boolean", typeOf()), + TypeMapping("TINYINT", Types.TINYINT, "java.lang.Integer", typeOf()), + TypeMapping("SMALLINT", Types.SMALLINT, "java.lang.Integer", typeOf()), + TypeMapping("INTEGER", Types.INTEGER, "java.lang.Integer", typeOf()), + TypeMapping("BIGINT", Types.BIGINT, "java.lang.Long", typeOf()), + TypeMapping("FLOAT", Types.FLOAT, "java.lang.Float", typeOf()), + TypeMapping("REAL", Types.REAL, "java.lang.Float", typeOf()), + TypeMapping("DOUBLE", Types.DOUBLE, "java.lang.Double", typeOf()), + TypeMapping("NUMERIC", Types.NUMERIC, "java.math.BigDecimal", typeOf()), + TypeMapping("DECIMAL", Types.DECIMAL, "java.math.BigDecimal", typeOf()), + TypeMapping("CHAR", Types.CHAR, "java.lang.String", typeOf()), + TypeMapping("VARCHAR", Types.VARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGVARCHAR", Types.LONGVARCHAR, "java.lang.String", typeOf()), + TypeMapping("NCHAR", Types.NCHAR, "java.lang.String", typeOf()), + TypeMapping("NVARCHAR", Types.NVARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGNVARCHAR", Types.LONGNVARCHAR, "java.lang.String", typeOf()), + TypeMapping("DATE", Types.DATE, "java.sql.Date", typeOf()), + TypeMapping("TIME", Types.TIME, "java.sql.Time", typeOf