From bf8ce52904698bf0231f75456d8bfca3b45bcd2b Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Thu, 23 Jul 2026 15:36:37 +0400 Subject: [PATCH 1/7] Handle SQLite BOOLEAN columns correctly and add tests --- .../kotlinx/dataframe/io/db/Sqlite.kt | 21 ++++++++ .../kotlinx/dataframe/io/sqliteTest.kt | 53 +++++++++++++++++-- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt index d1e897ce3a..2716638dc9 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt @@ -5,6 +5,7 @@ import org.sqlite.SQLiteConfig import java.sql.Connection import java.sql.DriverManager import java.sql.ResultSet +import java.sql.Types import kotlin.reflect.KType import kotlin.reflect.full.withNullability @@ -27,6 +28,26 @@ public class Sqlite(public val customTypesMap: Map = mapOf()) : D customTypesMap[tableColumnMetadata.sqlTypeName]?.withNullability(tableColumnMetadata.isNullable) ?: super.getExpectedJdbcType(tableColumnMetadata) + // SQLite has no native BOOLEAN storage class — values are kept as INTEGER. + // The Xerial JDBC driver reports Types.BOOLEAN in metadata but returns Integer from getObject. + override fun getValueFromResultSet( + rs: ResultSet, + columnIndex: Int, + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + ): J { + val idx = columnIndex + 1 + return when (tableColumnMetadata.jdbcType) { + Types.BOOLEAN, Types.BIT -> { + val value = rs.getBoolean(idx) + @Suppress("UNCHECKED_CAST") + (if (rs.wasNull()) null else value) as J + } + + else -> super.getValueFromResultSet(rs, columnIndex, tableColumnMetadata, expectedJdbcType) + } + } + override fun isSystemTable(tableMetadata: TableMetadata): Boolean = tableMetadata.name.startsWith("sqlite_") override fun buildTableMetadata(tables: ResultSet): TableMetadata = diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt index e0cb852a8d..1f5e12ba4e 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt @@ -14,6 +14,7 @@ import java.io.File import java.nio.file.Files import java.sql.Connection import java.sql.DriverManager +import java.sql.Types import kotlin.reflect.typeOf @DataSchema @@ -47,6 +48,13 @@ interface CustomerOrderSQLite { val orderDetails: ByteArray? } +@DataSchema +interface FlagSQLite { + val id: Int? + val enabled: Boolean + val optional: Boolean? +} + class SqliteTest { companion object { private lateinit var connection: Connection @@ -93,6 +101,29 @@ class SqliteTest { connection.createStatement().execute(createOrderTableQuery) + @Language("SQL") + val createFlagsTableQuery = """ + CREATE TABLE Flags ( + id INTEGER PRIMARY KEY, + enabled BOOLEAN NOT NULL, + optional BOOLEAN + ) + """ + + connection.createStatement().execute(createFlagsTableQuery) + + connection.prepareStatement("INSERT INTO Flags (enabled, optional) VALUES (?, ?)").use { + it.setBoolean(1, true) + it.setBoolean(2, false) + it.executeUpdate() + } + + connection.prepareStatement("INSERT INTO Flags (enabled, optional) VALUES (?, ?)").use { + it.setBoolean(1, false) + it.setNull(2, Types.BOOLEAN) + it.executeUpdate() + } + val profilePicture = "SampleProfilePictureData".toByteArray() val orderDetails = "OrderDetailsData".toByteArray() @@ -247,18 +278,34 @@ class SqliteTest { @Test fun `read from all tables`() { - val dataframes = DataFrame.readAllSqlTables(connection).values.toList() + val dataframes = DataFrame.readAllSqlTables(connection) - val customerDf = dataframes[0].cast() + val customerDf = dataframes.getValue("Customers").cast() customerDf.rowsCount() shouldBe 2 customerDf.filter { "age"()?.let { it > 30 } ?: false }.rowsCount() shouldBe 1 customerDf[0][1] shouldBe "John Doe" - val orderDf = dataframes[1].cast() + val orderDf = dataframes.getValue("Orders").cast() orderDf.rowsCount() shouldBe 2 orderDf.filter { "totalAmount"() > 200 }.rowsCount() shouldBe 1 orderDf[0][1] shouldBe null } + + @Test + fun `read boolean column`() { + val flagsTableName = "Flags" + val df = DataFrame.readSqlTable(connection, flagsTableName).cast() + + df.rowsCount() shouldBe 2 + df["enabled"][0] shouldBe true + df["enabled"][1] shouldBe false + df["optional"][0] shouldBe false + df["optional"][1] shouldBe null + + val schema = DataFrameSchema.readSqlTable(connection, flagsTableName) + schema.columns["enabled"]!!.type shouldBe typeOf() + schema.columns["optional"]!!.type shouldBe typeOf() + } } From 71ee483522a397b294c23dee83dcf5b51a8ebf88 Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Fri, 24 Jul 2026 16:58:00 +0400 Subject: [PATCH 2/7] Support idiomatic Kotlin type conversions for SQLite date/time columns and add tests --- .../kotlinx/dataframe/io/db/Sqlite.kt | 258 +++++++++++++++++- .../kotlinx/dataframe/io/sqliteTest.kt | 58 ++++ 2 files changed, 311 insertions(+), 5 deletions(-) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt index 2716638dc9..4c26c847f4 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt @@ -1,13 +1,28 @@ package org.jetbrains.kotlinx.dataframe.io.db +import kotlinx.datetime.toKotlinLocalDate +import kotlinx.datetime.toKotlinLocalDateTime +import kotlinx.datetime.toKotlinLocalTime import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig import org.sqlite.SQLiteConfig import java.sql.Connection import java.sql.DriverManager import java.sql.ResultSet +import java.sql.Timestamp import java.sql.Types +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.ZoneOffset import kotlin.reflect.KType import kotlin.reflect.full.withNullability +import kotlin.reflect.typeOf +import kotlin.time.Instant +import kotlin.time.toKotlinInstant +import java.util.Date as JavaDate +import kotlinx.datetime.LocalDate as KotlinLocalDate +import kotlinx.datetime.LocalDateTime as KotlinLocalDateTime +import kotlinx.datetime.LocalTime as KotlinLocalTime /** * Represents the Sqlite database type. @@ -24,12 +39,56 @@ public class Sqlite(public val customTypesMap: Map = mapOf()) : D override val driverClassName: String get() = "org.sqlite.JDBC" - override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType = - customTypesMap[tableColumnMetadata.sqlTypeName]?.withNullability(tableColumnMetadata.isNullable) - ?: super.getExpectedJdbcType(tableColumnMetadata) + // SQLite is dynamically typed with only five storage classes (NULL, INTEGER, REAL, TEXT, BLOB). + // The declared column type is a hint (type affinity), so a column declared DATE/DATETIME/ + // TIMESTAMP/DECIMAL/NUMERIC can actually hold a String, Integer, or Double at runtime. + // + // - For DATE / DATETIME / TIME / TIMESTAMP we detect the declared type by name (Xerial changes + // the reported `jdbcType` based on the stored value's storage class — e.g. a DATE column + // with a REAL value is reported as `Types.FLOAT`) and return an idiomatic Kotlin date-time + // type (`kotlinx.datetime.LocalDate` / `LocalDateTime` / `LocalTime` / `kotlin.time.Instant`). + // The raw storage value is converted in `preprocessValue`. + // - For DECIMAL and NUMERIC, we trust the driver-reported `javaClassName` (the actual stored + // value's class): a NUMERIC column can hold a genuinely mixed set of ints and doubles, and + // there's no natural "canonical" numeric type to promote them to. + override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType { + customTypesMap[tableColumnMetadata.sqlTypeName]?.let { + return it.withNullability(tableColumnMetadata.isNullable) + } + val nullable = tableColumnMetadata.isNullable + val declaredUpper = tableColumnMetadata.sqlTypeName.uppercase() + + // Date/time detection by declared type name (SQLite type affinity via substring + // matching). Order matters: DATETIME must be checked before DATE/TIME; TIMESTAMP before + // TIME. + when { + "DATETIME" in declaredUpper -> + return typeOf().withNullability(nullable) + + "TIMESTAMP" in declaredUpper -> + return typeOf().withNullability(nullable) + + "DATE" in declaredUpper -> + return typeOf().withNullability(nullable) + + "TIME" in declaredUpper -> + return typeOf().withNullability(nullable) + } + + // Numeric ambiguity: trust storage class. + when (tableColumnMetadata.jdbcType) { + Types.DECIMAL, Types.NUMERIC -> + javaClassNameToKType(tableColumnMetadata.javaClassName)?.let { + return it.withNullability(nullable) + } + } + + return super.getExpectedJdbcType(tableColumnMetadata) + } - // SQLite has no native BOOLEAN storage class — values are kept as INTEGER. - // The Xerial JDBC driver reports Types.BOOLEAN in metadata but returns Integer from getObject. + // Reads a raw value from the ResultSet. + // - BOOLEAN/BIT: SQLite stores booleans as INTEGER; use rs.getBoolean so the value matches + // the Boolean schema type produced by getExpectedJdbcType. override fun getValueFromResultSet( rs: ResultSet, columnIndex: Int, @@ -48,6 +107,191 @@ public class Sqlite(public val customTypesMap: Map = mapOf()) : D } } + // For DECIMAL/NUMERIC we already resolved the DataFrame type from the storage class in + // getExpectedJdbcType, so we keep that as-is. For other types we let the base decide + // (base maps TIMESTAMP → Instant, BINARY(UUID) → Uuid, etc.). + override fun getPreprocessedValueType( + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + ): KType = + when (tableColumnMetadata.jdbcType) { + Types.DECIMAL, Types.NUMERIC -> expectedJdbcType + else -> super.getPreprocessedValueType(tableColumnMetadata, expectedJdbcType) + } + + // Converts the raw stored value into the type the DataFrame column expects. Dispatched by + // the target Kotlin type, so a user's `customTypesMap` opt-out (e.g. forcing DATETIME → + // String) automatically skips conversion. Unsupported combinations throw with a clear + // message. + override fun preprocessValue( + value: J, + tableColumnMetadata: TableColumnMetadata, + expectedJdbcType: KType, + expectedPreprocessedValueType: KType, + ): D { + val target = expectedPreprocessedValueType.classifier + @Suppress("UNCHECKED_CAST") + return when (target) { + Instant::class -> convertToInstant(value, tableColumnMetadata) as D + KotlinLocalDate::class -> convertToLocalDate(value, tableColumnMetadata) as D + KotlinLocalDateTime::class -> convertToLocalDateTime(value, tableColumnMetadata) as D + KotlinLocalTime::class -> convertToLocalTime(value, tableColumnMetadata) as D + + // DECIMAL / NUMERIC (or any other type resolved via storage class): return as-is. + else -> { + if (tableColumnMetadata.jdbcType == Types.DECIMAL || + tableColumnMetadata.jdbcType == Types.NUMERIC + ) { + return value as D + } + super.preprocessValue( + value = value, + tableColumnMetadata = tableColumnMetadata, + expectedJdbcType = expectedJdbcType, + expectedPreprocessedValueType = expectedPreprocessedValueType, + ) + } + } + } + + private fun javaClassNameToKType(className: String): KType? = + when (className) { + "java.lang.String" -> typeOf() + "java.lang.Integer" -> typeOf() + "java.lang.Long" -> typeOf() + "java.lang.Double" -> typeOf() + "java.lang.Float" -> typeOf() + "java.lang.Boolean" -> typeOf() + "[B" -> typeOf() + else -> null + } + + // ---------- date/time storage → target conversions ---------- + + private fun convertToInstant(value: Any?, meta: TableColumnMetadata): Instant? = + when (value) { + null -> null + is Timestamp -> value.toInstant().toKotlinInstant() + is LocalDateTime -> value.toInstant(ZoneOffset.UTC).toKotlinInstant() + is JavaDate -> value.toInstant().toKotlinInstant() + // SQLite convention: INTEGER = Unix seconds since 1970-01-01 UTC. + is Long -> Instant.fromEpochSeconds(value) + is Int -> Instant.fromEpochSeconds(value.toLong()) + // SQLite convention: REAL = Julian day (days since -4713-11-24 12:00 UTC). + is Double -> julianDayToInstant(value) + is String -> parseStringAsInstant(value, meta) + else -> unsupportedConversion(value, "kotlin.time.Instant", meta) + } + + private fun convertToLocalDate(value: Any?, meta: TableColumnMetadata): KotlinLocalDate? = + when (value) { + null -> null + is LocalDate -> value.toKotlinLocalDate() + is java.sql.Date -> value.toLocalDate().toKotlinLocalDate() + is Timestamp -> value.toLocalDateTime().toLocalDate().toKotlinLocalDate() + is JavaDate -> value.toInstant().atZone(ZoneOffset.UTC).toLocalDate().toKotlinLocalDate() + is Long -> instantToLocalDate(Instant.fromEpochSeconds(value)) + is Int -> instantToLocalDate(Instant.fromEpochSeconds(value.toLong())) + is Double -> instantToLocalDate(julianDayToInstant(value)) + is String -> parseStringAsLocalDate(value, meta) + else -> unsupportedConversion(value, "kotlinx.datetime.LocalDate", meta) + } + + private fun convertToLocalDateTime(value: Any?, meta: TableColumnMetadata): KotlinLocalDateTime? = + when (value) { + null -> null + is LocalDateTime -> value.toKotlinLocalDateTime() + is Timestamp -> value.toLocalDateTime().toKotlinLocalDateTime() + is JavaDate -> LocalDateTime.ofInstant(value.toInstant(), ZoneOffset.UTC).toKotlinLocalDateTime() + is Long -> instantToLocalDateTime(Instant.fromEpochSeconds(value)) + is Int -> instantToLocalDateTime(Instant.fromEpochSeconds(value.toLong())) + is Double -> instantToLocalDateTime(julianDayToInstant(value)) + is String -> parseStringAsLocalDateTime(value, meta) + else -> unsupportedConversion(value, "kotlinx.datetime.LocalDateTime", meta) + } + + private fun convertToLocalTime(value: Any?, meta: TableColumnMetadata): KotlinLocalTime? = + when (value) { + null -> null + is LocalTime -> value.toKotlinLocalTime() + is java.sql.Time -> value.toLocalTime().toKotlinLocalTime() + // Interpret as seconds since midnight. + is Long -> LocalTime.ofSecondOfDay(value).toKotlinLocalTime() + is Int -> LocalTime.ofSecondOfDay(value.toLong()).toKotlinLocalTime() + is String -> parseStringAsLocalTime(value, meta) + else -> unsupportedConversion(value, "kotlinx.datetime.LocalTime", meta) + } + + private fun parseStringAsInstant(s: String, meta: TableColumnMetadata): Instant { + // Try, in order: full ISO instant, LocalDateTime (T or space separator), LocalDate. + runCatching { return Instant.parse(s) } + val normalised = s.replace(' ', 'T') + runCatching { + return LocalDateTime.parse(normalised).toInstant(ZoneOffset.UTC).toKotlinInstant() + } + runCatching { + return LocalDate.parse(s).atStartOfDay(ZoneOffset.UTC).toInstant().toKotlinInstant() + } + error( + "SQLite: cannot parse '$s' from column '${meta.name}' (declared '${meta.sqlTypeName}') " + + "as an ISO 8601 date/time. Use `Sqlite(customTypesMap = mapOf(\"${meta.sqlTypeName}\" to typeOf()))` " + + "to read it as raw text instead.", + ) + } + + private fun parseStringAsLocalDate(s: String, meta: TableColumnMetadata): KotlinLocalDate { + runCatching { return LocalDate.parse(s).toKotlinLocalDate() } + // Also accept full date-time / instant strings — truncate to the date portion. + runCatching { return instantToLocalDate(parseStringAsInstant(s, meta)) } + error( + "SQLite: cannot parse '$s' from column '${meta.name}' (declared '${meta.sqlTypeName}') " + + "as an ISO 8601 date. Use customTypesMap to read it as raw text instead.", + ) + } + + private fun parseStringAsLocalDateTime(s: String, meta: TableColumnMetadata): KotlinLocalDateTime { + val normalised = s.replace(' ', 'T') + runCatching { return LocalDateTime.parse(normalised).toKotlinLocalDateTime() } + runCatching { return LocalDate.parse(s).atStartOfDay().toKotlinLocalDateTime() } + // As a last resort, accept ISO instant strings and convert to LocalDateTime at UTC. + runCatching { return instantToLocalDateTime(Instant.parse(s)) } + error( + "SQLite: cannot parse '$s' from column '${meta.name}' (declared '${meta.sqlTypeName}') " + + "as an ISO 8601 date-time. Use customTypesMap to read it as raw text instead.", + ) + } + + private fun parseStringAsLocalTime(s: String, meta: TableColumnMetadata): KotlinLocalTime { + runCatching { return LocalTime.parse(s).toKotlinLocalTime() } + error( + "SQLite: cannot parse '$s' from column '${meta.name}' (declared '${meta.sqlTypeName}') " + + "as an ISO 8601 time. Use customTypesMap to read it as raw text instead.", + ) + } + + private fun instantToLocalDate(instant: Instant): KotlinLocalDate = + java.time.Instant.ofEpochSecond(instant.epochSeconds) + .atZone(ZoneOffset.UTC) + .toLocalDate() + .toKotlinLocalDate() + + private fun instantToLocalDateTime(instant: Instant): KotlinLocalDateTime = + java.time.Instant.ofEpochSecond(instant.epochSeconds) + .atZone(ZoneOffset.UTC) + .toLocalDateTime() + .toKotlinLocalDateTime() + + private fun julianDayToInstant(julianDay: Double): Instant { + val epochSeconds = ((julianDay - JULIAN_DAY_UNIX_EPOCH) * SECONDS_PER_DAY).toLong() + return Instant.fromEpochSeconds(epochSeconds) + } + + private fun unsupportedConversion(value: Any?, target: String, meta: TableColumnMetadata): Nothing = + error( + "SQLite: cannot convert value of type ${value?.javaClass?.name} " + + "from column '${meta.name}' (declared '${meta.sqlTypeName}') to $target.", + ) + override fun isSystemTable(tableMetadata: TableMetadata): Boolean = tableMetadata.name.startsWith("sqlite_") override fun buildTableMetadata(tables: ResultSet): TableMetadata = @@ -70,5 +314,9 @@ public class Sqlite(public val customTypesMap: Map = mapOf()) : D public val default: Sqlite = Sqlite() public fun withCustomTypes(customTypesMap: Map): Sqlite = Sqlite(customTypesMap) + + // Julian day number at Unix epoch (1970-01-01 00:00 UTC). + private const val JULIAN_DAY_UNIX_EPOCH = 2440587.5 + private const val SECONDS_PER_DAY = 86_400 } } diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt index 1f5e12ba4e..c549112cfa 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteTest.kt @@ -124,6 +124,36 @@ class SqliteTest { it.executeUpdate() } + // Dates and timestamps: SQLite stores each in one of three encodings — ISO text, + // Unix seconds (INTEGER), or Julian days (REAL). The library returns the raw stored + // value; downstream code is responsible for parsing. + @Language("SQL") + val createTemporalTableQuery = """ + CREATE TABLE Temporal ( + id INTEGER PRIMARY KEY, + isoDate DATE, + isoDateTime DATETIME, + isoTimestamp TIMESTAMP, + unixTimestamp TIMESTAMP, + julianDate DATE, + julianTimestamp TIMESTAMP + ) + """ + + connection.createStatement().execute(createTemporalTableQuery) + + // Julian day 2460146.5 = 2023-07-21 00:00:00 UTC. + // The `.5` fraction forces SQLite to store it as REAL (Julian day convention). + connection.createStatement().execute( + """ + INSERT INTO Temporal + (isoDate, isoDateTime, isoTimestamp, unixTimestamp, julianDate, julianTimestamp) + VALUES + ('2023-07-21', '2023-07-21 10:30:00', '2023-07-21T10:30:00Z', 1690000000, + 2460146.5, 2460146.5) + """.trimIndent(), + ) + val profilePicture = "SampleProfilePictureData".toByteArray() val orderDetails = "OrderDetailsData".toByteArray() @@ -308,4 +338,32 @@ class SqliteTest { schema.columns["enabled"]!!.type shouldBe typeOf() schema.columns["optional"]!!.type shouldBe typeOf() } + + @Test + fun `read date and timestamp columns converts storage class to declared type`() { + // SQLite doesn't have native DATE/TIMESTAMP storage — values may be TEXT (ISO), INTEGER + // (Unix seconds), or REAL (Julian day). The library preserves an idiomatic Kotlin + // date-time type in the schema and converts each value in preprocessing based on its + // runtime storage class. + val df = DataFrame.readSqlTable(connection, "Temporal") + + df.rowsCount() shouldBe 1 + // TEXT storage — ISO strings. + df["isoDate"][0] shouldBe kotlinx.datetime.LocalDate.parse("2023-07-21") + df["isoDateTime"][0] shouldBe kotlinx.datetime.LocalDateTime.parse("2023-07-21T10:30:00") + df["isoTimestamp"][0] shouldBe kotlin.time.Instant.parse("2023-07-21T10:30:00Z") + // INTEGER storage — Unix seconds. + df["unixTimestamp"][0] shouldBe kotlin.time.Instant.fromEpochSeconds(1690000000) + // REAL storage — Julian day 2460146.5 = 2023-07-21 00:00:00 UTC. + df["julianDate"][0] shouldBe kotlinx.datetime.LocalDate.parse("2023-07-21") + df["julianTimestamp"][0] shouldBe kotlin.time.Instant.parse("2023-07-21T00:00:00Z") + + val schema = DataFrameSchema.readSqlTable(connection, "Temporal") + schema.columns["isoDate"]!!.type shouldBe typeOf() + schema.columns["isoDateTime"]!!.type shouldBe typeOf() + schema.columns["isoTimestamp"]!!.type shouldBe typeOf() + schema.columns["unixTimestamp"]!!.type shouldBe typeOf() + schema.columns["julianDate"]!!.type shouldBe typeOf() + schema.columns["julianTimestamp"]!!.type shouldBe typeOf() + } } From cce57cce0aa1df69e1da9d21a9b504e55af56a3f Mon Sep 17 00:00:00 2001 From: "andrei.kislitsyn" Date: Fri, 24 Jul 2026 16:58:57 +0400 Subject: [PATCH 3/7] Add SQL type mapping documentation and SQLite metadata probe for testing --- .../kotlinx/dataframe/io/db/_sqliteProbe.kt | 129 ++++ .../kotlinx/dataframe/io/db/jdbcTypesTest.kt | 630 +++++++++++++++--- docs/StardustDocs/d.tree | 9 + .../StardustDocs/topics/readSqlTypeMapping.md | 54 ++ .../topics/readSqlTypeMapping_DuckDB.md | 123 ++++ .../topics/readSqlTypeMapping_H2.md | 113 ++++ .../topics/readSqlTypeMapping_MariaDB.md | 119 ++++ .../topics/readSqlTypeMapping_MsSql.md | 113 ++++ .../topics/readSqlTypeMapping_MySQL.md | 114 ++++ .../topics/readSqlTypeMapping_PostgreSQL.md | 171 +++++ .../topics/readSqlTypeMapping_SQLite.md | 176 +++++ 11 files changed, 1673 insertions(+), 78 deletions(-) create mode 100644 dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_DuckDB.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_H2.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_MariaDB.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_MsSql.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_MySQL.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_PostgreSQL.md create mode 100644 docs/StardustDocs/topics/readSqlTypeMapping_SQLite.md diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt new file mode 100644 index 0000000000..e84e1b3ebe --- /dev/null +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/_sqliteProbe.kt @@ -0,0 +1,129 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +import org.junit.Ignore +import org.junit.Test +import java.io.File +import java.nio.file.Files +import java.sql.DriverManager +import java.sql.Types + +/** + * Probe used to figure out what the Xerial SQLite JDBC driver reports in the metadata + * for common declared column types (SQLite type affinity is applied at storage level, but + * the driver preserves the declared type name in `sqlTypeName`). + * + * Not part of the regular suite; enable ad-hoc if you need to refresh the mapping. + */ +@Ignore("Investigation probe — not a real test") +class SqliteMetadataProbe { + + @Test + fun `dump getObject class vs metadata jdbcType for problematic types`() { + val file = Files.createTempFile("dataframe_sqlite_probe2_", ".db").toFile() + file.deleteOnExit() + val url = "jdbc:sqlite:${file.absolutePath}" + + // (declaredType, sql-literal, description) + val cases = listOf( + Triple("BOOLEAN", "1", "boolean-as-int"), + Triple("BOOLEAN", "0", "boolean-as-int-false"), + Triple("DATE", "'2020-01-15'", "date-as-iso-text"), + Triple("DATE", "1579046400", "date-as-unix-int"), + Triple("DATETIME", "'2020-01-15 10:30:00'", "datetime-as-iso-text"), + Triple("TIMESTAMP", "'2020-01-15T10:30:00Z'", "timestamp-as-iso-text"), + Triple("TIMESTAMP", "1579083000", "timestamp-as-unix-int"), + Triple("NUMERIC", "1.5", "numeric-as-real"), + Triple("NUMERIC", "42", "numeric-as-int"), + Triple("DECIMAL(10,5)", "1.5", "decimal-as-real"), + Triple("DATE", "2460146.5", "date-as-julian-real"), + Triple("TIMESTAMP", "2460146.5", "timestamp-as-julian-real"), + ) + + DriverManager.getConnection(url).use { conn -> + cases.forEachIndexed { i, (decl, literal, _) -> + conn.createStatement().execute("CREATE TABLE t$i (col $decl)") + conn.createStatement().execute("INSERT INTO t$i (col) VALUES ($literal)") + } + for ((i, case) in cases.withIndex()) { + val (decl, literal, desc) = case + conn.prepareStatement("SELECT col FROM t$i").executeQuery().use { rs -> + val md = rs.metaData + val jdbc = md.getColumnType(1) + val jdbcName = jdbcName(jdbc) + val cls = md.getColumnClassName(1) + rs.next() + val got = rs.getObject(1) + val gotClass = got?.javaClass?.name + println( + "%-25s declared=%-14s literal=%-24s meta.jdbc=%-9s meta.class=%-22s getObject.class=%s value=%s".format( + desc, decl, literal, jdbcName, cls, gotClass, got, + ), + ) + } + } + } + file.delete() + } + + @Test + fun `dump metadata for common declared types`() { + val file = Files.createTempFile("dataframe_sqlite_probe_", ".db").toFile() + file.deleteOnExit() + val url = "jdbc:sqlite:${file.absolutePath}" + + val declaredTypes = listOf( + "INTEGER", "INT", "TINYINT", "SMALLINT", "MEDIUMINT", "BIGINT", "UNSIGNED BIG INT", + "INT2", "INT8", + "REAL", "DOUBLE", "DOUBLE PRECISION", "FLOAT", + "NUMERIC", "DECIMAL(10,5)", + "BOOLEAN", + "DATE", "DATETIME", "TIMESTAMP", + "TEXT", "VARCHAR(255)", "CHAR(10)", "CLOB", "NVARCHAR(10)", "NCHAR(10)", + "BLOB", + "", // no declared type + "CUSTOM_TYPE", // unknown → NUMERIC affinity + ) + + DriverManager.getConnection(url).use { conn -> + declaredTypes.forEachIndexed { i, decl -> + val col = "col$i ${if (decl.isEmpty()) "" else decl}".trim() + conn.createStatement().execute("CREATE TABLE t$i ($col)") + // Insert one representative value so the driver can infer the actual storage class. + conn.createStatement().execute("INSERT INTO t$i (col$i) VALUES (${sampleValueFor(decl)})") + } + for ((i, decl) in declaredTypes.withIndex()) { + conn.prepareStatement("SELECT col$i FROM t$i").executeQuery().use { rs -> + val md = rs.metaData + val jdbc = md.getColumnType(1) + val jdbcName = jdbcName(jdbc) + val typeName = md.getColumnTypeName(1) + val cls = md.getColumnClassName(1) + println( + "declared=%-24s -> sqlTypeName=%-16s jdbcType=%-8s (%s) classNm=%s".format( + "\"$decl\"", typeName, jdbc.toString(), jdbcName, cls, + ), + ) + } + } + } + file.delete() + } + + private fun jdbcName(t: Int): String = + Types::class.java.fields.firstOrNull { it.getInt(null) == t }?.name ?: "?" + + private fun sampleValueFor(decl: String): String { + val u = decl.uppercase() + return when { + u.contains("BLOB") -> "x'00'" + u.contains("BOOLEAN") -> "1" + u.contains("DATE") || u.contains("TIMESTAMP") || u.contains("TIME") -> "'2020-01-01'" + u.contains("CHAR") || u.contains("TEXT") || u.contains("CLOB") -> "'x'" + u.contains("REAL") || u.contains("DOUB") || u.contains("FLOA") || + u.contains("NUMERIC") || u.contains("DECIMAL") -> "1.5" + u.contains("INT") -> "1" + u.isEmpty() -> "1" + else -> "'x'" + } + } +} diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt index 56758ec27b..2b86bd102c 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt @@ -3,113 +3,587 @@ package org.jetbrains.kotlinx.dataframe.io.db import io.kotest.matchers.shouldBe -import org.jetbrains.kotlinx.dataframe.io.db.JdbcTypesTest.MySqlDBTypes.BIGINT_UNSIGNED import org.junit.Test import org.junit.experimental.runners.Enclosed import org.junit.runner.RunWith +import org.postgresql.geometric.PGbox +import org.postgresql.geometric.PGcircle +import org.postgresql.geometric.PGline +import org.postgresql.geometric.PGlseg +import org.postgresql.geometric.PGpath +import org.postgresql.geometric.PGpoint +import org.postgresql.geometric.PGpolygon +import org.postgresql.util.PGInterval +import org.postgresql.util.PGmoney +import java.math.BigDecimal import java.math.BigInteger +import java.sql.Blob +import java.sql.Clob +import java.sql.NClob +import java.sql.Ref +import java.sql.RowId +import java.sql.SQLXML +import java.sql.Time +import java.sql.Types +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.util.Date import kotlin.reflect.KType +import kotlin.reflect.full.withNullability import kotlin.reflect.typeOf +import kotlin.time.Instant +import kotlin.uuid.Uuid +import kotlinx.datetime.LocalDate as KotlinLocalDate +import kotlinx.datetime.LocalDateTime as KotlinLocalDateTime +import kotlinx.datetime.LocalTime as KotlinLocalTime -// TODO: complete and enhance (#1736) +/** + * Non-integration tests for [DbType.getExpectedJdbcType] and related type-mapping logic. + * + * Each DB owns a [TypeMapping] list that acts as the source of truth for its SQL → Kotlin type + * mapping. The list is exercised for both nullable and non-nullable columns. + * + * See https://github.com/Kotlin/dataframe/issues/1736. + */ @RunWith(Enclosed::class) class JdbcTypesTest { - abstract class ColumnType( - val sqlTypeName: String, - val jdbcType: Int, - val javaClassName: String, - val isNullable: Boolean, - val expectedKotlinType: KType, - ) { - fun mockkColMetaData() = - TableColumnMetadata( - "name", - sqlTypeName, - jdbcType, - 10, - javaClassName, - isNullable, - ) + class DefaultDbTypeTypes { + + // A concrete DbType whose behavior is exactly the default one from the base class. + private object DefaultDbType : DbType("default") { + override val driverClassName: String get() = "does.not.matter" + + override fun isSystemTable(tableMetadata: TableMetadata): Boolean = false + + override fun buildTableMetadata(tables: java.sql.ResultSet): TableMetadata = + TableMetadata("t", null, null) + } + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(DefaultDbType, commonJdbcTypeMappings) + } + + @Test + fun `TIMESTAMP with LocalDateTime driver class maps to java_time_LocalDateTime`() { + assertMappings(DefaultDbType, listOf(timestampAsLocalDateTime)) + } + + @Test + fun `BINARY with UUID driver class maps to UUID`() { + assertMappings(DefaultDbType, listOf(binaryAsUuid)) + } + + @Test + fun `Types_OTHER with byte array javaClassName maps to ByteArray`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "[B", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `Types_OTHER with generic Object maps to Any`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "OTHER", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + DefaultDbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + } } - class MariaDBTypes { + class MariaDbTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MariaDb, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MariaDB-specific overrides`() { + assertMappings(MariaDb, mariaDbSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MariaDb.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MariaDb) } } - class MySqlDBTypes { + class MySqlTypes { - object BIGINT_UNSIGNED : ColumnType( - "BIGINT UNSIGNED", - 20, - "java.math.BigInteger", - false, - typeOf(), - ) + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MySql, commonJdbcTypeMappings) + } - val types: List = listOf( - BIGINT_UNSIGNED, - ) + @Test + fun `MySQL-specific overrides`() { + assertMappings(MySql, mySqlSpecificMappings) + } @Test - fun `all MariaDB SQL types should match expected type`() { - types.forEach { type -> - MySql.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MySql) } } + /** + * SQLite is dynamically typed: it has only 5 storage classes (NULL, INTEGER, REAL, TEXT, BLOB) + * and picks one per row based on the value, guided by "type affinity" derived from the + * declared column type. The Xerial JDBC driver reports metadata based on the actual stored + * value, so `getExpectedJdbcType` sees driver-specific `jdbcType` and `javaClassName` combos + * that differ from other databases. The tests below reflect that. + */ class SqliteTypes { - // Taken from #964 - - object LONGVARCHAR_1 : ColumnType( - "LONGVARCHAR", - -2, - "java.lang.Object", - false, - typeOf(), - ) - - object LONGVARCHAR_2 : ColumnType( - "LONGVARCHAR", - 12, - "java.lang.String", - true, - typeOf(), - ) - - val customTypes: List = listOf( - LONGVARCHAR_1, - LONGVARCHAR_2, - ) - - @Test - fun `SQLite custom types`() { - val sqliteCustom = Sqlite( - mapOf("LONGVARCHAR" to typeOf()), + @Test + fun `INTEGER affinity — declared int-like types map to Int or Long`() { + assertMappings(Sqlite.default, sqliteIntegerAffinityMappings) + } + + @Test + fun `REAL affinity — declared real-like types map to Double`() { + assertMappings(Sqlite.default, sqliteRealAffinityMappings) + } + + @Test + fun `TEXT affinity — declared text-like types map to String`() { + assertMappings(Sqlite.default, sqliteTextAffinityMappings) + } + + @Test + fun `BLOB affinity — declared BLOB maps to ByteArray`() { + assertMappings(Sqlite.default, sqliteBlobAffinityMappings) + } + + @Test + fun `NUMERIC affinity — declared numeric-like types map by declared type`() { + assertMappings(Sqlite.default, sqliteNumericAffinityMappings) + } + + @Test + fun `BOOLEAN declared type resolves to Boolean`() { + // Xerial reports Types.BOOLEAN metadata even though values are stored as INTEGER; + // the schema type is Boolean, and `getValueFromResultSet` converts each row via + // `rs.getBoolean` (see Sqlite.getValueFromResultSet). + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "BOOLEAN", + jdbcType = Types.BOOLEAN, + javaClassName = "java.lang.Integer", + isNullable = false, + ), + ) shouldBe typeOf() + + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "BOOLEAN", + jdbcType = Types.BOOLEAN, + javaClassName = "java.lang.Integer", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unrecognised declared type is treated by NUMERIC affinity`() { + // For an unknown declared type Xerial applies NUMERIC affinity and reports the + // jdbcType of the actual stored value; for a text sample that is VARCHAR. + Sqlite.default.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "CUSTOM_TYPE", + jdbcType = Types.VARCHAR, + javaClassName = "java.lang.String", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `customTypesMap overrides the default mapping by SQL type name`() { + val custom = Sqlite( + customTypesMap = mapOf( + "INTEGER" to typeOf(), + "MY_TYPE" to typeOf(), + ), + ) + // INTEGER is normally Int, but is overridden to Long + custom.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "INTEGER", + jdbcType = Types.INTEGER, + javaClassName = "java.lang.Integer", + isNullable = true, + ), + ) shouldBe typeOf() + // Custom type name is respected as-is + custom.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MY_TYPE", + jdbcType = Types.OTHER, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(Sqlite.default) + } + } + + class PostgreSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(PostgreSql, commonJdbcTypeMappings) + } + + @Test + fun `PGobject types map to their PGobject Kotlin types`() { + assertMappings(PostgreSql, postgreSqlSpecificMappings) + } + + @Test + fun `PGobject lookup is case-insensitive`() { + PostgreSql.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "POINT", + jdbcType = Types.OTHER, + javaClassName = "org.postgresql.geometric.PGpoint", + isNullable = true, + ), + ) shouldBe typeOf() + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(PostgreSql) + } + } + + class MsSqlTypes { + + @Test + fun `common SQL types map to the expected Kotlin type`() { + assertMappings(MsSql, commonJdbcTypeMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(MsSql) + } + } + + class H2Types { + + @Test + fun `Regular mode uses default type mappings`() { + assertMappings(H2(H2.Mode.Regular), commonJdbcTypeMappings) + } + + @Test + fun `MySql mode delegates to MySQL-specific overrides`() { + assertMappings(H2(H2.Mode.MySql), mySqlSpecificMappings) + } + + @Test + fun `MariaDb mode delegates to MariaDB-specific overrides`() { + assertMappings(H2(H2.Mode.MariaDb), mariaDbSpecificMappings) + } + + @Test + fun `unknown jdbcType falls back to String`() { + assertUnknownMapsToString(H2(H2.Mode.Regular)) + } + } +} + +// -------------------- Type mapping model & helpers -------------------- + +/** + * A single row in the JDBC → Kotlin type mapping table. + * + * @property sqlTypeName the human-readable SQL type name (e.g. "BIGINT") + * @property jdbcType a constant from [java.sql.Types] + * @property javaClassName the JDBC-reported class name for this column (as returned by + * [java.sql.ResultSetMetaData.getColumnClassName]) + * @property expectedType the expected non-nullable Kotlin type + */ +internal data class TypeMapping( + val sqlTypeName: String, + val jdbcType: Int, + val javaClassName: String, + val expectedType: KType, +) + +internal const val UNKNOWN_JDBC_TYPE: Int = -9999 + +/** + * Test helper that constructs a [TableColumnMetadata] with sensible defaults. + * Not a mock — a lightweight factory to keep test call sites readable. + */ +internal fun createColumnMetadata( + name: String = "col", + sqlTypeName: String, + jdbcType: Int, + size: Int = 10, + javaClassName: String, + isNullable: Boolean, +): TableColumnMetadata = + TableColumnMetadata( + name = name, + sqlTypeName = sqlTypeName, + jdbcType = jdbcType, + size = size, + javaClassName = javaClassName, + isNullable = isNullable, + ) + +/** + * Verifies each mapping resolves correctly for both nullable and non-nullable columns. + * Runs the full type-resolution pipeline (`getExpectedJdbcType` → `getPreprocessedValueType`) + * and compares against the **final DataFrame column type**, which is what the reference + * documentation describes. + */ +internal fun assertMappings(dbType: DbType, mappings: List) { + mappings.forEach { m -> + listOf(false, true).forEach { isNullable -> + val meta = createColumnMetadata( + sqlTypeName = m.sqlTypeName, + jdbcType = m.jdbcType, + javaClassName = m.javaClassName, + isNullable = isNullable, ) - customTypes.forEach { type -> - sqliteCustom.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType - } + val jdbcType = dbType.getExpectedJdbcType(meta) + val finalType = dbType.getPreprocessedValueType(meta, jdbcType) + finalType shouldBe m.expectedType.withNullability(isNullable) } } } + +internal fun assertUnknownMapsToString(dbType: DbType) { + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = false, + ), + ) shouldBe typeOf() + + dbType.getExpectedJdbcType( + createColumnMetadata( + sqlTypeName = "MADE_UP", + jdbcType = UNKNOWN_JDBC_TYPE, + javaClassName = "java.lang.Object", + isNullable = true, + ), + ) shouldBe typeOf() +} + +// -------------------- Type mapping tables -------------------- + +/** + * The default SQL → Kotlin type mapping applied by [DbType]. + * Every DB that does not override the given entry falls through to this table. + */ +internal val commonJdbcTypeMappings: List = listOf( + TypeMapping("BIT", Types.BIT, "java.lang.Boolean", typeOf()), + TypeMapping("TINYINT", Types.TINYINT, "java.lang.Integer", typeOf()), + TypeMapping("SMALLINT", Types.SMALLINT, "java.lang.Integer", typeOf()), + TypeMapping("INTEGER", Types.INTEGER, "java.lang.Integer", typeOf()), + TypeMapping("BIGINT", Types.BIGINT, "java.lang.Long", typeOf()), + TypeMapping("FLOAT", Types.FLOAT, "java.lang.Float", typeOf()), + TypeMapping("REAL", Types.REAL, "java.lang.Float", typeOf()), + TypeMapping("DOUBLE", Types.DOUBLE, "java.lang.Double", typeOf()), + TypeMapping("NUMERIC", Types.NUMERIC, "java.math.BigDecimal", typeOf()), + TypeMapping("DECIMAL", Types.DECIMAL, "java.math.BigDecimal", typeOf()), + TypeMapping("CHAR", Types.CHAR, "java.lang.String", typeOf()), + TypeMapping("VARCHAR", Types.VARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGVARCHAR", Types.LONGVARCHAR, "java.lang.String", typeOf()), + TypeMapping("NCHAR", Types.NCHAR, "java.lang.String", typeOf()), + TypeMapping("NVARCHAR", Types.NVARCHAR, "java.lang.String", typeOf()), + TypeMapping("LONGNVARCHAR", Types.LONGNVARCHAR, "java.lang.String", typeOf()), + TypeMapping("DATE", Types.DATE, "java.sql.Date", typeOf()), + TypeMapping("TIME", Types.TIME, "java.sql.Time", typeOf