Copy views and UNIQUE constraints (0.5.0)

.gitignore

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Test databases generated at the repo root
+/*.db

README.md

@@ -68,23 +68,27 @@ uvx sqlite2duckdb --force source.db target.db   # overwrite target.db without as
 from sqlite2duckdb import sqlite_to_duckdb
 
 result = sqlite_to_duckdb("source.sqlite", "target.duckdb")
-print(result.tables, result.elapsed)
+print(result.tables, result.views, result.elapsed)
 ```
 
 ## What is converted
 
 | | |
 |---|---|
 | Tables and data | ✅ |
-| Primary keys, NOT NULL constraints, indexes | ✅ |
-| UNIQUE, FOREIGN KEY and CHECK constraints | ❌ |
-| Views | ❌ (silently dropped) |
+| Primary keys, NOT NULL and UNIQUE constraints | ✅ |
+| Indexes | ✅ |
+| Views | ✅ best effort |
+| FOREIGN KEY and CHECK constraints | ❌ |
 
-## Todo
+A view whose SQL uses something duckdb has no equivalent for (`MATCH`, `julianday()`) is
+skipped with a warning instead of failing the conversion.
+
+FOREIGN KEY and CHECK are not copied, though not for lack of support: duckdb accepts and
+enforces both in `CREATE TABLE`. It checks foreign keys row by row, so a self-referencing
+table cannot be bulk loaded, and there is no `ALTER TABLE ADD CONSTRAINT` to add them once
+the data is in.
 
-- [ ] Custom type mapping
-- [x] Primary keys, NOT NULL constraints and indexes
-- [ ] Views, and UNIQUE / FOREIGN KEY / CHECK constraints
 
 ## Contributing
 

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sqlite2duckdb"
-version = "0.4.0"
+version = "0.5.0"
 authors = [{name="Sacha Schutz", email="sacha.schutz@pm.me"}]
 description = "A tool to convert sqlite database to duckdb database"
 readme = "README.md"

sqlite2duckdb/sqlite_to_duckdb.py

@@ -20,6 +20,7 @@ class ConversionResult:
     tables: int
     elapsed: float
     """Wall clock duration of the conversion, in seconds."""
+    views: int = 0
 
 
 def _quote_identifier(name: str) -> str:
@@ -96,6 +97,88 @@ def _copy_indexes(conn: duckdb.DuckDBPyConnection, sqlite_path: str) -> None:
             logger.warning("Could not recreate index %s: %s", name, error)
 
 
+def _copy_unique_constraints(
+    conn: duckdb.DuckDBPyConnection, sqlite_path: str, table_names: list[str]
+) -> None:
+    """Replay the UNIQUE constraints that sqlite records without any SQL.
+
+    A column or table level UNIQUE becomes an autoindex whose sqlite_master row
+    has a NULL sql, so _copy_indexes cannot see it. Duckdb has no ALTER TABLE ADD
+    CONSTRAINT either, so a unique index is how the guarantee is carried over.
+    """
+
+    with contextlib.closing(sqlite3.connect(sqlite_path)) as source:
+        for table in table_names:
+            indexes = source.execute(
+                f"PRAGMA index_list({_quote_identifier(table)})"
+            ).fetchall()
+
+            for _, index_name, unique, origin, _partial in indexes:
+                # 'c' indexes carry their own SQL and are handled by _copy_indexes,
+                # and 'pk' is already part of the table DDL.
+                if not unique or origin != "u":
+                    continue
+
+                columns = [
+                    row[2]
+                    for row in source.execute(
+                        f"PRAGMA index_info({_quote_identifier(index_name)})"
+                    ).fetchall()
+                ]
+                if any(column is None for column in columns):
+                    logger.warning(
+                        "Skipping unique index %s: it is built on an expression",
+                        index_name,
+                    )
+                    continue
+
+                targets = ", ".join(_quote_identifier(column) for column in columns)
+                try:
+                    conn.sql(
+                        f"CREATE UNIQUE INDEX {_quote_identifier(index_name)} "
+                        f"ON {_quote_identifier(table)} ({targets})"
+                    )
+                except duckdb.Error as error:
+                    logger.warning(
+                        "Could not recreate unique index %s: %s", index_name, error
+                    )
+
+
+def _copy_views(conn: duckdb.DuckDBPyConnection, sqlite_path: str) -> int:
+    """Recreate the source views, and return how many made it across."""
+
+    with contextlib.closing(sqlite3.connect(sqlite_path)) as source:
+        views = source.execute(
+            "SELECT name, sql FROM sqlite_master WHERE type = 'view' AND sql IS NOT NULL"
+        ).fetchall()
+
+    pending = [(name, _brackets_to_quotes(sql)) for name, sql in views]
+    errors: dict[str, duckdb.Error] = {}
+    created = 0
+
+    # A view can sit on top of another one and sqlite_master does not guarantee
+    # dependency order, so keep retrying while a pass still makes progress.
+    while pending:
+        failed = []
+        for name, statement in pending:
+            try:
+                conn.sql(statement)
+            except duckdb.Error as error:
+                errors[name] = error
+                failed.append((name, statement))
+            else:
+                created += 1
+
+        if len(failed) == len(pending):
+            break
+        pending = failed
+
+    for name, _ in pending:
+        logger.warning("Could not recreate view %s: %s", name, errors[name])
+
+    return created
+
+
 def _copy_tables(
     conn: duckdb.DuckDBPyConnection, tables: list[tuple[str, str]]
 ) -> None:
@@ -118,9 +201,13 @@ def sqlite_to_duckdb(
 ) -> ConversionResult:
     """Copy a sqlite database into a new duckdb database.
 
-    Tables, data, primary keys, NOT NULL constraints and indexes are copied.
-    Views and UNIQUE / FOREIGN KEY / CHECK constraints are not: duckdb's sqlite
-    extension does not expose them on the attached database.
+    Tables, data, views, primary keys, NOT NULL and UNIQUE constraints and
+    indexes are copied. FOREIGN KEY and CHECK constraints are not: duckdb checks
+    foreign keys row by row, so a self-referencing table cannot be bulk loaded,
+    and there is no ALTER TABLE ADD CONSTRAINT to add them once the data is in.
+
+    A view duckdb cannot bind is skipped with a warning rather than failing the
+    whole conversion.
 
     Raises FileNotFoundError if `sqlite_db` does not exist, and FileExistsError
     if `duck_db` already exists and `overwrite` is False.
@@ -157,6 +244,10 @@ def sqlite_to_duckdb(
 
         _copy_tables(conn, tables)
         _copy_indexes(conn, sqlite_path)
+        _copy_unique_constraints(conn, sqlite_path, [name for name, _ in tables])
+        views = _copy_views(conn, sqlite_path)
+        if views:
+            logger.info("%d view(s) copied", views)
 
         conn.sql("DETACH __other")
     except BaseException:
@@ -171,4 +262,6 @@ def sqlite_to_duckdb(
     elapsed = time.perf_counter() - start_time
     logger.info("Done in %s !", _format_duration(elapsed))
 
-    return ConversionResult(target=duck_path, tables=len(tables), elapsed=elapsed)
+    return ConversionResult(
+        target=duck_path, tables=len(tables), elapsed=elapsed, views=views
+    )

tests/conftest.py

@@ -49,3 +49,13 @@ def duckdb_path(tmp_path):
 @pytest.fixture(scope="module")
 def bracket_index_sqlite(tmp_path_factory):
     return _module_db(tmp_path_factory, "bindex", utils.build_bracket_index_sqlite)
+
+
+@pytest.fixture(scope="module")
+def views_sqlite(tmp_path_factory):
+    return _module_db(tmp_path_factory, "views", utils.build_views_sqlite)
+
+
+@pytest.fixture(scope="module")
+def unique_sqlite(tmp_path_factory):
+    return _module_db(tmp_path_factory, "unique", utils.build_unique_sqlite)

tests/test_convert.py

@@ -95,6 +95,11 @@ def test_bracket_quoted_view(bracket_sqlite, duckdb_path):
         (2, 7),
     ]
 
+    # The bracket quoted view built on it comes across too.
+    assert d_conn.sql(
+        'SELECT * FROM "Order Subtotals" ORDER BY OrderID'
+    ).fetchall() == [(1, 5), (2, 7)]
+
 
 def test_dotted_column_names(dotted_column_sqlite, duckdb_path):
     """Regression test for issue #4: a STRICT table with numeric-looking column

tests/test_views_and_unique.py

@@ -0,0 +1,87 @@
+"""Views and UNIQUE constraints, which duckdb's sqlite extension does not expose
+on an attached database and which have to be read back from sqlite_master."""
+
+import logging
+
+import duckdb
+import pytest
+
+from sqlite2duckdb import sqlite_to_duckdb
+
+
+def test_views_are_copied(views_sqlite, duckdb_path):
+    result = sqlite_to_duckdb(views_sqlite, duckdb_path)
+
+    d_conn = duckdb.connect(str(duckdb_path))
+
+    assert d_conn.sql("SELECT * FROM by_region ORDER BY region").fetchall() == [
+        ("north", 15),
+        ("south", 7),
+    ]
+    assert result.views == 3
+
+
+def test_view_built_on_another_view(views_sqlite, duckdb_path):
+    sqlite_to_duckdb(views_sqlite, duckdb_path)
+
+    d_conn = duckdb.connect(str(duckdb_path))
+
+    assert d_conn.sql("SELECT * FROM big_regions").fetchall() == [("north",)]
+
+
+def test_bracket_quoted_view_is_translated(views_sqlite, duckdb_path):
+    sqlite_to_duckdb(views_sqlite, duckdb_path)
+
+    d_conn = duckdb.connect(str(duckdb_path))
+
+    assert d_conn.sql('SELECT COUNT(*) FROM "north sales"').fetchone() == (2,)
+
+
+def test_view_duckdb_cannot_parse_is_skipped_with_a_warning(
+    views_sqlite, duckdb_path, caplog
+):
+    with caplog.at_level(logging.WARNING, logger="sqlite2duckdb.sqlite_to_duckdb"):
+        sqlite_to_duckdb(views_sqlite, duckdb_path)
+
+    assert "matched" in caplog.text
+
+    d_conn = duckdb.connect(str(duckdb_path))
+    views = {
+        row[0]
+        for row in d_conn.sql(
+            "SELECT view_name FROM duckdb_views() WHERE NOT internal"
+        ).fetchall()
+    }
+
+    assert "matched" not in views
+    # The rest of the database must still be intact.
+    assert d_conn.sql("SELECT COUNT(*) FROM sales").fetchone() == (3,)
+
+
+def test_unique_constraints_are_enforced(unique_sqlite, duckdb_path):
+    sqlite_to_duckdb(unique_sqlite, duckdb_path)
+
+    d_conn = duckdb.connect(str(duckdb_path))
+
+    # Column level UNIQUE, recorded by sqlite as an autoindex with no SQL.
+    with pytest.raises(duckdb.ConstraintException):
+        d_conn.sql("INSERT INTO members VALUES (2, 'ada@example.com', 'x', 'y')")
+
+    # Table level UNIQUE over two columns.
+    with pytest.raises(duckdb.ConstraintException):
+        d_conn.sql(
+            "INSERT INTO members VALUES (3, 'other@example.com', 'ada', 'lovelace')"
+        )
+
+    # An explicit CREATE UNIQUE INDEX, which does carry its SQL.
+    with pytest.raises(duckdb.ConstraintException):
+        d_conn.sql("INSERT INTO members VALUES (4, 'x@example.com', 'x', 'lovelace')")
+
+
+def test_non_unique_rows_still_insert(unique_sqlite, duckdb_path):
+    sqlite_to_duckdb(unique_sqlite, duckdb_path)
+
+    d_conn = duckdb.connect(str(duckdb_path))
+    d_conn.sql("INSERT INTO members VALUES (5, 'grace@example.com', 'grace', 'hopper')")
+
+    assert d_conn.sql("SELECT COUNT(*) FROM members").fetchone() == (2,)

tests/utils.py

@@ -200,3 +200,53 @@ def build_bracket_index_sqlite(path):
     conn.close()
 
     return path
+
+
+def build_views_sqlite(path):
+    """Views, including one chained on another and one duckdb cannot bind."""
+
+    conn = sqlite3.connect(path)
+    conn.executescript(
+        """
+        CREATE TABLE sales (id INTEGER, region TEXT, amount INTEGER);
+        INSERT INTO sales VALUES (1, 'north', 10), (2, 'north', 5), (3, 'south', 7);
+
+        CREATE VIEW by_region AS
+            SELECT region, SUM(amount) AS total FROM sales GROUP BY region;
+
+        CREATE VIEW big_regions AS
+            SELECT region FROM by_region WHERE total > 8;
+
+        CREATE VIEW [north sales] AS SELECT * FROM sales WHERE [region] = 'north';
+
+        -- MATCH is sqlite only, and duckdb's parser rejects it outright.
+        CREATE VIEW matched AS SELECT * FROM sales WHERE region MATCH 'north';
+        """
+    )
+    conn.commit()
+    conn.close()
+
+    return path
+
+
+def build_unique_sqlite(path):
+    """The three ways sqlite records uniqueness."""
+
+    conn = sqlite3.connect(path)
+    conn.executescript(
+        """
+        CREATE TABLE members (
+            id INTEGER PRIMARY KEY,
+            email TEXT UNIQUE,
+            first TEXT,
+            last TEXT,
+            UNIQUE (first, last)
+        );
+        INSERT INTO members VALUES (1, 'ada@example.com', 'ada', 'lovelace');
+        CREATE UNIQUE INDEX idx_members_last ON members (last);
+        """
+    )
+    conn.commit()
+    conn.close()
+
+    return path