diff --git a/Cargo.lock b/Cargo.lock index cfc55247..3de6fedc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,6 +187,9 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arc-swap" @@ -220,34 +223,20 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "55.2.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" +checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" dependencies = [ - "arrow-arith 55.2.0", - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-cast 55.2.0", - "arrow-data 55.2.0", - "arrow-ord 55.2.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", "arrow-row", - "arrow-schema 55.2.0", - "arrow-select 55.2.0", - "arrow-string 55.2.0", -] - -[[package]] -name = "arrow-arith" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" -dependencies = [ - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "chrono", - "num", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -256,30 +245,14 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num-traits 0.2.19", ] -[[package]] -name = "arrow-array" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "chrono", - "half 2.7.1", - "hashbrown 0.15.5", - "num", -] - [[package]] name = "arrow-array" version = "57.3.0" @@ -287,9 +260,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" dependencies = [ "ahash 0.8.12", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "half 2.7.1", "hashbrown 0.16.0", @@ -298,17 +271,6 @@ dependencies = [ "num-traits 0.2.19", ] -[[package]] -name = "arrow-buffer" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" -dependencies = [ - "bytes", - "half 2.7.1", - "num", -] - [[package]] name = "arrow-buffer" version = "57.3.0" @@ -321,68 +283,36 @@ dependencies = [ "num-traits 0.2.19", ] -[[package]] -name = "arrow-cast" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" -dependencies = [ - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "arrow-select 55.2.0", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half 2.7.1", - "lexical-core 1.0.6", - "num", - "ryu", -] - [[package]] name = "arrow-cast" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", "atoi", "base64 0.22.1", "chrono", + "comfy-table", "half 2.7.1", "lexical-core 1.0.6", "num-traits 0.2.19", "ryu", ] -[[package]] -name = "arrow-data" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" -dependencies = [ - "arrow-buffer 55.2.0", - "arrow-schema 55.2.0", - "half 2.7.1", - "num", -] - [[package]] name = "arrow-data" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" dependencies = [ - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer", + "arrow-schema", "half 2.7.1", "num-integer", "num-traits 0.2.19", @@ -394,47 +324,25 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "flatbuffers", ] -[[package]] -name = "arrow-json" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" -dependencies = [ - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-cast 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "chrono", - "half 2.7.1", - "indexmap 2.12.0", - "lexical-core 1.0.6", - "memchr", - "num", - "serde 1.0.228", - "serde_json", - "simdutf8", -] - [[package]] name = "arrow-json" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half 2.7.1", "indexmap 2.12.0", @@ -448,72 +356,39 @@ dependencies = [ "simdutf8", ] -[[package]] -name = "arrow-ord" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" -dependencies = [ - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "arrow-select 55.2.0", -] - [[package]] name = "arrow-ord" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] name = "arrow-row" -version = "55.2.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" +checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" dependencies = [ - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half 2.7.1", ] -[[package]] -name = "arrow-schema" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" -dependencies = [ - "bitflags 2.13.1", -] - [[package]] name = "arrow-schema" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" - -[[package]] -name = "arrow-select" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" dependencies = [ - "ahash 0.8.12", - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "num", + "bitflags 2.13.1", ] [[package]] @@ -523,41 +398,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" dependencies = [ "ahash 0.8.12", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num-traits 0.2.19", ] -[[package]] -name = "arrow-string" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" -dependencies = [ - "arrow-array 55.2.0", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "arrow-select 55.2.0", - "memchr", - "num", - "regex", - "regex-syntax", -] - [[package]] name = "arrow-string" version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num-traits 0.2.19", "regex", @@ -2796,6 +2654,17 @@ dependencies = [ "syn 2.0.108", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -2966,9 +2835,9 @@ dependencies = [ [[package]] name = "duckdb" -version = "1.3.2" +version = "1.10500.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ab83a22530667ffc8cc0e31c0549bb07bea5dba3b957a8e315effc38923701" +checksum = "6a2048e4963a37ec458d9e93444192e0e949ac0a188d201ea53472f2c42db822" dependencies = [ "arrow", "cast", @@ -2978,7 +2847,6 @@ dependencies = [ "libduckdb-sys", "num-integer", "rust_decimal", - "smallvec", "strum 0.27.2", ] @@ -4202,14 +4070,14 @@ dependencies = [ "anyhow", "apache-avro", "array-init", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "arrow-string 57.3.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", "as-any", "async-trait", "backon", @@ -4864,17 +4732,19 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libduckdb-sys" -version = "1.3.2" +version = "1.10500.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e02f6069513efb67a0743aff3b846090de14763802b0e95c352ebc6e1bdc1da" +checksum = "6df9db064ff17120305ec6b7aee048f766cdf2a3ce9ffbb6953aaa3634605030" dependencies = [ "cc", "flate2", "pkg-config", + "reqwest", "serde 1.0.228", "serde_json", "tar", "vcpkg", + "zip", ] [[package]] @@ -5964,13 +5834,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" dependencies = [ "ahash 0.8.12", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", "arrow-ipc", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-schema", + "arrow-select", "base64 0.22.1", "brotli", "bytes", @@ -6654,7 +6524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -6689,7 +6559,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.108", @@ -7234,6 +7104,7 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2 0.4.12", @@ -10619,12 +10490,9 @@ name = "wrappers" version = "0.6.3" dependencies = [ "anyhow", - "arrow-array 55.2.0", - "arrow-array 57.3.0", - "arrow-json 55.2.0", - "arrow-json 57.3.0", - "arrow-schema 55.2.0", - "arrow-schema 57.3.0", + "arrow-array", + "arrow-json", + "arrow-schema", "async-compression 0.3.15", "aws-config", "aws-sdk-cognitoidentityprovider", @@ -10938,12 +10806,38 @@ dependencies = [ "syn 2.0.108", ] +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.12.0", + "memchr", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/docs/catalog/duckdb.md b/docs/catalog/duckdb.md index ec4c2de4..81702aed 100644 --- a/docs/catalog/duckdb.md +++ b/docs/catalog/duckdb.md @@ -506,6 +506,67 @@ A `create server` statement example used to access local Iceberg service: ); ``` +#### DuckLake + +This is to access an existing [DuckLake](https://ducklake.select/) catalog backed by PostgreSQL, with data files on S3-compatible storage. The wrapper attaches the catalog read-only and does not create or migrate it. Local catalog and data files are disabled, as with the other remote server types. + +| Server Option | Description | Required | Default | +| ------------- | ----------- | :------: | ------- | +| type | Server type, must be `ducklake` | Y | | +| metadata_path | Catalog connection string, including the `postgres:` prefix | Y | | +| metadata_schema | Schema containing the DuckLake metadata tables in PostgreSQL | | DuckLake default (`main`) | +| data_path | Optional data location override; omit to use the catalog's stored location | | Stored catalog path | +| key_id | S3 access key ID | For private S3 storage | | +| secret | S3 secret access key | For private S3 storage | | + +The S3 options listed above, including `region`, `endpoint`, `url_style`, `use_ssl` and `session_token`, are also supported. Use these for services such as MinIO, Supabase Storage, or Cloudflare R2's S3 endpoint. The `ducklake`, `postgres`, `httpfs` and `parquet` extensions are installed and loaded automatically. + +Store the complete catalog connection string in Vault to protect its credentials: + +```sql +select vault.create_secret( + 'postgres:host=catalog.example.com port=5432 dbname=ducklake user=reader password= sslmode=require', + 'ducklake_catalog' +); + +create server ducklake_server + foreign data wrapper duckdb_wrapper + options ( + type 'ducklake', + vault_metadata_path '', + vault_key_id '', + vault_secret '', + region 'us-east-1' + ); +``` + +Alternatively, set `metadata_path`, `key_id` and `secret` directly. The metadata catalog must already be initialized and accessible to the PostgreSQL user in the connection string. + +Import tables from a logical DuckLake schema (this is separate from `metadata_schema`): + +```sql +create schema if not exists ducklake; + +import foreign schema "main" + limit to (products) + from server ducklake_server into ducklake; + +select * from ducklake.products; +``` + +Imported tables retain their original names. `EXCEPT` and importing the whole schema are also supported. To define a table manually, qualify its source with the `ducklake` catalog alias: + +```sql +create foreign table ducklake.products ( + id bigint, + name text +) +server ducklake_server +options (table 'ducklake.main.products'); +``` + +See [DuckLake connection options](https://ducklake.select/docs/stable/duckdb/usage/connecting) for details about metadata and data paths. + #### MotherDuck This is to access [MotherDuck](https://motherduck.com/), a cloud-hosted DuckDB service. @@ -605,7 +666,7 @@ import foreign schema "docs_example" from server duckdb_server into duckdb; ``` -Currently only MotherDuck and Iceberg-like servers, such as S3 Tables, R2 Data Catalog and etc., support `import foreign schema` without specifying source tables. For other types of servers, source tables must be explicitly specified in options. For example, +MotherDuck, DuckLake and Iceberg-like servers, such as S3 Tables and R2 Data Catalog, support `import foreign schema` without specifying source tables. For other types of servers, source tables must be explicitly specified in options. For example, ```sql -- 'duckdb_server_md' server type is 'md', all tables under 'main' schema diff --git a/supabase-wrappers/src/utils.rs b/supabase-wrappers/src/utils.rs index 215f5ed0..7d8cd79b 100644 --- a/supabase-wrappers/src/utils.rs +++ b/supabase-wrappers/src/utils.rs @@ -97,6 +97,29 @@ pub fn is_sensitive_option(option_name: &str) -> bool { SENSITIVE_OPTION_NAMES.iter().any(|&s| lower.contains(s)) } +fn redact_postgres_urls(message: &str) -> String { + let mut result = String::with_capacity(message.len()); + for token in message.split_inclusive(char::is_whitespace) { + // ASCII case folding keeps byte offsets valid for UTF-8 messages. + let lower = token.to_ascii_lowercase(); + let start = ["postgres://", "postgresql://"] + .iter() + .filter_map(|scheme| lower.find(scheme)) + .min(); + if let Some(start) = start { + result.push_str(&token[..start]); + result.push_str("[REDACTED]"); + // Conservatively hide the whole URL, including query credentials + // and trailing punctuation. Valid URLs encode embedded whitespace. + let end = token.trim_end_matches(char::is_whitespace).len(); + result.push_str(&token[end..]); + } else { + result.push_str(token); + } + } + result +} + /// Masks credential values in an error message string. /// Scans for patterns like `key = 'value'` or `key: value` and masks sensitive values. /// @@ -104,6 +127,7 @@ pub fn is_sensitive_option(option_name: &str) -> bool { /// - SQL-style: `secret = 'value'` /// - JSON-style: `"secret": "value"` /// - URL parameters: `secret=value` +/// - PostgreSQL connection URLs: the complete URL is hidden /// /// # Examples /// ``` @@ -114,7 +138,9 @@ pub fn is_sensitive_option(option_name: &str) -> bool { /// assert!(masked.contains("wJal***")); /// ``` pub fn mask_credentials_in_message(message: &str) -> String { - let mut result = message.to_string(); + // Do this before named-field masking, which could otherwise modify a URL + // containing query parameters before its embedded credentials are removed. + let mut result = redact_postgres_urls(message); for sensitive_name in SENSITIVE_OPTION_NAMES { let lower_name = sensitive_name.to_lowercase(); diff --git a/wrappers/.ci/docker-compose-native.yaml b/wrappers/.ci/docker-compose-native.yaml index b9c057d5..f58935dd 100644 --- a/wrappers/.ci/docker-compose-native.yaml +++ b/wrappers/.ci/docker-compose-native.yaml @@ -272,4 +272,18 @@ services: test: ["CMD", "sh", "-c", "echo 'db.runCommand({ ping: 1 })' | mongosh --quiet"] interval: 10s timeout: 5s - retries: 5 \ No newline at end of file + retries: 5 + + ducklake-postgres: + image: postgres:17 + environment: + POSTGRES_USER: ducklake + POSTGRES_PASSWORD: ducklake + POSTGRES_DB: ducklake + ports: + - "5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ducklake -d ducklake"] + interval: 5s + timeout: 5s + retries: 10 diff --git a/wrappers/Cargo.toml b/wrappers/Cargo.toml index 4bdbe065..9f8b8f6c 100644 --- a/wrappers/Cargo.toml +++ b/wrappers/Cargo.toml @@ -185,9 +185,9 @@ iceberg_fdw = [ "uuid", ] duckdb_fdw = [ - "arrow-array-compat", - "arrow-schema-compat", - "arrow-json-compat", + "arrow-array", + "arrow-schema", + "arrow-json", "chrono", "duckdb", "regex", @@ -356,13 +356,9 @@ iceberg-catalog-rest = { version = "0.8.0", optional = true } rust_decimal = { version = "1.37.1", optional = true } # for duckdb_fdw -duckdb = { version = "=1.3.2", features = ["bundled"], optional = true } -# Version-pinned arrow compat deps matching duckdb's bundled arrow (55.x). -# The workspace uses arrow 57.x for iceberg_fdw; using 57.x here causes type -# incompatibilities because duckdb::arrow::* types implement the 55.x traits. -arrow-array-compat = { package = "arrow-array", version = "55", optional = true } -arrow-json-compat = { package = "arrow-json", version = "55", optional = true } -arrow-schema-compat = { package = "arrow-schema", version = "55", optional = true } +# Bundles DuckDB 1.5.0 and shares Arrow 57 with iceberg_fdw and s3_fdw. +# Keep pinned: duckdb-rs 1.10501.0 moves to Arrow 58. +duckdb = { version = "=1.10500.0", features = ["bundled"], optional = true } [dev-dependencies] pgrx-tests = "=0.19.2" diff --git a/wrappers/src/fdw/duckdb_fdw/README.md b/wrappers/src/fdw/duckdb_fdw/README.md index d0b6b7d2..d25fb182 100644 --- a/wrappers/src/fdw/duckdb_fdw/README.md +++ b/wrappers/src/fdw/duckdb_fdw/README.md @@ -8,8 +8,9 @@ This is a foreign data wrapper for [DuckDB](https://duckdb.org/). It is develope ## Changelog -| Version | Date | Notes | -| ------- | ---------- | ---------------------------------------------------- | -| 0.1.2 | 2025-10-16 | Add MotherDuck support | -| 0.1.1 | 2025-08-15 | Replace execute_batch() with execute() | -| 0.1.0 | 2024-10-31 | Initial version | +| Version | Date | Notes | +| ------- | ---------- | ---------------------------------------------------------- | +| 0.1.3 | 2026-09-29 | Upgrade DuckDB to 1.5.0 and add read-only DuckLake support | +| 0.1.2 | 2025-10-16 | Add MotherDuck support | +| 0.1.1 | 2025-08-15 | Replace execute_batch() with execute() | +| 0.1.0 | 2024-10-31 | Initial version | diff --git a/wrappers/src/fdw/duckdb_fdw/duckdb_fdw.rs b/wrappers/src/fdw/duckdb_fdw/duckdb_fdw.rs index b43296f7..6e5d5a0c 100644 --- a/wrappers/src/fdw/duckdb_fdw/duckdb_fdw.rs +++ b/wrappers/src/fdw/duckdb_fdw/duckdb_fdw.rs @@ -9,7 +9,7 @@ use supabase_wrappers::prelude::*; use super::{DuckdbFdwError, DuckdbFdwResult, mapper, server_type::ServerType}; #[wrappers_fdw( - version = "0.1.2", + version = "0.1.3", author = "Supabase", website = "https://github.com/supabase/wrappers/tree/main/wrappers/src/fdw/duckdb_fdw", error_type = "DuckdbFdwError" @@ -27,20 +27,18 @@ impl DuckdbFdw { const FDW_NAME: &'static str = "DuckdbFdw"; fn init_duckdb(&self) -> DuckdbFdwResult<()> { - let sql_batch = String::default() - + self.svr_type.get_duckdb_extension_sql() - + &self.svr_type.get_settings_sql(&self.svr_opts) - + &self.svr_type.get_create_secret_sql(&self.svr_opts) - + &self.svr_type.get_attach_sql(&self.svr_opts)?; - - // execute_batch() won't raise error when one of the statements failed, - // so we execute each sql separately - for sql in sql_batch - .split(";") - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - { - self.conn.execute(sql, [])?; + let statements = self + .svr_type + .get_duckdb_extension_sql() + .into_iter() + .chain(self.svr_type.get_settings_sql(&self.svr_opts)) + .chain(self.svr_type.get_create_secret_sql(&self.svr_opts)) + .chain([self.svr_type.get_attach_sql(&self.svr_opts)?]); + + // Execute each statement separately to propagate errors, preserving + // semicolons inside quoted credentials and connection strings. + for sql in statements.filter(|sql| !sql.is_empty()) { + self.conn.execute(&sql, [])?; } Ok(()) diff --git a/wrappers/src/fdw/duckdb_fdw/mapper.rs b/wrappers/src/fdw/duckdb_fdw/mapper.rs index bab02952..9ae34de8 100644 --- a/wrappers/src/fdw/duckdb_fdw/mapper.rs +++ b/wrappers/src/fdw/duckdb_fdw/mapper.rs @@ -1,5 +1,5 @@ -use arrow_array_compat::{Array, RecordBatch, array::ArrayRef}; -use arrow_json_compat::ArrayWriter; +use arrow_array::{Array, RecordBatch, array::ArrayRef}; +use arrow_json::ArrayWriter; use duckdb::{ self, types::{EnumType, ListType, ValueRef}, diff --git a/wrappers/src/fdw/duckdb_fdw/mod.rs b/wrappers/src/fdw/duckdb_fdw/mod.rs index e8b82325..77a4a511 100644 --- a/wrappers/src/fdw/duckdb_fdw/mod.rs +++ b/wrappers/src/fdw/duckdb_fdw/mod.rs @@ -28,7 +28,7 @@ enum DuckdbFdwError { NumericError(#[from] pgrx::datum::numeric_support::error::Error), #[error("arrow error: {0}")] - ArrowError(#[from] arrow_schema_compat::ArrowError), + ArrowError(#[from] arrow_schema::ArrowError), #[error("uuid error: {0}")] UuidConversionError(#[from] uuid::Error), diff --git a/wrappers/src/fdw/duckdb_fdw/server_type.rs b/wrappers/src/fdw/duckdb_fdw/server_type.rs index 85bbb33c..01d380cc 100644 --- a/wrappers/src/fdw/duckdb_fdw/server_type.rs +++ b/wrappers/src/fdw/duckdb_fdw/server_type.rs @@ -22,6 +22,7 @@ pub(super) enum ServerType { // SQL-like Remotes MotherDuck, + DuckLake, } impl ServerType { @@ -36,6 +37,7 @@ impl ServerType { "polaris" => Self::Polaris, "lakekeeper" => Self::Lakekeeper, "md" => Self::MotherDuck, + "ducklake" => Self::DuckLake, _ => return Err(DuckdbFdwError::InvalidServerType(svr_type.to_owned())), }; Ok(ret) @@ -51,6 +53,7 @@ impl ServerType { Self::Polaris => "polaris", Self::Lakekeeper => "lakekeeper", Self::MotherDuck => "md", + Self::DuckLake => "ducklake", } } @@ -62,17 +65,23 @@ impl ServerType { } pub(super) fn is_sql_like(&self) -> bool { - matches!(self, Self::MotherDuck) + matches!(self, Self::MotherDuck | Self::DuckLake) } - pub(super) fn get_duckdb_extension_sql(&self) -> &'static str { - match self { + pub(super) fn get_duckdb_extension_sql(&self) -> Vec { + let extensions = match self { Self::Iceberg | Self::S3Tables | Self::R2Catalog | Self::Polaris | Self::Lakekeeper => { - "install iceberg;load iceberg;" + vec!["iceberg"] } - Self::MotherDuck => "install md;load md;", - _ => "", - } + Self::MotherDuck => vec!["md"], + // Load dependencies before disabling local filesystem access. + Self::DuckLake => vec!["ducklake", "postgres", "httpfs", "parquet"], + _ => vec![], + }; + extensions + .into_iter() + .flat_map(|ext| [format!("install {ext}"), format!("load {ext}")]) + .collect() } fn allowed_secret_params(&self) -> Vec<&'static str> { @@ -100,7 +109,7 @@ impl ServerType { "oauth2_scope", "oauth2_server_uri", ], - Self::MotherDuck => vec![], + Self::MotherDuck | Self::DuckLake => vec![], } } @@ -126,10 +135,11 @@ impl ServerType { } // make 'create secret' sql for DuckDB from server options - pub(super) fn get_create_secret_sql(&self, svr_opts: &ServerOptions) -> String { + pub(super) fn get_create_secret_sql(&self, svr_opts: &ServerOptions) -> Vec { let secrets: Vec<(&str, Vec<&str>)> = match self { Self::S3 | Self::S3Tables => vec![("s3", self.allowed_secret_params())], Self::R2 => vec![("r2", self.allowed_secret_params())], + Self::DuckLake => vec![("s3", Self::S3.allowed_secret_params())], // note: for generic Iceberg, we only support S3 compatible storage for now, // so we need to create 2 secrets: one for S3 and one for Iceberg @@ -147,16 +157,19 @@ impl ServerType { _ => vec![], }; - let mut ret = String::default(); + let mut ret = Vec::new(); for (typ, params) in secrets { let opts = self.format_options(svr_opts, ¶ms); - ret.push_str(&format!("create or replace secret (type {typ}, {opts});")); + // Public storage and catalogs with inlined data need no S3 secret. + if !opts.is_empty() { + ret.push(format!("create or replace secret (type {typ}, {opts})")); + } } ret } - pub(super) fn get_settings_sql(&self, svr_opts: &ServerOptions) -> String { + pub(super) fn get_settings_sql(&self, svr_opts: &ServerOptions) -> Vec { let settings: Vec<(&str, String)> = match self { Self::MotherDuck => { let token = if svr_opts.contains_key("vault_motherduck_token") { @@ -186,16 +199,33 @@ impl ServerType { ] } }; - let mut ret = String::default(); - for (key, value) in settings { - ret.push_str(&format!("set {key}={value};")); - } - - ret + settings + .into_iter() + .map(|(key, value)| format!("set {key}={value}")) + .collect() } pub(super) fn get_attach_sql(&self, svr_opts: &ServerOptions) -> DuckdbFdwResult { let ret = match self { + Self::DuckLake => { + let metadata_path = if let Some(secret_id) = svr_opts.get("vault_metadata_path") { + get_vault_secret(secret_id).ok_or_else(|| { + OptionsError::OptionNameNotFound("vault_metadata_path".to_string()) + })? + } else { + require_option("metadata_path", svr_opts)?.to_string() + }; + let opts = self.format_options(svr_opts, &["data_path", "metadata_schema"]); + let opts = if opts.is_empty() { + String::new() + } else { + format!(", {opts}") + }; + format!( + "attach 'ducklake:{}' as ducklake (read_only, create_if_not_exists false{opts})", + metadata_path.replace("'", "''") + ) + } Self::S3Tables => { let arn = require_option("s3_tables_arn", svr_opts)?; let db_name = self.as_str(); diff --git a/wrappers/src/fdw/duckdb_fdw/tests.rs b/wrappers/src/fdw/duckdb_fdw/tests.rs index 76954ac9..52b3aeb4 100644 --- a/wrappers/src/fdw/duckdb_fdw/tests.rs +++ b/wrappers/src/fdw/duckdb_fdw/tests.rs @@ -5,6 +5,150 @@ mod tests { use serde_json::json; use std::str::FromStr; + use super::super::server_type::ServerType; + + #[pg_test] + fn ducklake_import_foreign_schema() { + let options = [ + ("type", "ducklake"), + ( + "metadata_path", + "postgres:host=localhost port=5433 dbname=ducklake user=ducklake password=ducklake", + ), + ("metadata_schema", "fdw_test"), + ("key_id", "admin"), + ("secret", "password"), + ("region", "us-east-1"), + ("endpoint", "localhost:8000"), + ("url_style", "path"), + ("use_ssl", "false"), + ] + .map(|(key, value)| (key.to_string(), value.to_string())) + .into_iter() + .collect(); + let server_type = ServerType::new(&options).unwrap(); + + // Provision an existing catalog independently of the read-only FDW. + let conn = duckdb::Connection::open_in_memory().unwrap(); + for sql in server_type + .get_duckdb_extension_sql() + .into_iter() + .chain(server_type.get_create_secret_sql(&options)) + { + conn.execute(&sql, []).unwrap(); + } + conn + .execute_batch( + "attach 'ducklake:postgres:host=localhost port=5433 dbname=ducklake user=ducklake password=ducklake' + as lake (metadata_schema 'fdw_test', data_path 's3://warehouse/ducklake_fdw_test/', + data_inlining_row_limit 0); + create schema if not exists lake.inventory; + create or replace table lake.inventory.products as + select 1::integer as id, 'Apple' as name + union all select 2, 'Pear'; + create or replace table lake.inventory.excluded (id integer); + detach lake;", + ) + .unwrap(); + + // Reattach with the FDW's read-only settings. + for sql in server_type + .get_settings_sql(&options) + .into_iter() + .chain([server_type.get_attach_sql(&options).unwrap()]) + { + conn.execute(&sql, []).unwrap(); + } + assert!( + conn.execute( + "insert into ducklake.inventory.products values (3, 'Plum')", + [] + ) + .is_err() + ); + assert!( + conn.prepare("select * from read_text('/etc/passwd')") + .and_then(|mut stmt| stmt.query([]).map(|_| ())) + .is_err() + ); + + Spi::run( + "CREATE FOREIGN DATA WRAPPER duckdb_wrapper + HANDLER duckdb_fdw_handler VALIDATOR duckdb_fdw_validator; + CREATE SERVER ducklake_server FOREIGN DATA WRAPPER duckdb_wrapper OPTIONS ( + type 'ducklake', + metadata_path 'postgres:host=localhost port=5433 dbname=ducklake user=ducklake password=ducklake', + metadata_schema 'fdw_test', + key_id 'admin', secret 'password', region 'us-east-1', + endpoint 'localhost:8000', url_style 'path', use_ssl 'false' + ); + CREATE SCHEMA ducklake_all; + CREATE SCHEMA ducklake_limited; + CREATE SCHEMA ducklake_except; + IMPORT FOREIGN SCHEMA inventory FROM SERVER ducklake_server INTO ducklake_all; + IMPORT FOREIGN SCHEMA inventory LIMIT TO (products) + FROM SERVER ducklake_server INTO ducklake_limited; + IMPORT FOREIGN SCHEMA inventory EXCEPT (excluded) + FROM SERVER ducklake_server INTO ducklake_except;", + ) + .unwrap(); + + for schema in ["ducklake_all", "ducklake_limited", "ducklake_except"] { + assert_eq!( + Spi::get_one::(&format!( + "select name from {schema}.products where id > 0 order by id desc limit 1" + )) + .unwrap(), + Some("Pear".to_string()) + ); + } + assert_eq!( + Spi::get_one::( + "select count(*) from information_schema.foreign_tables + where foreign_table_schema in ('ducklake_all', 'ducklake_limited', 'ducklake_except')" + ) + .unwrap(), + Some(4) + ); + assert_eq!( + Spi::get_one::("select count(*) from ducklake_all.excluded").unwrap(), + Some(0) + ); + } + + #[pg_test] + fn ducklake_options_preserve_quoted_values() { + let options = [ + ("type", "ducklake"), + ( + "metadata_path", + "postgres:dbname=catalog password=quo'te;value", + ), + ("metadata_schema", "custom's;schema"), + ("key_id", "test"), + ("secret", "quo'te;value"), + ] + .map(|(key, value)| (key.to_string(), value.to_string())) + .into_iter() + .collect(); + let server_type = ServerType::new(&options).unwrap(); + let attach = server_type.get_attach_sql(&options).unwrap(); + assert!(attach.contains("password=quo''te;value'")); + assert!(attach.contains("metadata_schema 'custom''s;schema'")); + assert!(attach.contains("read_only, create_if_not_exists false")); + + let conn = duckdb::Connection::open_in_memory().unwrap(); + conn.execute("install httpfs", []).unwrap(); + conn.execute("load httpfs", []).unwrap(); + for sql in server_type.get_create_secret_sql(&options) { + conn.execute(&sql, []).unwrap(); + } + + let empty_options = Default::default(); + assert!(server_type.get_attach_sql(&empty_options).is_err()); + assert!(server_type.get_create_secret_sql(&empty_options).is_empty()); + } + #[pg_test] fn duckdb_smoketest() { Spi::connect_mut(|c| {