From cb56443648b1ab08e58ae6eaad718b2db5f2560c Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 8 Sep 2026 06:39:38 -0700 Subject: [PATCH 1/3] feat: add S3 gateway service --- Cargo.lock | 342 ++- Cargo.toml | 2 + crab/docs/architecture/crab-s3-gateway.md | 20 +- crab/docs/architecture/s3-gateway-contract.md | 268 ++ crates/crab-http-server/src/maintenance.rs | 80 +- crates/crab-s3-gateway/Cargo.toml | 47 + crates/crab-s3-gateway/Dockerfile | 15 + crates/crab-s3-gateway/README.md | 45 + .../crab-s3-gateway/s3-gateway.example.toml | 19 + crates/crab-s3-gateway/src/attributes.rs | 164 ++ crates/crab-s3-gateway/src/auth.rs | 64 + crates/crab-s3-gateway/src/config.rs | 230 ++ crates/crab-s3-gateway/src/gateway.rs | 2248 +++++++++++++++++ crates/crab-s3-gateway/src/lib.rs | 49 + crates/crab-s3-gateway/src/main.rs | 28 + crates/crab-s3-gateway/src/multipart.rs | 467 ++++ crates/crab-s3-gateway/src/mutation.rs | 870 +++++++ crates/crab-s3-gateway/src/namespace.rs | 140 + crates/crab-s3-gateway/src/repository.rs | 89 + crates/crab-s3-gateway/src/server.rs | 81 + crates/crab-write/src/generation.rs | 79 +- 21 files changed, 5156 insertions(+), 191 deletions(-) create mode 100644 crab/docs/architecture/s3-gateway-contract.md create mode 100644 crates/crab-s3-gateway/Cargo.toml create mode 100644 crates/crab-s3-gateway/Dockerfile create mode 100644 crates/crab-s3-gateway/README.md create mode 100644 crates/crab-s3-gateway/s3-gateway.example.toml create mode 100644 crates/crab-s3-gateway/src/attributes.rs create mode 100644 crates/crab-s3-gateway/src/auth.rs create mode 100644 crates/crab-s3-gateway/src/config.rs create mode 100644 crates/crab-s3-gateway/src/gateway.rs create mode 100644 crates/crab-s3-gateway/src/lib.rs create mode 100644 crates/crab-s3-gateway/src/main.rs create mode 100644 crates/crab-s3-gateway/src/multipart.rs create mode 100644 crates/crab-s3-gateway/src/mutation.rs create mode 100644 crates/crab-s3-gateway/src/namespace.rs create mode 100644 crates/crab-s3-gateway/src/repository.rs create mode 100644 crates/crab-s3-gateway/src/server.rs diff --git a/Cargo.lock b/Cargo.lock index 0283410de..fba4232e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -213,7 +213,7 @@ dependencies = [ "arrow-data", "arrow-schema", "arrow-select", - "atoi", + "atoi 2.0.0", "base64 0.22.1", "chrono", "half", @@ -403,6 +403,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.6.1" @@ -446,7 +455,7 @@ dependencies = [ "bytes", "fastrand 2.4.1", "hex", - "http 1.4.0", + "http 1.4.1", "sha1 0.10.6", "time", "tokio", @@ -508,7 +517,7 @@ dependencies = [ "bytes-utils", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body 0.4.6", "http-body 1.0.1", "percent-encoding", @@ -537,7 +546,7 @@ dependencies = [ "bytes", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] @@ -563,7 +572,7 @@ dependencies = [ "aws-types", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] @@ -593,7 +602,7 @@ dependencies = [ "hex", "hmac 0.13.0", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "lru 0.16.4", "percent-encoding", @@ -623,7 +632,7 @@ dependencies = [ "aws-types", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "md-5", "regex-lite", "tracing", @@ -650,7 +659,7 @@ dependencies = [ "bytes", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] @@ -675,7 +684,7 @@ dependencies = [ "bytes", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] @@ -701,7 +710,7 @@ dependencies = [ "aws-types", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "regex-lite", "tracing", ] @@ -723,7 +732,7 @@ dependencies = [ "hex", "hmac 0.13.0", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "p256", "percent-encoding", "sha2 0.11.0", @@ -746,16 +755,16 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.64.7" +version = "0.64.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10efbbcec1e044b81600e2fc562a391951d291152d95b482d5b7e7132299d762" +checksum = "e9e8e65f4f81fcccdeb6c3eca2af17ac21d421a1786a26a394aecf421d616d3a" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "crc-fast", "hex", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "md-5", @@ -789,7 +798,7 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "percent-encoding", @@ -808,12 +817,12 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.13", + "h2 0.4.19", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", @@ -874,7 +883,7 @@ dependencies = [ "bytes", "fastrand 2.4.1", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -895,7 +904,7 @@ dependencies = [ "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "pin-project-lite", "tokio", "tracing", @@ -921,7 +930,7 @@ checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.4.0", + "http 1.4.1", ] [[package]] @@ -935,7 +944,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.12", - "http 1.4.0", + "http 1.4.1", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -984,7 +993,7 @@ dependencies = [ "axum-core 0.4.5", "bytes", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "itoa", @@ -1011,10 +1020,10 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-util", "itoa", "matchit 0.8.4", @@ -1043,7 +1052,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "mime", @@ -1062,7 +1071,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "mime", @@ -1083,9 +1092,9 @@ dependencies = [ "bytes", "either", "fs-err", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-util", "pin-project-lite", "rustls 0.23.40", @@ -1493,6 +1502,15 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" +[[package]] +name = "bytestring" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86566c496f2f47d9b8147a4c8b02ffdb69c919fe0c2b2e7195d22cbba0e635c9" +dependencies = [ + "bytes", +] + [[package]] name = "cc" version = "1.2.61" @@ -1560,9 +1578,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -1674,7 +1692,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2386,6 +2404,50 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "crab-s3-gateway" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "blake3", + "bytes", + "clap", + "crab-cache-store", + "crab-coordination", + "crab-git", + "crab-lfs", + "crab-metadata", + "crab-read", + "crab-remote-git", + "crab-storage", + "crab-write", + "crc-fast", + "flate2", + "futures-util", + "gix-hash", + "gix-object", + "http 1.4.1", + "hyper-util", + "md-5", + "object_store", + "percent-encoding", + "s3s", + "serde", + "serde_json", + "sha1 0.10.6", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-util", + "toml", + "tracing", + "tracing-subscriber", + "ulid", +] + [[package]] name = "crab-staging" version = "0.1.0" @@ -2570,30 +2632,13 @@ dependencies = [ "xet-runtime", ] -[[package]] -name = "crc" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" - [[package]] name = "crc-fast" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "crc", "digest 0.10.7", - "rustversion", "spin 0.10.0", ] @@ -2897,7 +2942,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3035,7 +3080,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4069,7 +4114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b305d85504de270ad3525d726a6b69cc59ee7b2269b014387651107ab9f0755b" dependencies = [ "bstr", - "hashbrown 0.17.0", + "hashbrown 0.16.1", ] [[package]] @@ -4728,16 +4773,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http 1.4.1", "indexmap 2.14.0", "slab", "tokio", @@ -4863,6 +4908,16 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7685beb53fc20efc2605f32f5d51e9ba18b8ef237961d1760169d2290d3bee" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -4912,9 +4967,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -4938,7 +4993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http 1.4.1", ] [[package]] @@ -4949,7 +5004,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "pin-project-lite", ] @@ -5033,16 +5088,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.13", - "http 1.4.0", + "h2 0.4.19", + "http 1.4.1", "http-body 1.0.1", "httparse", "httpdate", @@ -5074,8 +5129,8 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", - "hyper 1.9.0", + "http 1.4.1", + "hyper 1.10.1", "hyper-util", "rustls 0.23.40", "rustls-native-certs", @@ -5091,7 +5146,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.9.0", + "hyper 1.10.1", "hyper-util", "pin-project-lite", "tokio", @@ -5106,7 +5161,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-util", "native-tls", "tokio", @@ -5124,14 +5179,14 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", - "hyper 1.9.0", + "hyper 1.10.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -5927,9 +5982,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" @@ -5977,7 +6032,7 @@ checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" dependencies = [ "base64 0.22.1", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-rustls 0.27.9", "hyper-util", "indexmap 2.14.0", @@ -6204,7 +6259,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6350,6 +6405,12 @@ dependencies = [ "libc", ] +[[package]] +name = "numeric_cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093" + [[package]] name = "oauth2" version = "4.4.2" @@ -6378,7 +6439,7 @@ dependencies = [ "base64 0.22.1", "chrono", "getrandom 0.2.17", - "http 1.4.0", + "http 1.4.1", "rand 0.8.6", "reqwest 0.12.28", "serde", @@ -6433,11 +6494,11 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body-util", "httparse", "humantime", - "hyper 1.9.0", + "hyper 1.10.1", "itertools 0.15.0", "md-5", "nix 0.31.3", @@ -6495,7 +6556,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "hmac 0.12.1", - "http 1.4.0", + "http 1.4.1", "itertools 0.10.5", "log", "oauth2 5.0.0", @@ -6581,7 +6642,7 @@ checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", - "http 1.4.0", + "http 1.4.1", "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", @@ -7193,7 +7254,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.40", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -7231,7 +7292,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.59.0", ] @@ -7550,10 +7611,10 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-rustls 0.27.9", "hyper-tls", "hyper-util", @@ -7596,11 +7657,11 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2 0.4.13", - "http 1.4.0", + "h2 0.4.19", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-rustls 0.27.9", "hyper-util", "js-sys", @@ -7635,7 +7696,7 @@ checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" dependencies = [ "anyhow", "async-trait", - "http 1.4.0", + "http 1.4.1", "reqwest 0.12.28", "serde", "thiserror 1.0.69", @@ -7650,7 +7711,7 @@ checksum = "199dda04a536b532d0cc04d7979e39b1c763ea749bf91507017069c00b96056f" dependencies = [ "anyhow", "async-trait", - "http 1.4.0", + "http 1.4.1", "reqwest 0.13.3", "thiserror 2.0.18", "tower-service", @@ -7785,7 +7846,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7865,7 +7926,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7920,6 +7981,58 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "s3s" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe1bd31748cb69848c2cf4028cecdadf5f8299ef3b02a83e21b20e7bda3aba9" +dependencies = [ + "arc-swap", + "arrayvec", + "async-trait", + "atoi 3.1.0", + "base64-simd", + "bytes", + "bytestring", + "cfg-if 1.0.4", + "chrono", + "crc-fast", + "futures", + "hex-simd", + "hmac 0.13.0", + "http 1.4.1", + "http-body 1.0.1", + "http-body-util", + "httparse", + "hyper 1.10.1", + "itoa", + "md-5", + "memchr", + "mime", + "nom", + "numeric_cast", + "pin-project-lite", + "quick-xml 0.41.0", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", + "sha2 0.11.0", + "smallvec", + "std-next", + "subtle", + "sync_wrapper", + "thiserror 2.0.18", + "time", + "tokio", + "tower 0.5.3", + "tracing", + "transform-stream", + "url", + "urlencoding", + "zeroize", +] + [[package]] name = "safe-transmute" version = "0.11.3" @@ -8127,9 +8240,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -8582,6 +8695,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "std-next" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc" +dependencies = [ + "simdutf8", + "thiserror 2.0.18", +] + [[package]] name = "strsim" version = "0.11.1" @@ -8718,7 +8841,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8851,9 +8974,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -9050,11 +9173,11 @@ dependencies = [ "axum 0.7.9", "base64 0.22.1", "bytes", - "h2 0.4.13", - "http 1.4.0", + "h2 0.4.19", + "http 1.4.1", "http-body 1.0.1", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -9115,7 +9238,7 @@ dependencies = [ "bitflags 2.11.1", "bytes", "futures-util", - "http 1.4.0", + "http 1.4.1", "http-body 1.0.1", "http-body-util", "iri-string", @@ -9255,6 +9378,15 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "transform-stream" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a814d25437963577f6221d33a2aaa60bfb44acc3330cdc7c334644e9832022" +dependencies = [ + "futures-core", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -9749,7 +9881,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10217,8 +10349,8 @@ dependencies = [ "bytes", "crc32fast", "futures", - "http 1.4.0", - "hyper 1.9.0", + "http 1.4.1", + "hyper 1.10.1", "more-asserts", "rand 0.10.1", "redb", @@ -10285,7 +10417,7 @@ dependencies = [ "bytes", "chrono", "gearhash", - "http 1.4.0", + "http 1.4.1", "itertools 0.14.0", "more-asserts", "rand 0.10.1", diff --git a/Cargo.toml b/Cargo.toml index 0bfd36527..7b2185f05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/crab-metadata", "crates/crab-read", "crates/crab-remote-git", + "crates/crab-s3-gateway", "crates/crab-staging", "crates/crab-storage", "crates/crab-types", @@ -43,6 +44,7 @@ crab-lfs = { path = "crates/crab-lfs", default-features = false } crab-metadata = { path = "crates/crab-metadata", default-features = false } crab-read = { path = "crates/crab-read", default-features = false } crab-remote-git = { path = "crates/crab-remote-git", default-features = false } +crab-s3-gateway = { path = "crates/crab-s3-gateway", default-features = false } crab-staging = { path = "crates/crab-staging", default-features = false } crab-storage = { path = "crates/crab-storage", default-features = false } crab-types = { path = "crates/crab-types", default-features = false } diff --git a/crab/docs/architecture/crab-s3-gateway.md b/crab/docs/architecture/crab-s3-gateway.md index 5eb7da274..62ae33050 100644 --- a/crab/docs/architecture/crab-s3-gateway.md +++ b/crab/docs/architecture/crab-s3-gateway.md @@ -1,11 +1,14 @@ # Crab S3 gateway: executable design and phased implementation plan -Status: implementation plan; all phases below are pending. Gateway crate and -protocol support are not implemented. Proposed product defaults require the -phase-0 decision record before dependent implementation. +Status: initial gateway implemented in `crates/crab-s3-gateway`; broader +cross-client, cross-provider, failure-injection, and deployment qualification +remains a release gate. The frozen delivered surface and deliberate limits are +recorded in `s3-gateway-contract.md`. Depends on the [SDK delivery plan](crab-sdk.md), especially remote writes, publication recovery and backend qualification. This document does not mark any -SDK or gateway capability delivered. +SDK capability delivered beyond the shared contracts used by the gateway. +Where this phased plan retains proposed or future work, the frozen contract is +the authority for the currently delivered behavior. ## Outcome and ownership @@ -81,19 +84,18 @@ and parent components; Git trees also cannot contain both file `a` and file lossless object-key representation, including folder markers and Git-client round trips. No silent normalization, dropped markers or false full-key parity. -## Versioning and write visibility: decisions pending +## Versioning and write visibility -Recommended initial model: each successful object mutation publishes a commit; +Selected initial model: each successful object mutation publishes a commit; multipart parts remain invisible until completion. Reads pin a commit per request. Writable branches advance through expected-OID publication, while tags and commit snapshots are read-only. Unrelated concurrent file writes must not overwrite each other; bounded re-preparation must preserve object preconditions. Reauthorize and recheck conditions at publication, not just upload admission. -The alternative is durable uncommitted branch state plus explicit commit. +The rejected initial alternative is durable uncommitted branch state plus explicit commit. That requires shared overlay/read/commit semantics beyond the current SDK plan; it must not be introduced as an invisible gateway-only second repository state. -The product choice is pending user input. Repository versioning and AWS object versioning are separate contracts. Full S3 versioning additionally needs `GetBucketVersioning`, `PutBucketVersioning`, @@ -1569,4 +1571,4 @@ because its code exists, and no later phase waives an earlier missing proof. - [GET conditions and ranges](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) - [PUT metadata and preconditions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) - [Multi-delete results](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) -- [Multipart completion and embedded errors](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html) \ No newline at end of file +- [Multipart completion and embedded errors](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html) diff --git a/crab/docs/architecture/s3-gateway-contract.md b/crab/docs/architecture/s3-gateway-contract.md new file mode 100644 index 000000000..2e1bc8fcf --- /dev/null +++ b/crab/docs/architecture/s3-gateway-contract.md @@ -0,0 +1,268 @@ +# Crab S3 gateway protocol contract + +Status: accepted initial-release contract. This record resolves the phase-0 +choices in `crab-s3-gateway.md`. Implementation and qualification status is +tracked by the gateway crate and its test reports, not by this document. + +## Release model + +The gateway exposes existing Crab repositories through the S3 REST protocol. +Each successful `PutObject`, `CopyObject`, `DeleteObject`, or completed multipart +upload publishes one commit immediately to the addressed branch. `DeleteObjects` +performs ordered, individually reported mutations and is not atomic across keys. +A delete of a missing key succeeds without advancing the branch. + +Repository history is the only version model. The initial release does not +implement AWS bucket versioning, delete markers, or version-ID parameters. A +branch names a mutable view; a tag or full commit ID names an immutable read-only +view. Every request resolves and pins one commit before reading. Writes recheck +authorization, branch protection, and the branch tip under the canonical per-ref +publication lock. A conflicting branch update returns `OperationAborted` and is +safe for the S3 client to retry. Conditional PUT and DELETE headers are not part +of the initial surface and return `NotImplemented` before reading the body. + +## Bucket and key namespace + +One configured logical repository is one S3 bucket. Bucket names are configured +explicitly and are unique ignoring ASCII case. They contain 3–63 lowercase +ASCII letters, digits, dots, and hyphens, start and end with a letter or digit, +and are never derived from a backing provider bucket. Backing bucket and prefix +values never appear in S3 responses. + +An object key is `REF/KEY`. The first slash separates the encoded ref segment +from the repository path. A short ref selects `refs/heads/REF`; +`refs%2Fheads%2F...` and `refs%2Ftags%2F...` select a fully qualified ref; a +40-digit hexadecimal segment selects a commit. A branch/tag collision is +impossible for a short ref because short refs select branches only. Abbreviated +object IDs and Git revision expressions such as `~`, `^`, and `@{}` are rejected. +Writes require a branch ref. + +The gateway verifies the signature against the original HTTP path and query, +then applies HTTP percent-decoding once. It splits the resulting S3 key at the +first literal slash, percent-decodes the ref segment once more, and does not +decode the remaining repository path again. A literal percent in a ref is +therefore represented in the logical key as `%25` and on the HTTP wire as +`%2525`. Copy source parsing follows the same rule independently of the +destination URI. + +Examples: + +| Logical URI | S3 bucket/key | Raw path | Ref | Repository path | +| --- | --- | --- | --- | --- | +| `crabfs://demo/main/a.txt` | `demo`, `main/a.txt` | `/demo/main/a.txt` | `refs/heads/main` | `a.txt` | +| `crabfs://demo/feature%2Fdata/a.txt` | `demo`, `feature%2Fdata/a.txt` | `/demo/feature%252Fdata/a.txt` | `refs/heads/feature/data` | `a.txt` | +| `crabfs://demo/refs%2Ftags%2Fv1/a%2Fb` | `demo`, `refs%2Ftags%2Fv1/a%2Fb` | `/demo/refs%252Ftags%252Fv1/a%252Fb` | `refs/tags/v1` | literal `a%2Fb` | +| `crabfs://demo/0123456789012345678901234567890123456789/a` | same bucket/key | `/demo/0123456789012345678901234567890123456789/a` | that commit | `a` | + +`HeadBucket` addresses the repository. `ListObjects` and `ListObjectsV2` require +a ref in `prefix`; listing with an empty prefix returns authorized branch +prefixes only. Tag and commit namespaces are not synthesized at repository root. +The empty key, a ref without a trailing slash, and a ref root are prefixes, not +objects. `GetObject`, `HeadObject`, and mutations require a non-empty repository +path. + +## Supported key profile + +S3 keys are restricted to paths representable without loss in a Git tree. The +complete logical key is at most 1024 UTF-8 bytes. Its repository path is +non-empty, uses `/` separators, and has components of 1–255 bytes. Empty, `.`, +`..`, and case-insensitive `.git` components are rejected. NUL, ASCII control +characters, repeated separators, leading or trailing separators, and trailing +folder-marker objects are rejected. The gateway never normalizes Unicode or +path separators. + +Writes create ordinary non-executable Git blobs. They reject a path whose +ancestor is a blob or whose existing entry is a tree. Overwriting a symlink, +executable, or submodule replaces that entry with an ordinary blob; deleting it +removes the entry. Reads expose pre-existing ordinary and executable blobs as +objects; symlinks and submodules return `InvalidObjectState`. A Git tree cannot +contain both `a` and `a/b`, and the gateway reports the conflict instead of +inventing a lossless side namespace. These restrictions are intentional +compatibility limits, never silent transformations. + +## Authentication and authorization + +The gateway requires Crab-issued access-key credentials and accepts S3 SigV4 +header signing, SigV4 presigned queries, SigV4 streaming payloads supported by +the selected protocol library, SigV2 headers, and SigV2 presigned queries. +Unsigned requests return `AccessDenied`. Header-signed requests allow 15 minutes +of clock skew. Presigned request expiry is verified by the protocol layer; +operators should issue SigV4 URLs for no more than seven days. + +Access-key lookup yields current HMAC verification material and one Crab +principal. Unknown keys fail before repository authorization. Static key +rotation or revocation takes effect when the process reloads its configuration. +Gateway keys are never backend cloud credentials. Secret values must come from +protected files, never command-line arguments, logs, error bodies, persisted +multipart records, or reports. + +Every request authorizes its logical repository, ref, path, and action after +signature verification. Historical commit and tag reads require current +repository read permission. Copy independently authorizes the pinned source and +destination. Direct writes to protected branches return `AccessDenied`. + +## Object representation + +The object ETag is the quoted lowercase hexadecimal MD5 of logical object bytes. +It is stable across metadata-only changes but is not a Git OID. Multipart ETags +use the S3-compatible quoted MD5 of concatenated binary part MD5 values followed +by `-PART_COUNT`. ETags are validators, not integrity claims beyond the exact +response contract. + +Object attributes are stored in an immutable versioned manifest named by the +new commit and uploaded before that commit becomes reachable. Readers select the +manifest by their pinned commit, so tree bytes and attributes cannot be mixed +across versions. Attributes contain user metadata, `Content-Type`, +`Content-Encoding`, `Content-Disposition`, `Content-Language`, `Cache-Control`, +`Expires`, ETag, logical size, and modification time. Ordinary Git commits that +lack an attribute entry project an empty metadata map, inferred +`application/octet-stream`, an ETag computed from logical bytes, and the +selected commit's committer time. A content write replaces prior attributes; +`CopyObject` with `COPY` copies them and `REPLACE` uses request attributes. +Deletes remove the current attribute entry. Renames and merges performed outside +the gateway follow normal Git projection until a later gateway write commits an +explicit entry. + +`Last-Modified` is the committed attribute modification time, rounded to whole +seconds. A metadata-only write changes `Last-Modified` while retaining the ETag. +Content and attributes are never read from different commits. + +## Protocol surface + +The initial release supports path-style addressing. Virtual-hosted addressing +is not configured by the gateway. HTTPS is mandatory beyond loopback and is +terminated by the deployment ingress. Browser POST, bucket create/delete, ACLs, policies, IAM, +versioning, tagging, Select, object lock, retention, torrent, website, inventory, +replication, acceleration, notification, storage-class selection, and all SSE +request headers return `NotImplemented` or the operation-specific documented S3 +error before mutation. + +Supported operations: + +| Operation | Supported contract | +| --- | --- | +| `ListBuckets`, `HeadBucket` | Authorized logical repositories only; deterministic order | +| `GetObject`, `HeadObject` | metadata, response overrides, RFC dates, ETag/date conditions, one byte range including open and suffix forms | +| `ListObjects`, `ListObjectsV2` | prefix, delimiter `/`, marker/start-after, max keys, reusable keys, common prefixes | +| `PutObject` | body up to 256 MiB, `Content-MD5`, SigV4 payload hash, CRC32/CRC32C/CRC64NVME/SHA1/SHA256 checksums, metadata and standard content headers | +| `DeleteObject`, `DeleteObjects` | S3 missing-key success, per-key authorization/results, quiet mode, and at most 1000 XML entries | +| `CopyObject` | pinned source, source conditions/range where defined, `COPY`/`REPLACE`, separately authorized destination | +| Multipart create/upload/copy/list/abort/complete | durable opaque sessions, part replacement, ordered selection, 10,000-part and 256 MiB completed-object limits, restart and multi-instance retry | + +Modeled unsupported request headers and query parameters are rejected rather +than ignored. Multi-range GET returns `InvalidRange`. `versionId` returns +`NotImplemented`. Checksums are validated before publication. A response never +attaches a full-object checksum to a partial range unless the protocol defines +the matching checksum mode. + +Conditional reads use S3 precedence: match conditions are evaluated before +unmodified conditions, then modified conditions; a failed read condition returns +`NotModified` or `PreconditionFailed` as defined by that header. Conditional +PUT, DELETE, multipart completion, and destination COPY are not in the initial +surface. + +## Listings and continuation + +Keys are ordered by their complete UTF-8 byte representation, including the +encoded ref prefix. `delimiter=/` groups each subtree once and counts a common +prefix against `MaxKeys`. V1 markers are visible keys and each page resolves the +current branch according to S3's non-snapshot behavior. + +V2 continuation tokens are the last emitted raw key or common prefix. They are +portable across gateway nodes and are reauthorized when the next request is +handled. Like V1 markers, they resume against the branch state current for that +request; listings are not snapshot-pinned. `encoding-type=url` percent-encodes +the S3-defined response fields but does not transform the continuation token. + +## Multipart durability + +Gateway multipart state is provider-neutral and separate from native provider +multipart uploads. The durable catalog uses versioned records and conditional +updates. An upload records its repository placement, branch/key, initiator, +attributes, timestamp, state, revision, registered immutable parts, the frozen +completion request, and the terminal +completion ETag. The committed attribute manifest carries the upload identity +so an identical retry can recognize a publication that completed before its +multipart record reached the terminal state. + +The state machine is `Open -> Completing -> Completed` or `Open -> Aborted`. +Registration, freeze, and abort use conditional revisions. A network transfer +does not hold a branch lock. A part is acknowledged only after its immutable +bytes and catalog registration are durable. Replacing a part number swaps the +record to a new immutable object. Frozen and active objects remain GC roots. + +Completion requires 1–10,000 strictly ascending selected parts, matching quoted +ETags, and at least 5 MiB for every selected part except the last. It freezes the +exact ordered identities before execution and persists the committed response. +An identical retry returns the recorded result; a different selection cannot +cause another mutation. An uncertain result remains `Completing` and an +identical retry resumes from the frozen part set. + +List APIs are ordered and bounded, reauthorize every page, exclude terminal and +replaced transfers, and work without process-local iterator state. Abort first +persists the terminal state, then synchronously removes part objects. Completion +persists its terminal outcome before best-effort part cleanup; a cleanup failure +does not erase the completed outcome. + +Limits: 10,000 parts per upload, 256 MiB per logical object read, single PUT, +part, and final multipart object, and 1000 results per multipart listing page. +Unknown-length streams are counted as they arrive. Larger objects require later +streaming reader and content-writer implementations; the gateway returns +`EntityTooLarge` instead of risking unbounded memory or a non-durable upload. + +## Error and response contract + +Errors use S3 XML with a stable `Code` and safe `Message`. HEAD errors have a +status and headers but no body. Internal source errors are logged at the service +boundary without credentials or request bodies. + +| Condition | S3 code | +| --- | --- | +| Unknown/revoked credential or invalid signature | `InvalidAccessKeyId` / `SignatureDoesNotMatch` | +| Missing authentication or denied repository/path | `AccessDenied` | +| Unknown logical repository | `NoSuchBucket` | +| Missing object | `NoSuchKey` | +| Invalid ref/key/encoding, body, XML, header, checksum | `InvalidArgument`, `MalformedXML`, or `BadDigest` | +| Unsupported operation/feature/version | `NotImplemented` | +| Failed condition | `PreconditionFailed` or `NotModified` | +| Invalid range | `InvalidRange` | +| Branch contention or transient shared-state conflict | `OperationAborted` | +| Missing/terminal multipart session | `NoSuchUpload` | +| Bad completion selection/order/size | `InvalidPart`, `InvalidPartOrder`, `EntityTooSmall` | +| Admission capacity limit | `SlowDown` | +| Corrupt/unavailable committed data | `InternalError` | + +The initial implementation materializes each bounded object before starting a +response, so backend failures still return an S3 error rather than truncating a +successful response stream. Successful writes are returned only after their +committed outcome is durable and read-ready. + +## Repository extension API + +S3 methods are not repurposed for Git concepts. A separate authenticated +`/crab/v1/repositories/{bucket}` API may expose ref listing, bounded history, +diff, and expected-OID branch/tag create/update/delete. Merge and explicit +staged commits remain excluded until their shared SDK contracts are specified. +`crabfs://` consumers need a filesystem/scheme adapter; configuring an S3 +endpoint alone does not teach a library that URI scheme. + +## Qualification matrix + +Release qualification covers the AWS CLI v2, current AWS SDKs for Rust, Python +(boto3), JavaScript v3, Java v2, Go v2, and `s3cmd`. It runs against S3, GCS, and +Azure-backed Crab repositories through the provider-neutral storage layer. +Path-style HTTP is permitted only for loopback tests; deployment suites use HTTPS. + +Every supported operation requires positive, protocol-error, authorization, +restart/two-instance, and independent Git/SDK visibility evidence. Multipart, +publication, and listing state is qualified under process termination and +concurrency. Unsupported/skipped cells make a report incomplete rather than +passing. Backend/client versions, source SHA and dirty digest, fixture identity, +assertion and byte counts, latency, RSS, scratch usage, and terminal state are +recorded without credentials. + +The checked-in implementation currently has unit coverage for namespace, +multipart persistence/retry, checksum validation, and branch-preserving Git +mutation. Local release qualification additionally runs the AWS CLI against a +RustFS-backed Crab repository. The broader client/backend matrix remains a +release gate, not an inferred claim from that local smoke test. diff --git a/crates/crab-http-server/src/maintenance.rs b/crates/crab-http-server/src/maintenance.rs index b96580607..186fca968 100644 --- a/crates/crab-http-server/src/maintenance.rs +++ b/crates/crab-http-server/src/maintenance.rs @@ -1,9 +1,5 @@ use std::{sync::Arc, time::Duration}; -use crab_coordination::{ - CoordinationError, GIT_GENERATION_OWNER_RESOURCE, GcFenceHeartbeat, GcFenceLease, - PushLockAcquireContext, -}; use crab_remote_git::{RemoteGitRuntime, RepositoryIdentity, RepositoryOptions}; use crab_storage::{Store, StoreLayout}; use crab_write::{Result, WriteError}; @@ -13,27 +9,6 @@ use tokio_util::sync::CancellationToken; const LEASE_TTL: Duration = Duration::from_secs(60); const PASS_BUDGET: Duration = Duration::from_secs(3 * 60); -struct WriterFence { - lease: GcFenceLease, - heartbeat: GcFenceHeartbeat, -} - -impl WriterFence { - async fn acquire(store: &Store, domain: &str, cancel: &CancellationToken) -> Result { - if cancel.is_cancelled() { - return Err(WriteError::Cancelled); - } - let lease = GcFenceLease::acquire_writer(store.inner(), domain, LEASE_TTL).await?; - let heartbeat = GcFenceHeartbeat::spawn(&lease, cancel.clone(), LEASE_TTL / 3); - Ok(Self { lease, heartbeat }) - } - - async fn release(self) -> Result<()> { - self.heartbeat.stop().await; - self.lease.release().await.map_err(Into::into) - } -} - async fn publish( store: &Store, layout: &StoreLayout, @@ -42,57 +17,10 @@ async fn publish( options: RepositoryOptions, cancel: &CancellationToken, ) -> Result<()> { - let mut context = PushLockAcquireContext::new(Arc::clone(store.inner())); - let mut owner = match context - .try_acquire_internal( - layout.repo_prefix(), - GIT_GENERATION_OWNER_RESOURCE, - LEASE_TTL, - ) - .await - { - Ok(owner) => owner, - // Another server or CLI owner is already responsible for publication. - Err(CoordinationError::PushLockHeld { .. }) => return Ok(()), - Err(error) => return Err(error.into()), - }; - let result = crab_coordination::while_renewing(&mut owner, Some(cancel), async { - let global = WriterFence::acquire(store, layout.global_prefix(), cancel).await?; - let repo = match WriterFence::acquire(store, layout.repo_prefix(), cancel).await { - Ok(repo) => repo, - Err(error) => { - let _ = global.release().await; - return Err(error); - } - }; - let mut result = async { - let (manifest, _) = crab_metadata::manifest_store::read_manifest(store, layout).await?; - let Some(manifest) = crab_write::generation::make_readable( - store, - layout, - LEASE_TTL, - manifest.pusher, - cancel, - ) - .await? - else { - return Ok(()); - }; - crab_write::generation::maintain_commit_graph( - store, layout, &manifest, identity, runtime, options, cancel, - ) - .await?; - Ok(()) - } - .await; - // Release both domains even if publication or an earlier release failed. - for fence in [repo, global] { - result = result.and(fence.release().await); - } - result - }) - .await; - result.and(owner.release().await.map_err(Into::into)) + crab_write::generation::ensure_readable( + store, layout, identity, runtime, options, LEASE_TTL, cancel, + ) + .await } pub(crate) async fn run( diff --git a/crates/crab-s3-gateway/Cargo.toml b/crates/crab-s3-gateway/Cargo.toml new file mode 100644 index 000000000..d81d87311 --- /dev/null +++ b/crates/crab-s3-gateway/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "crab-s3-gateway" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "S3-compatible gateway for Crab repositories." +publish = false + +[dependencies] +async-trait = { workspace = true } +base64 = "0.22" +bytes = { workspace = true } +blake3 = { workspace = true } +clap = { version = "4", features = ["derive"] } +crab-coordination = { workspace = true, features = ["object-store-lock"] } +crab-cache-store = { workspace = true } +crab-lfs = { workspace = true } +crab-metadata = { workspace = true, features = ["storage"] } +crab-remote-git = { workspace = true } +crab-read = { workspace = true } +crab-git = { workspace = true } +crab-storage = { workspace = true } +crab-write = { workspace = true } +crc-fast = "1" +flate2 = { workspace = true } +futures-util = { workspace = true } +gix-hash = { workspace = true, features = ["sha1"] } +gix-object = { workspace = true } +http = "1" +hyper-util = { version = "0.1", features = ["server-auto", "server-graceful", "http1", "http2", "tokio"] } +md-5 = { workspace = true } +object_store = { workspace = true } +percent-encoding = "2" +s3s = "=0.14.1" +serde = { workspace = true } +serde_json = { workspace = true } +sha1 = "0.10" +sha2 = "0.10" +tempfile = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio-util = { workspace = true, features = ["rt"] } +toml = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +ulid = "1" diff --git a/crates/crab-s3-gateway/Dockerfile b/crates/crab-s3-gateway/Dockerfile new file mode 100644 index 000000000..b5c4b4fc5 --- /dev/null +++ b/crates/crab-s3-gateway/Dockerfile @@ -0,0 +1,15 @@ +FROM rust:1.90-bookworm AS builder +WORKDIR /src +COPY . . +RUN cargo build --locked --release -p crab-s3-gateway + +FROM debian:bookworm-slim +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates tini \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 10001 crab +COPY --from=builder /src/target/release/crab-s3-gateway /usr/local/bin/crab-s3-gateway +USER crab +EXPOSE 8080 +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/crab-s3-gateway"] +CMD ["--config", "/etc/crab/s3-gateway.toml"] diff --git a/crates/crab-s3-gateway/README.md b/crates/crab-s3-gateway/README.md new file mode 100644 index 000000000..bdbb961c4 --- /dev/null +++ b/crates/crab-s3-gateway/README.md @@ -0,0 +1,45 @@ +# Crab S3 gateway + +`crab-s3-gateway` presents configured Crab repositories as S3 buckets. Existing +S3 clients use their normal endpoint, region, access-key, and secret-key +configuration. Object keys use `REF/path`, for example +`s3://my-repository/main/data/model.bin`. + +The gateway accepts S3 SigV4 and SigV2 authentication through `s3s`. It maps +each access key to a Crab principal and authorizes that principal against the +logical repository catalog. Gateway credentials are independent of the cloud +credentials used for the backing object store. + +The initial client-compatible surface includes bucket listing/head, object +GET/HEAD/PUT/DELETE/COPY, V1/V2 object listing, multi-delete, and durable +multipart create/upload/copy/list/abort/complete. GET/HEAD support conditions +and a single byte range; PUT validates Content-MD5 and the standard S3 checksum +headers. Path-style addressing is required. Object and completed multipart +payloads are limited to 256 MiB while the canonical streaming writer remains +future work. The complete frozen surface and deliberate exclusions are in the +protocol contract linked below. + +## Build and run + +```sh +cargo build --release -p crab-s3-gateway --locked +crab-s3-gateway --config /etc/crab/s3-gateway.toml --initialize +crab-s3-gateway --config /etc/crab/s3-gateway.toml +``` + +`--initialize` creates missing canonical Crab metadata only for empty configured +prefixes, then exits. It is safe to run repeatedly. Normal serving never +initializes or converts repository storage. + +The backing provider uses Crab's existing environment credential chain. Set +the usual AWS, GCP, or Azure credentials for the selected provider. For an +S3-compatible endpoint, `AWS_ENDPOINT_URL_S3`, `AWS_ALLOW_HTTP`, and +`AWS_VIRTUAL_HOSTED_STYLE_REQUEST` are supported by the shared storage layer. + +See `s3-gateway.example.toml` for configuration and +`crab/docs/architecture/s3-gateway-contract.md` for the protocol contract. +Terminate with SIGTERM or SIGINT for graceful connection draining. + +Production deployments should bind to a private listener and terminate TLS at +an ingress, load balancer, or service mesh. Do not expose the plain HTTP +listener beyond a trusted network boundary. diff --git a/crates/crab-s3-gateway/s3-gateway.example.toml b/crates/crab-s3-gateway/s3-gateway.example.toml new file mode 100644 index 000000000..5076c4050 --- /dev/null +++ b/crates/crab-s3-gateway/s3-gateway.example.toml @@ -0,0 +1,19 @@ +listen = "127.0.0.1:8080" +region = "us-east-1" + +[[credentials]] +access_key = "replace-with-issued-access-key" +secret_key_file = "/run/secrets/crab-s3-secret" +principal = "service-account:analytics" + +[[repositories]] +name = "analytics-repository" +provider = "s3" +bucket = "physical-storage-bucket" +prefix = "repositories/analytics" +default_branch = "main" +protected_branches = ["release"] + +[[repositories.members]] +principal = "service-account:analytics" +access = "write" diff --git a/crates/crab-s3-gateway/src/attributes.rs b/crates/crab-s3-gateway/src/attributes.rs new file mode 100644 index 000000000..15a20eb03 --- /dev/null +++ b/crates/crab-s3-gateway/src/attributes.rs @@ -0,0 +1,164 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use gix_hash::ObjectId; +use serde::{Deserialize, Serialize}; + +use crate::gateway::Repository; + +const VERSION: u32 = 1; +const MAX_MANIFEST_BYTES: u64 = 32 * 1024 * 1024; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PutAttributes { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) etag_override: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) completion_upload_id: Option, + pub(crate) cache_control: Option, + pub(crate) content_disposition: Option, + pub(crate) content_encoding: Option, + pub(crate) content_language: Option, + pub(crate) content_type: Option, + pub(crate) expires: Option, + pub(crate) metadata: BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ObjectAttributes { + pub(crate) blob_oid: String, + pub(crate) etag: String, + pub(crate) size: u64, + pub(crate) modified_seconds: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) completion_upload_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) cache_control: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) content_disposition: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) content_encoding: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) content_language: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) content_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) expires: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) metadata: BTreeMap, +} + +impl ObjectAttributes { + pub(crate) fn new( + blob_oid: ObjectId, + etag: String, + size: u64, + modified_seconds: u64, + pending: PutAttributes, + ) -> Self { + Self { + blob_oid: blob_oid.to_string(), + etag, + size, + modified_seconds, + completion_upload_id: pending.completion_upload_id, + cache_control: pending.cache_control, + content_disposition: pending.content_disposition, + content_encoding: pending.content_encoding, + content_language: pending.content_language, + content_type: pending.content_type, + expires: pending.expires, + metadata: pending.metadata, + } + } + + pub(crate) fn matches_pending(&self, pending: &PutAttributes, etag: &str, size: usize) -> bool { + self.etag == etag + && usize::try_from(self.size).ok() == Some(size) + && self.completion_upload_id == pending.completion_upload_id + && self.cache_control == pending.cache_control + && self.content_disposition == pending.content_disposition + && self.content_encoding == pending.content_encoding + && self.content_language == pending.content_language + && self.content_type == pending.content_type + && self.expires == pending.expires + && self.metadata == pending.metadata + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Manifest { + version: u32, + objects: BTreeMap, +} + +impl Default for Manifest { + fn default() -> Self { + Self { + version: VERSION, + objects: BTreeMap::new(), + } + } +} + +impl Manifest { + pub(crate) fn object(&self, path: &str, oid: ObjectId) -> Option<&ObjectAttributes> { + self.objects + .get(path) + .filter(|attributes| attributes.blob_oid == oid.to_string()) + } + + pub(crate) fn put(&mut self, path: String, attributes: ObjectAttributes) { + self.objects.insert(path, attributes); + } + + pub(crate) fn remove(&mut self, path: &str) { + self.objects.remove(path); + } +} + +pub(crate) async fn load(repository: &Repository, commit: ObjectId) -> crate::Result { + let path = repository + .layout + .repo_path(&format!("s3/attributes/{commit}.json")); + let bytes = match repository + .store + .get_with_etag_bounded(&path, MAX_MANIFEST_BYTES) + .await + { + Ok((bytes, _)) => bytes, + Err(crab_storage::StorageError::NotFound { .. }) => return Ok(Manifest::default()), + Err(error) => return Err(error.into()), + }; + let manifest: Manifest = + serde_json::from_slice(&bytes).map_err(|source| crate::Error::Attributes { source })?; + if manifest.version != VERSION { + return Err(crate::Error::Config( + "unsupported S3 attribute manifest version", + )); + } + Ok(manifest) +} + +pub(crate) async fn save( + repository: &Repository, + commit: ObjectId, + manifest: &Manifest, +) -> crate::Result<()> { + let path = repository + .layout + .repo_path(&format!("s3/attributes/{commit}.json")); + let bytes = + serde_json::to_vec(manifest).map_err(|source| crate::Error::Attributes { source })?; + if bytes.len() as u64 > MAX_MANIFEST_BYTES { + return Err(crate::Error::Config("S3 attribute manifest exceeds 32 MiB")); + } + repository + .store + .put_exact(&path, Bytes::from(bytes)) + .await?; + Ok(()) +} diff --git a/crates/crab-s3-gateway/src/auth.rs b/crates/crab-s3-gateway/src/auth.rs new file mode 100644 index 000000000..0dfe9ec04 --- /dev/null +++ b/crates/crab-s3-gateway/src/auth.rs @@ -0,0 +1,64 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use s3s::{ + S3Result, + auth::{S3Auth, SecretKey}, +}; + +use crate::{Config, CredentialConfig, Error, Result}; + +#[derive(Clone)] +pub(crate) struct GatewayAuth { + keys: Arc>, +} + +struct Credential { + secret: SecretKey, + principal: String, +} + +impl GatewayAuth { + pub(crate) fn load(config: &Config) -> Result { + let keys = config + .credentials + .iter() + .map(load_credential) + .collect::>>()?; + Ok(Self { + keys: Arc::new(keys), + }) + } + + pub(crate) fn principal(&self, access_key: &str) -> Option<&str> { + self.keys + .get(access_key) + .map(|credential| credential.principal.as_str()) + } +} + +fn load_credential(config: &CredentialConfig) -> Result<(String, Credential)> { + let value = std::fs::read_to_string(&config.secret_key_file)?; + let secret = value.trim_end_matches(['\r', '\n']); + if secret.len() < 16 || secret.len() > 256 || secret.chars().any(char::is_whitespace) { + return Err(Error::Config( + "credential secret files must contain one 16-256 character value", + )); + } + Ok(( + config.access_key.clone(), + Credential { + secret: SecretKey::from(secret.to_owned()), + principal: config.principal.clone(), + }, + )) +} + +#[async_trait::async_trait] +impl S3Auth for GatewayAuth { + async fn get_secret_key(&self, access_key: &str) -> S3Result { + self.keys + .get(access_key) + .map(|credential| credential.secret.clone()) + .ok_or_else(|| s3s::s3_error!(InvalidAccessKeyId)) + } +} diff --git a/crates/crab-s3-gateway/src/config.rs b/crates/crab-s3-gateway/src/config.rs new file mode 100644 index 000000000..eb9683181 --- /dev/null +++ b/crates/crab-s3-gateway/src/config.rs @@ -0,0 +1,230 @@ +use std::{ + collections::HashSet, + net::SocketAddr, + path::{Path, PathBuf}, +}; + +use crab_storage::StorageProviderKind; +use serde::Deserialize; + +use crate::{Error, Result}; + +/// Listener, credentials, and logical repository catalog. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Config { + pub listen: SocketAddr, + #[serde(default = "default_region")] + pub region: String, + pub credentials: Vec, + pub repositories: Vec, +} + +/// One Crab-issued S3 access key mapped to a logical principal. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CredentialConfig { + pub access_key: String, + pub secret_key_file: PathBuf, + pub principal: String, +} + +/// One logical S3 bucket backed by a Crab repository placement. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepositoryConfig { + pub name: String, + #[serde(default = "default_provider")] + pub provider: StorageProviderKind, + pub bucket: String, + pub prefix: String, + #[serde(default = "default_branch")] + pub default_branch: String, + #[serde(default)] + pub members: Vec, + #[serde(default)] + pub protected_branches: Vec, +} + +/// One principal's repository permission. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepositoryMember { + pub principal: String, + pub access: RepositoryAccess, +} + +/// Repository permission ordered from reads through administration. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum RepositoryAccess { + Read, + Write, + Admin, +} + +impl Config { + /// Read and validate configuration without exposing secret values. + pub fn read(path: &Path) -> Result { + let config: Self = toml::from_str(&std::fs::read_to_string(path)?)?; + config.validate()?; + Ok(config) + } + + pub(crate) fn validate(&self) -> Result<()> { + if self.repositories.is_empty() || self.credentials.is_empty() { + return Err(Error::Config( + "configure at least one repository and one credential", + )); + } + if self.region.is_empty() + || self.region.len() > 63 + || !self + .region + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + { + return Err(Error::Config("region must be a non-empty AWS region name")); + } + let mut access_keys = HashSet::new(); + let mut configured_principals = HashSet::new(); + for credential in &self.credentials { + if credential.access_key.len() < 3 + || credential.access_key.len() > 128 + || credential.access_key.chars().any(char::is_whitespace) + || credential.principal.trim() != credential.principal + || credential.principal.is_empty() + || !access_keys.insert(&credential.access_key) + { + return Err(Error::Config( + "credentials require unique access keys and non-empty principals", + )); + } + let metadata = std::fs::metadata(&credential.secret_key_file)?; + if !metadata.is_file() { + return Err(Error::Config("credential secret path must be a file")); + } + validate_secret_permissions(&metadata)?; + configured_principals.insert(credential.principal.as_str()); + } + let mut names = HashSet::new(); + let mut placements = HashSet::new(); + for repository in &self.repositories { + if repository.provider == StorageProviderKind::Local { + return Err(Error::Config( + "repository providers must be s3, gcs, or azure", + )); + } + if !valid_bucket_name(&repository.name) + || !names.insert(repository.name.to_ascii_lowercase()) + { + return Err(Error::Config( + "repository names must be unique valid lowercase S3 bucket names", + )); + } + if repository.bucket.is_empty() + || repository.prefix.is_empty() + || !placements.insert((repository.provider, &repository.bucket, &repository.prefix)) + { + return Err(Error::Config( + "repository bucket/prefix placements must be present and unique", + )); + } + let branch = format!("refs/heads/{}", repository.default_branch); + if repository.default_branch.starts_with("refs/") + || crab_git::validate_push_refname(&branch).is_err() + { + return Err(Error::Config("default_branch must be a valid short branch")); + } + let mut principals = HashSet::new(); + if repository.members.iter().any(|member| { + member.principal.trim() != member.principal + || member.principal.is_empty() + || !configured_principals.contains(member.principal.as_str()) + || !principals.insert(&member.principal) + }) { + return Err(Error::Config( + "repository members require unique configured principals", + )); + } + let mut protected = HashSet::new(); + if repository.protected_branches.iter().any(|name| { + name.starts_with("refs/") + || crab_git::validate_push_refname(&format!("refs/heads/{name}")).is_err() + || !protected.insert(name) + }) { + return Err(Error::Config( + "protected branches must be unique valid short branch names", + )); + } + } + Ok(()) + } +} + +#[cfg(unix)] +fn validate_secret_permissions(metadata: &std::fs::Metadata) -> Result<()> { + use std::os::unix::fs::PermissionsExt as _; + + if metadata.permissions().mode() & 0o077 != 0 { + return Err(Error::Config( + "credential secret files must not be accessible by group or other users", + )); + } + Ok(()) +} + +#[cfg(not(unix))] +fn validate_secret_permissions(_metadata: &std::fs::Metadata) -> Result<()> { + Ok(()) +} + +fn valid_bucket_name(value: &str) -> bool { + let bytes = value.as_bytes(); + (3..=63).contains(&bytes.len()) + && bytes.first().is_some_and(u8::is_ascii_alphanumeric) + && bytes.last().is_some_and(u8::is_ascii_alphanumeric) + && bytes.iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'.' | b'-') + }) + && !value.contains("..") +} + +fn default_branch() -> String { + "main".to_owned() +} + +fn default_region() -> String { + "us-east-1".to_owned() +} + +fn default_provider() -> StorageProviderKind { + StorageProviderKind::S3 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn logical_bucket_names_follow_the_frozen_profile() { + for value in ["abc", "a-b.c9", &format!("a{}z", "b".repeat(61))] { + assert!(valid_bucket_name(value)); + } + for value in ["ab", "UPPER", "-abc", "abc-", "a..b", "a_b"] { + assert!(!valid_bucket_name(value)); + } + } + + #[cfg(unix)] + #[test] + fn credential_secret_rejects_group_or_other_access() { + use std::os::unix::fs::PermissionsExt as _; + + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o640)).unwrap(); + assert!(validate_secret_permissions(&std::fs::metadata(file.path()).unwrap()).is_err()); + std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o600)).unwrap(); + validate_secret_permissions(&std::fs::metadata(file.path()).unwrap()).unwrap(); + } +} diff --git a/crates/crab-s3-gateway/src/gateway.rs b/crates/crab-s3-gateway/src/gateway.rs new file mode 100644 index 000000000..ff8df6912 --- /dev/null +++ b/crates/crab-s3-gateway/src/gateway.rs @@ -0,0 +1,2248 @@ +use std::{ + collections::BTreeMap, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use base64::Engine as _; +use bytes::{Bytes, BytesMut}; +use crab_cache_store::{CacheConfig, CachingStore}; +use crab_git::pointer_detect::PointerKind; +use crab_remote_git::{ + ContentClassification, EntryKind, OperationKind, RemoteGitRepository, RemoteGitRuntime, + RepositoryIdentity, RepositoryOptions, Revision, +}; +use crab_storage::{StorageProviderKind, Store, StoreLayout, build_static_env_store}; +use s3s::{S3, S3Request, S3Response, S3Result, dto::*, s3_error}; +use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; + +use crate::{Config, RepositoryAccess, RepositoryConfig, auth::GatewayAuth, mutation, namespace}; + +const MAX_SINGLE_OBJECT_BYTES: usize = 256 * 1024 * 1024; + +pub(crate) struct Repository { + pub(crate) config: RepositoryConfig, + pub(crate) store: Store, + pub(crate) layout: StoreLayout, + pub(crate) identity: RepositoryIdentity, + hydrator: crab_read::ShardHydrator, + lfs: crab_lfs::LfsObjectStore, +} + +impl Repository { + pub(crate) fn new(config: RepositoryConfig, store: Store) -> crate::Result { + let layout = StoreLayout::new(store.clone(), config.prefix.clone()); + let caching = CachingStore::new(store.clone(), CacheConfig::default())?; + let read_layout = crab_read::ReadStoreLayout::with_global_prefix( + store.clone(), + layout.repo_prefix().to_owned(), + layout.global_prefix().to_owned(), + ); + Ok(Self { + hydrator: crab_read::ReadRuntimeBuilder::new(caching, read_layout, 16).build()?, + lfs: crab_lfs::LfsObjectStore::new(store.clone(), layout.repo_prefix()), + identity: RepositoryIdentity::new( + format!("{}:{}", provider_name(config.provider), config.bucket), + config.prefix.clone(), + 1, + )?, + config, + store, + layout, + }) + } + + fn access(&self, principal: &str) -> Option { + self.config + .members + .iter() + .find(|member| member.principal == principal) + .map(|member| member.access) + } +} + +#[derive(Clone)] +pub(crate) struct Gateway { + repositories: Arc>, + runtime: Arc, + options: RepositoryOptions, + auth: GatewayAuth, + region: Arc, + admission: Arc, + cancellation: CancellationToken, +} + +struct ReadObject { + bytes: Bytes, + size: u64, + etag: String, + modified: Timestamp, + attributes: Option, +} + +impl Gateway { + pub(crate) fn new(config: Config, cancellation: CancellationToken) -> crate::Result { + let auth = GatewayAuth::load(&config)?; + let region = Arc::from(config.region.clone()); + let mut stores: BTreeMap = BTreeMap::new(); + let mut repositories = BTreeMap::new(); + for entry in config.repositories { + let store_key = format!("{}:{}", provider_name(entry.provider), entry.bucket); + let store = match stores.get(&store_key) { + Some(store) => store.clone(), + None => { + let store = build_store(&entry)?; + stores.insert(store_key, store.clone()); + store + } + }; + let repository = Repository::new(entry.clone(), store)?; + repositories.insert(entry.name.clone(), repository); + } + Ok(Self { + repositories: Arc::new(repositories), + runtime: Arc::new(RemoteGitRuntime::default()), + options: RepositoryOptions::default(), + auth, + region, + admission: Arc::new(Semaphore::new(32)), + cancellation, + }) + } + + pub(crate) async fn initialize_repositories(&self) -> crate::Result<()> { + for repository in self.repositories.values() { + let head = format!("refs/heads/{}", repository.config.default_branch); + crab_write::initialize::initialize_repository( + &repository.store, + &repository.layout, + &head, + ) + .await?; + } + Ok(()) + } + + pub(crate) fn auth(&self) -> GatewayAuth { + self.auth.clone() + } + + pub(crate) async fn shutdown(&self) { + self.cancellation.cancel(); + self.runtime.shutdown().await; + } + + fn principal<'a, T>(&'a self, req: &S3Request) -> S3Result<&'a str> { + if req + .region + .as_ref() + .is_some_and(|region| region.as_str() != self.region.as_ref()) + { + return Err(s3_error!( + AuthorizationHeaderMalformed, + "The authorization region does not match this endpoint" + )); + } + if req + .service + .as_deref() + .is_some_and(|service| service != "s3") + { + return Err(s3_error!(SignatureDoesNotMatch)); + } + let access_key = req + .credentials + .as_ref() + .map(|credentials| credentials.access_key.as_str()) + .ok_or_else(|| s3_error!(AccessDenied, "Signature is required"))?; + self.auth + .principal(access_key) + .ok_or_else(|| s3_error!(AccessDenied)) + } + + fn repository<'a, T>( + &'a self, + req: &S3Request, + bucket: &str, + required: RepositoryAccess, + ) -> S3Result<&'a Repository> { + let principal = self.principal(req)?; + let repository = self + .repositories + .get(bucket) + .ok_or_else(|| s3_error!(NoSuchBucket))?; + if repository + .access(principal) + .is_some_and(|access| access >= required) + { + Ok(repository) + } else { + Err(s3_error!(AccessDenied)) + } + } + + async fn open(&self, repository: &Repository) -> S3Result { + crate::repository::open_current( + repository, + Arc::clone(&self.runtime), + self.options, + &self.cancellation, + ) + .await + .map_err(gateway_error) + } + + async fn read_object(&self, repository: &Repository, key: &str) -> S3Result { + let address = namespace::object_address(key).map_err(namespace_error)?; + let repo = self.open(repository).await?; + let operation = repo + .operation(OperationKind::Repository, &self.cancellation) + .await + .map_err(remote_error)?; + let result = async { + let snapshot = repo + .snapshot( + &Revision::parse(&address.reference).map_err(remote_error)?, + &operation, + ) + .await + .map_err(remote_error)?; + let commit = snapshot.commit(&operation).await.map_err(remote_error)?; + let blob = snapshot + .read_blob(&address.path, &operation) + .await + .map_err(remote_error)?; + if blob.metadata.kind != EntryKind::Blob { + return Err(s3_error!(InvalidObjectState)); + } + let manifest = crate::attributes::load(repository, snapshot.commit_oid()) + .await + .map_err(gateway_error)?; + let path = std::str::from_utf8(address.path.as_bytes()) + .map_err(|_| s3_error!(InvalidObjectState))?; + let attributes = manifest.object(path, blob.metadata.oid).cloned(); + let modified_seconds = attributes + .as_ref() + .and_then(|value| i64::try_from(value.modified_seconds).ok()) + .unwrap_or(commit.committer.seconds); + let modified = timestamp(modified_seconds)?; + let bytes = materialize_blob(repository, blob).await?; + let size = u64::try_from(bytes.len()).map_err(|_| s3_error!(InternalError))?; + let etag = attributes + .as_ref() + .map(|value| value.etag.clone()) + .unwrap_or_else(|| md5_hex(&bytes)); + Ok(ReadObject { + bytes, + size, + etag, + modified, + attributes, + }) + } + .await; + finish(operation, result).await + } + + fn writable_address( + &self, + req: &S3Request, + bucket: &str, + key: &str, + ) -> S3Result<(&Repository, namespace::ObjectAddress, String)> { + let principal = self.principal(req)?.to_owned(); + let repository = self.repository(req, bucket, RepositoryAccess::Write)?; + let address = namespace::object_address(key).map_err(namespace_error)?; + let branch = address + .branch + .as_deref() + .and_then(|name| name.strip_prefix("refs/heads/")) + .ok_or_else(|| s3_error!(MethodNotAllowed, "Writes require a branch key"))?; + if repository + .config + .protected_branches + .iter() + .any(|protected| protected == branch) + { + return Err(s3_error!( + AccessDenied, + "The destination branch is protected" + )); + } + Ok((repository, address, principal)) + } +} + +fn build_store(entry: &RepositoryConfig) -> crate::Result { + Ok(build_static_env_store(&entry.bucket, entry.provider)?) +} + +async fn materialize_blob(repository: &Repository, blob: crab_remote_git::Blob) -> S3Result { + let physical_size = u64::try_from(blob.bytes.len()).map_err(|_| s3_error!(EntityTooLarge))?; + let logical_size = blob.metadata.logical_size.unwrap_or(physical_size); + if logical_size > MAX_SINGLE_OBJECT_BYTES as u64 { + return Err(s3_error!(EntityTooLarge)); + } + match blob.metadata.classification { + ContentClassification::OrdinaryGit => Ok(blob.bytes), + ContentClassification::CrabPointer => Ok(Bytes::from( + repository + .hydrator + .reconstruct_from_pointer(&blob.bytes) + .await + .map_err(|error| gateway_error(error.into()))?, + )), + ContentClassification::LfsPointer => { + let PointerKind::Lfs(pointer) = crab_git::classify(&blob.bytes) else { + return Err(s3_error!(InvalidObjectState)); + }; + let bytes = repository + .lfs + .verify(&pointer.oid) + .await + .map_err(|error| gateway_error(error.into()))?; + if bytes.len() as u64 != pointer.size { + return Err(s3_error!(InvalidObjectState)); + } + Ok(bytes) + } + } +} + +fn provider_name(provider: StorageProviderKind) -> &'static str { + match provider { + StorageProviderKind::S3 => "s3", + StorageProviderKind::Gcs => "gcs", + StorageProviderKind::Azure => "azure", + StorageProviderKind::Local => "local", + } +} + +#[async_trait::async_trait] +impl S3 for Gateway { + async fn list_buckets( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + if req + .input + .bucket_region + .as_deref() + .is_some_and(|region| region != self.region.as_ref()) + { + return Err(s3_error!(InvalidRequest)); + } + let principal = self.principal(&req)?; + let mut buckets = self + .repositories + .values() + .filter(|repository| repository.access(principal).is_some()) + .filter(|repository| { + req.input + .prefix + .as_deref() + .is_none_or(|prefix| repository.config.name.starts_with(prefix)) + }) + .filter(|repository| { + req.input + .continuation_token + .as_deref() + .is_none_or(|token| repository.config.name.as_str() > token) + }) + .map(|repository| Bucket { + name: Some(repository.config.name.clone()), + ..Default::default() + }) + .collect::>(); + let limit = match req.input.max_buckets { + Some(value) if (1..=10_000).contains(&value) => { + usize::try_from(value).map_err(|_| s3_error!(InvalidArgument))? + } + Some(_) => return Err(s3_error!(InvalidArgument)), + None => usize::MAX, + }; + let truncated = buckets.len() > limit; + buckets.truncate(limit); + let continuation_token = truncated + .then(|| buckets.last().and_then(|bucket| bucket.name.clone())) + .flatten(); + Ok(S3Response::new(ListBucketsOutput { + buckets: Some(buckets), + continuation_token, + prefix: req.input.prefix, + ..Default::default() + })) + } + + async fn head_bucket( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + self.repository(&req, &req.input.bucket, RepositoryAccess::Read)?; + let mut response = S3Response::new(HeadBucketOutput::default()); + response.headers.insert( + "x-amz-bucket-region", + http::HeaderValue::from_str(&self.region).map_err(|_| s3_error!(InternalError))?, + ); + Ok(response) + } + + async fn get_object( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_get_extensions(&req.input)?; + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Read)?; + let object = self.read_object(repository, &req.input.key).await?; + evaluate_conditions( + req.input.if_match.as_ref(), + req.input.if_none_match.as_ref(), + req.input.if_modified_since.as_ref(), + req.input.if_unmodified_since.as_ref(), + &object.etag, + &object.modified, + )?; + let checked = req + .input + .range + .as_ref() + .map(|range| range.check(object.size)) + .transpose()?; + let (bytes, content_range) = match checked { + Some(range) => { + let start = usize::try_from(range.start).map_err(|_| s3_error!(InvalidRange))?; + let end = usize::try_from(range.end).map_err(|_| s3_error!(InvalidRange))?; + ( + object.bytes.slice(start..end), + Some(format!( + "bytes {}-{}/{}", + range.start, + range.end - 1, + object.size + )), + ) + } + None => (object.bytes, None), + }; + let content_length = i64::try_from(bytes.len()).map_err(|_| s3_error!(InternalError))?; + let output = GetObjectOutput { + accept_ranges: Some("bytes".to_owned()), + body: Some(StreamingBlob::from(s3s::Body::from(bytes))), + content_length: Some(content_length), + content_range, + content_type: req + .input + .response_content_type + .or_else(|| { + object + .attributes + .as_ref() + .and_then(|value| value.content_type.clone()) + }) + .or_else(|| Some("application/octet-stream".to_owned())), + cache_control: req.input.response_cache_control.or_else(|| { + object + .attributes + .as_ref() + .and_then(|value| value.cache_control.clone()) + }), + content_disposition: req.input.response_content_disposition.or_else(|| { + object + .attributes + .as_ref() + .and_then(|value| value.content_disposition.clone()) + }), + content_encoding: req.input.response_content_encoding.or_else(|| { + object + .attributes + .as_ref() + .and_then(|value| value.content_encoding.clone()) + }), + content_language: req.input.response_content_language.or_else(|| { + object + .attributes + .as_ref() + .and_then(|value| value.content_language.clone()) + }), + expires: req.input.response_expires.or_else(|| { + object + .attributes + .as_ref() + .and_then(|value| value.expires.clone()) + }), + e_tag: Some(ETag::Strong(object.etag)), + last_modified: Some(object.modified), + metadata: object + .attributes + .map(|value| value.metadata.into_iter().collect()), + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + async fn head_object( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_head_extensions(&req.input)?; + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Read)?; + let object = self.read_object(repository, &req.input.key).await?; + evaluate_conditions( + req.input.if_match.as_ref(), + req.input.if_none_match.as_ref(), + req.input.if_modified_since.as_ref(), + req.input.if_unmodified_since.as_ref(), + &object.etag, + &object.modified, + )?; + let checked = req + .input + .range + .as_ref() + .map(|range| range.check(object.size)) + .transpose()?; + let (content_length, content_range) = match checked { + Some(range) => ( + range.end - range.start, + Some(format!( + "bytes {}-{}/{}", + range.start, + range.end - 1, + object.size + )), + ), + None => (object.size, None), + }; + Ok(S3Response::new(HeadObjectOutput { + accept_ranges: Some("bytes".to_owned()), + content_length: Some( + i64::try_from(content_length).map_err(|_| s3_error!(InternalError))?, + ), + content_range, + cache_control: object + .attributes + .as_ref() + .and_then(|value| value.cache_control.clone()), + content_disposition: object + .attributes + .as_ref() + .and_then(|value| value.content_disposition.clone()), + content_encoding: object + .attributes + .as_ref() + .and_then(|value| value.content_encoding.clone()), + content_language: object + .attributes + .as_ref() + .and_then(|value| value.content_language.clone()), + content_type: object + .attributes + .as_ref() + .and_then(|value| value.content_type.clone()) + .or_else(|| Some("application/octet-stream".to_owned())), + e_tag: Some(ETag::Strong(object.etag)), + expires: object + .attributes + .as_ref() + .and_then(|value| value.expires.clone()), + last_modified: Some(object.modified), + metadata: object + .attributes + .map(|value| value.metadata.into_iter().collect()), + ..Default::default() + })) + } + + async fn put_object( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_put_extensions(&req.input)?; + let (repository, address, principal) = + self.writable_address(&req, &req.input.bucket, &req.input.key)?; + let content_length = req.input.content_length; + let content_md5 = req.input.content_md5.clone(); + let mut checksums = RequestChecksums::from(&req.input); + let trailing_headers = req.trailing_headers.clone(); + let body = read_body(req.input.body, content_length).await?; + checksums.merge_trailers(trailing_headers.as_ref())?; + verify_content_md5(&body, content_md5.as_deref())?; + checksums.verify(&body)?; + let outcome = mutation::apply( + repository, + Arc::clone(&self.runtime), + self.options, + address + .branch + .as_deref() + .ok_or_else(|| s3_error!(MethodNotAllowed))?, + &address.path, + mutation::Change::Put { + bytes: body, + attributes: Box::new(crate::attributes::PutAttributes { + etag_override: None, + completion_upload_id: None, + cache_control: req.input.cache_control, + content_disposition: req.input.content_disposition, + content_encoding: req.input.content_encoding, + content_language: req.input.content_language, + content_type: req.input.content_type, + expires: req.input.expires, + metadata: req.input.metadata.unwrap_or_default().into_iter().collect(), + }), + }, + &principal, + &self.cancellation, + ) + .await + .map_err(mutation_error)?; + Ok(S3Response::new(PutObjectOutput { + e_tag: outcome.etag.map(ETag::Strong), + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_crc64nvme: checksums.crc64nvme, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_type: checksums + .algorithm + .as_ref() + .map(|_| ChecksumType::from_static(ChecksumType::FULL_OBJECT)), + ..Default::default() + })) + } + + async fn delete_object( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_delete_extensions(&req.input)?; + let (repository, address, principal) = + self.writable_address(&req, &req.input.bucket, &req.input.key)?; + mutation::apply( + repository, + Arc::clone(&self.runtime), + self.options, + address + .branch + .as_deref() + .ok_or_else(|| s3_error!(MethodNotAllowed))?, + &address.path, + mutation::Change::Delete, + &principal, + &self.cancellation, + ) + .await + .map_err(mutation_error)?; + Ok(S3Response::new(DeleteObjectOutput::default())) + } + + async fn delete_objects( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + if req.input.bypass_governance_retention.is_some() + || req.input.expected_bucket_owner.is_some() + || req.input.mfa.is_some() + || req.input.request_payer.is_some() + { + return Err(s3_error!(NotImplemented)); + } + if req.input.delete.objects.len() > 1000 { + return Err(s3_error!( + MalformedXML, + "DeleteObjects accepts at most 1000 keys" + )); + } + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; + let principal = self.principal(&req)?.to_owned(); + let quiet = req.input.delete.quiet.unwrap_or(false); + let mut deleted = Vec::new(); + let mut errors = Vec::new(); + for object in req.input.delete.objects { + if object.version_id.is_some() + || object.e_tag.is_some() + || object.last_modified_time.is_some() + || object.size.is_some() + { + errors.push(s3s::dto::Error { + key: Some(object.key), + code: Some("NotImplemented".to_owned()), + message: Some( + "Per-object version and condition fields are unsupported".to_owned(), + ), + ..Default::default() + }); + continue; + } + let key = object.key; + let result = async { + let address = namespace::object_address(&key).map_err(namespace_error)?; + let branch = writable_branch(repository, &address)?; + mutation::apply( + repository, + Arc::clone(&self.runtime), + self.options, + branch, + &address.path, + mutation::Change::Delete, + &principal, + &self.cancellation, + ) + .await + .map_err(mutation_error) + } + .await; + match result { + Ok(_) if !quiet => deleted.push(DeletedObject { + key: Some(key), + ..Default::default() + }), + Ok(_) => {} + Err(error) => errors.push(s3s::dto::Error { + key: Some(key), + code: Some(error.code().as_str().to_owned()), + message: Some( + error + .message() + .unwrap_or("Object deletion failed") + .to_owned(), + ), + ..Default::default() + }), + } + } + Ok(S3Response::new(DeleteObjectsOutput { + deleted: (!deleted.is_empty()).then_some(deleted), + errors: (!errors.is_empty()).then_some(errors), + ..Default::default() + })) + } + + async fn copy_object( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_copy_extensions(&req.input)?; + let (source_bucket, source_key) = match &req.input.copy_source { + CopySource::Bucket { + bucket, + key, + version_id: None, + } => (bucket.to_string(), key.to_string()), + CopySource::Bucket { + version_id: Some(_), + .. + } => { + return Err(s3_error!( + NotImplemented, + "Copying an object version is unsupported" + )); + } + CopySource::AccessPoint { .. } | CopySource::Outpost { .. } => { + return Err(s3_error!( + NotImplemented, + "Copy access points are unsupported" + )); + } + }; + let source_repository = self.repository(&req, &source_bucket, RepositoryAccess::Read)?; + let source_object = self.read_object(source_repository, &source_key).await?; + evaluate_conditions( + req.input.copy_source_if_match.as_ref(), + req.input.copy_source_if_none_match.as_ref(), + req.input.copy_source_if_modified_since.as_ref(), + req.input.copy_source_if_unmodified_since.as_ref(), + &source_object.etag, + &source_object.modified, + )?; + let (repository, address, principal) = + self.writable_address(&req, &req.input.bucket, &req.input.key)?; + let attributes = if req + .input + .metadata_directive + .as_ref() + .is_some_and(|value| value.as_str() == MetadataDirective::REPLACE) + { + crate::attributes::PutAttributes { + etag_override: None, + completion_upload_id: None, + cache_control: req.input.cache_control, + content_disposition: req.input.content_disposition, + content_encoding: req.input.content_encoding, + content_language: req.input.content_language, + content_type: req.input.content_type, + expires: req.input.expires, + metadata: req.input.metadata.unwrap_or_default().into_iter().collect(), + } + } else { + source_object + .attributes + .as_ref() + .map(stored_to_pending) + .unwrap_or_default() + }; + let outcome = mutation::apply( + repository, + Arc::clone(&self.runtime), + self.options, + address + .branch + .as_deref() + .ok_or_else(|| s3_error!(MethodNotAllowed))?, + &address.path, + mutation::Change::Put { + bytes: source_object.bytes, + attributes: Box::new(attributes), + }, + &principal, + &self.cancellation, + ) + .await + .map_err(mutation_error)?; + Ok(S3Response::new(CopyObjectOutput { + copy_object_result: Some(CopyObjectResult { + e_tag: outcome.etag.map(ETag::Strong), + last_modified: Some(timestamp( + i64::try_from( + std::time::SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| s3_error!(InternalError))? + .as_secs(), + ) + .map_err(|_| s3_error!(InternalError))?, + )?), + ..Default::default() + }), + ..Default::default() + })) + } + + async fn create_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_create_multipart_extensions(&req.input)?; + let (repository, address, principal) = + self.writable_address(&req, &req.input.bucket, &req.input.key)?; + let branch = address + .branch + .as_deref() + .ok_or_else(|| s3_error!(MethodNotAllowed))?; + let path = + std::str::from_utf8(address.path.as_bytes()).map_err(|_| s3_error!(InvalidArgument))?; + let session = crate::multipart::create( + repository, + &req.input.bucket, + &req.input.key, + branch, + path, + &principal, + crate::attributes::PutAttributes { + etag_override: None, + completion_upload_id: None, + cache_control: req.input.cache_control, + content_disposition: req.input.content_disposition, + content_encoding: req.input.content_encoding, + content_language: req.input.content_language, + content_type: req.input.content_type, + expires: req.input.expires, + metadata: req.input.metadata.unwrap_or_default().into_iter().collect(), + }, + now_seconds()?, + ) + .await + .map_err(multipart_error)?; + Ok(S3Response::new(CreateMultipartUploadOutput { + bucket: Some(req.input.bucket), + key: Some(req.input.key), + upload_id: Some(session.id), + ..Default::default() + })) + } + + async fn upload_part( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_upload_part_extensions(&req.input)?; + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; + let principal = self.principal(&req)?.to_owned(); + let loaded = crate::multipart::load(repository, &req.input.upload_id) + .await + .map_err(multipart_error)?; + crate::multipart::authorize( + &loaded.session, + &req.input.bucket, + &req.input.key, + &principal, + ) + .map_err(multipart_error)?; + let content_length = req.input.content_length; + let content_md5 = req.input.content_md5.clone(); + let body = read_body(req.input.body, content_length).await?; + verify_content_md5(&body, content_md5.as_deref())?; + let etag = md5_hex(&body); + crate::multipart::register_part( + repository, + loaded, + req.input.part_number, + body, + etag.clone(), + now_seconds()?, + ) + .await + .map_err(multipart_error)?; + Ok(S3Response::new(UploadPartOutput { + e_tag: Some(ETag::Strong(etag)), + ..Default::default() + })) + } + + async fn upload_part_copy( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + if req.input.copy_source_sse_customer_algorithm.is_some() + || req.input.copy_source_sse_customer_key.is_some() + || req.input.copy_source_sse_customer_key_md5.is_some() + || req.input.expected_bucket_owner.is_some() + || req.input.expected_source_bucket_owner.is_some() + || req.input.request_payer.is_some() + || req.input.sse_customer_algorithm.is_some() + || req.input.sse_customer_key.is_some() + || req.input.sse_customer_key_md5.is_some() + { + return Err(s3_error!(NotImplemented)); + } + let (source_bucket, source_key) = copy_source_bucket_key(&req.input.copy_source)?; + let source_repository = self.repository(&req, &source_bucket, RepositoryAccess::Read)?; + let source = self.read_object(source_repository, &source_key).await?; + evaluate_conditions( + req.input.copy_source_if_match.as_ref(), + req.input.copy_source_if_none_match.as_ref(), + req.input.copy_source_if_modified_since.as_ref(), + req.input.copy_source_if_unmodified_since.as_ref(), + &source.etag, + &source.modified, + )?; + let bytes = match req.input.copy_source_range.as_deref() { + Some(value) => { + let range = Range::parse(value).map_err(|_| s3_error!(InvalidArgument))?; + let range = range.check(source.size)?; + let start = usize::try_from(range.start).map_err(|_| s3_error!(InvalidRange))?; + let end = usize::try_from(range.end).map_err(|_| s3_error!(InvalidRange))?; + source.bytes.slice(start..end) + } + None => source.bytes, + }; + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; + let principal = self.principal(&req)?.to_owned(); + let loaded = crate::multipart::load(repository, &req.input.upload_id) + .await + .map_err(multipart_error)?; + crate::multipart::authorize( + &loaded.session, + &req.input.bucket, + &req.input.key, + &principal, + ) + .map_err(multipart_error)?; + let etag = md5_hex(&bytes); + crate::multipart::register_part( + repository, + loaded, + req.input.part_number, + bytes, + etag.clone(), + now_seconds()?, + ) + .await + .map_err(multipart_error)?; + Ok(S3Response::new(UploadPartCopyOutput { + copy_part_result: Some(CopyPartResult { + e_tag: Some(ETag::Strong(etag)), + last_modified: Some(timestamp( + i64::try_from(now_seconds()?).map_err(|_| s3_error!(InternalError))?, + )?), + ..Default::default() + }), + ..Default::default() + })) + } + + async fn complete_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + reject_complete_multipart_extensions(&req.input)?; + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; + let principal = self.principal(&req)?.to_owned(); + let loaded = crate::multipart::load(repository, &req.input.upload_id) + .await + .map_err(multipart_error)?; + crate::multipart::authorize( + &loaded.session, + &req.input.bucket, + &req.input.key, + &principal, + ) + .map_err(multipart_error)?; + let selected = req + .input + .multipart_upload + .and_then(|upload| upload.parts) + .ok_or_else(|| s3_error!(InvalidPart))? + .into_iter() + .map(|part| { + let number = part.part_number.ok_or_else(|| s3_error!(InvalidPart))?; + let etag = match part.e_tag.ok_or_else(|| s3_error!(InvalidPart))? { + ETag::Strong(value) => value, + ETag::Weak(_) => return Err(s3_error!(InvalidPart)), + }; + Ok((number, etag)) + }) + .collect::>>()?; + if let Some(etag) = + crate::multipart::completed_etag(&loaded.session, &selected).map_err(multipart_error)? + { + return Ok(S3Response::new(CompleteMultipartUploadOutput { + bucket: Some(req.input.bucket), + key: Some(req.input.key), + e_tag: Some(ETag::Strong(etag.to_owned())), + ..Default::default() + })); + } + let (session, parts) = crate::multipart::freeze( + repository, + loaded, + &selected, + MAX_SINGLE_OBJECT_BYTES as u64, + ) + .await + .map_err(multipart_error)?; + let mut body = BytesMut::new(); + for part in &parts { + let bytes = crate::multipart::part_bytes(repository, part) + .await + .map_err(multipart_error)?; + body.extend_from_slice(&bytes); + } + let etag = multipart_etag(&parts)?; + let mut attributes = session.attributes.clone(); + attributes.etag_override = Some(etag.clone()); + attributes.completion_upload_id = Some(session.id.clone()); + let address = namespace::object_address(&session.key).map_err(namespace_error)?; + mutation::apply( + repository, + Arc::clone(&self.runtime), + self.options, + &session.branch, + &address.path, + mutation::Change::Put { + bytes: body.freeze(), + attributes: Box::new(attributes), + }, + &principal, + &self.cancellation, + ) + .await + .map_err(mutation_error)?; + let loaded = crate::multipart::load(repository, &session.id) + .await + .map_err(multipart_error)?; + crate::multipart::complete(repository, loaded, etag.clone()) + .await + .map_err(multipart_error)?; + Ok(S3Response::new(CompleteMultipartUploadOutput { + bucket: Some(req.input.bucket), + key: Some(req.input.key), + e_tag: Some(ETag::Strong(etag)), + ..Default::default() + })) + } + + async fn abort_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + if req.input.expected_bucket_owner.is_some() + || req.input.if_match_initiated_time.is_some() + || req.input.request_payer.is_some() + { + return Err(s3_error!(NotImplemented)); + } + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; + let principal = self.principal(&req)?.to_owned(); + let loaded = crate::multipart::load(repository, &req.input.upload_id) + .await + .map_err(multipart_error)?; + crate::multipart::authorize( + &loaded.session, + &req.input.bucket, + &req.input.key, + &principal, + ) + .map_err(multipart_error)?; + crate::multipart::abort(repository, loaded) + .await + .map_err(multipart_error)?; + Ok(S3Response::new(AbortMultipartUploadOutput::default())) + } + + async fn list_parts( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + if req.input.expected_bucket_owner.is_some() + || req.input.request_payer.is_some() + || req.input.sse_customer_algorithm.is_some() + || req.input.sse_customer_key.is_some() + || req.input.sse_customer_key_md5.is_some() + { + return Err(s3_error!(NotImplemented)); + } + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; + let principal = self.principal(&req)?; + let loaded = crate::multipart::load(repository, &req.input.upload_id) + .await + .map_err(multipart_error)?; + crate::multipart::authorize( + &loaded.session, + &req.input.bucket, + &req.input.key, + principal, + ) + .map_err(multipart_error)?; + let marker = req.input.part_number_marker.unwrap_or(0); + let max = req.input.max_parts.unwrap_or(1000); + if !(1..=1000).contains(&max) { + return Err(s3_error!(InvalidArgument)); + } + let mut parts = loaded + .session + .parts + .values() + .filter(|part| part.number > marker) + .cloned() + .collect::>(); + let truncated = parts.len() > max as usize; + parts.truncate(max as usize); + let next = truncated + .then(|| parts.last().map(|part| part.number)) + .flatten(); + Ok(S3Response::new(ListPartsOutput { + bucket: Some(req.input.bucket), + key: Some(req.input.key), + upload_id: Some(req.input.upload_id), + max_parts: Some(max), + part_number_marker: Some(marker), + next_part_number_marker: next, + is_truncated: Some(truncated), + parts: Some( + parts + .into_iter() + .map(|part| { + Ok(Part { + e_tag: Some(ETag::Strong(part.etag)), + last_modified: Some(timestamp( + i64::try_from(part.modified_seconds) + .map_err(|_| s3_error!(InternalError))?, + )?), + part_number: Some(part.number), + size: Some( + i64::try_from(part.size).map_err(|_| s3_error!(InternalError))?, + ), + ..Default::default() + }) + }) + .collect::>>()?, + ), + ..Default::default() + })) + } + + async fn list_multipart_uploads( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + if req + .input + .delimiter + .as_deref() + .is_some_and(|value| value != "/") + || req.input.expected_bucket_owner.is_some() + || req.input.request_payer.is_some() + { + return Err(s3_error!(NotImplemented)); + } + let url_encode = list_url_encoding(req.input.encoding_type.as_ref())?; + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; + let principal = self.principal(&req)?; + let prefix = req.input.prefix.as_deref().unwrap_or(""); + let sessions = crate::multipart::list(repository) + .await + .map_err(multipart_error)? + .into_iter() + .filter(|session| session.principal == principal && session.key.starts_with(prefix)) + .collect::>(); + let mut groups = std::collections::BTreeSet::new(); + let mut projected = Vec::new(); + for session in sessions { + if req.input.delimiter.as_deref() == Some("/") + && let Some(relative) = session.key.strip_prefix(prefix) + && let Some(position) = relative.find('/') + { + groups.insert(format!("{}{}", prefix, &relative[..=position])); + continue; + } + projected.push((session.key.clone(), session.id.clone(), Some(session), None)); + } + projected.extend(groups.into_iter().map(|prefix| { + ( + prefix.clone(), + String::new(), + None, + Some(CommonPrefix { + prefix: Some(prefix), + }), + ) + })); + projected.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + if let Some(key_marker) = req.input.key_marker.as_deref() { + projected.retain(|item| match req.input.upload_id_marker.as_deref() { + Some(upload_id_marker) if item.0 == key_marker => { + item.1.as_str() > upload_id_marker + } + _ => item.0.as_str() > key_marker, + }); + } + let max = req.input.max_uploads.unwrap_or(1000); + if !(1..=1000).contains(&max) { + return Err(s3_error!(InvalidArgument)); + } + let truncated = projected.len() > max as usize; + projected.truncate(max as usize); + let (next_key, next_id) = if truncated { + projected + .last() + .map(|item| { + ( + Some(item.0.clone()), + (!item.1.is_empty()).then(|| item.1.clone()), + ) + }) + .unwrap_or_default() + } else { + (None, None) + }; + let mut sessions = projected + .iter_mut() + .filter_map(|item| item.2.take()) + .collect::>(); + let mut common_prefixes = projected + .iter_mut() + .filter_map(|item| item.3.take()) + .collect::>(); + if url_encode { + for session in &mut sessions { + session.key = encode_list_value(&session.key, true); + } + for group in &mut common_prefixes { + group.prefix = encode_list_option(group.prefix.take(), true); + } + } + Ok(S3Response::new(ListMultipartUploadsOutput { + bucket: Some(req.input.bucket), + prefix: encode_list_option(req.input.prefix, url_encode), + delimiter: encode_list_option(req.input.delimiter, url_encode), + encoding_type: req.input.encoding_type, + key_marker: encode_list_option(req.input.key_marker, url_encode), + upload_id_marker: req.input.upload_id_marker, + max_uploads: Some(max), + is_truncated: Some(truncated), + next_key_marker: encode_list_option(next_key, url_encode), + next_upload_id_marker: next_id, + common_prefixes: (!common_prefixes.is_empty()).then_some(common_prefixes), + uploads: Some( + sessions + .into_iter() + .map(|session| { + Ok(MultipartUpload { + key: Some(session.key), + upload_id: Some(session.id), + initiated: Some(timestamp( + i64::try_from(session.created_seconds) + .map_err(|_| s3_error!(InternalError))?, + )?), + storage_class: Some(StorageClass::from_static("STANDARD")), + ..Default::default() + }) + }) + .collect::>>()?, + ), + ..Default::default() + })) + } + + async fn list_objects( + &self, + req: S3Request, + ) -> S3Result> { + let marker = req.input.marker.clone(); + let response = self.list_objects_v2(req.map_input(Into::into)).await?; + Ok(response.map_output(|output| { + let url_encode = output.encoding_type.is_some(); + ListObjectsOutput { + name: output.name, + prefix: output.prefix, + marker: encode_list_option(marker, url_encode), + max_keys: output.max_keys, + is_truncated: output.is_truncated, + contents: output.contents, + common_prefixes: output.common_prefixes, + delimiter: output.delimiter, + next_marker: encode_list_option(output.next_continuation_token, url_encode), + encoding_type: output.encoding_type, + ..Default::default() + } + })) + } + + async fn list_objects_v2( + &self, + req: S3Request, + ) -> S3Result> { + let _permit = self + .admission + .try_acquire() + .map_err(|_| s3_error!(SlowDown))?; + let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Read)?; + let url_encode = list_url_encoding(req.input.encoding_type.as_ref())?; + if req + .input + .delimiter + .as_deref() + .is_some_and(|value| value != "/") + || req.input.optional_object_attributes.is_some() + || req.input.request_payer.is_some() + { + return Err(s3_error!(NotImplemented)); + } + let prefix = req.input.prefix.as_deref().unwrap_or(""); + let Some((reference, _path_prefix)) = + namespace::listing_reference(prefix).map_err(namespace_error)? + else { + return self.list_refs(&req, repository).await; + }; + let repo = self.open(repository).await?; + let operation = repo + .operation(OperationKind::Repository, &self.cancellation) + .await + .map_err(remote_error)?; + let result = async { + let snapshot = repo + .snapshot( + &Revision::parse(&reference).map_err(remote_error)?, + &operation, + ) + .await + .map_err(remote_error)?; + let commit = snapshot.commit(&operation).await.map_err(remote_error)?; + let commit_modified = timestamp(commit.committer.seconds)?; + let attribute_manifest = crate::attributes::load(repository, snapshot.commit_oid()) + .await + .map_err(gateway_error)?; + let encoded_ref = prefix + .split_once('/') + .map(|(value, _)| value) + .unwrap_or_default(); + let mut keys = Vec::new(); + for entry in snapshot + .list_tree_recursive(&operation) + .await + .map_err(remote_error)? + { + if entry.kind != EntryKind::Blob { + continue; + } + let path = std::str::from_utf8(entry.path.as_bytes()) + .map_err(|_| s3_error!(InvalidObjectState))?; + let key = format!("{encoded_ref}/{path}"); + if !key.starts_with(prefix) { + continue; + } + let blob = snapshot + .read_blob(&entry.path, &operation) + .await + .map_err(remote_error)?; + let logical_size = blob.metadata.logical_size; + let attributes = attribute_manifest.object(path, entry.oid); + let modified = match attributes { + Some(attributes) => timestamp( + i64::try_from(attributes.modified_seconds) + .map_err(|_| s3_error!(InternalError))?, + )?, + None => commit_modified.clone(), + }; + let etag = match attributes { + Some(attributes) => attributes.etag.clone(), + None => md5_hex(&materialize_blob(repository, blob).await?), + }; + keys.push((key, etag, logical_size, modified)); + } + keys.sort_by(|left, right| left.0.cmp(&right.0)); + let max_keys = req.input.max_keys.unwrap_or(1000); + if max_keys < 0 { + return Err(s3_error!(InvalidArgument)); + } + let max_keys = max_keys.min(1000); + let limit = usize::try_from(max_keys).map_err(|_| s3_error!(InvalidArgument))?; + let delimiter = req.input.delimiter.as_deref(); + let mut values = Vec::new(); + let mut groups = std::collections::BTreeSet::new(); + for (key, etag, size, modified) in keys { + if let Some(delimiter) = delimiter + && let Some(relative) = key.strip_prefix(prefix) + && let Some(position) = relative.find(delimiter) + { + groups.insert(format!("{}{}", prefix, &relative[..=position])); + continue; + } + values.push((key, etag, size, modified)); + } + let mut projected = values + .into_iter() + .map(|(key, etag, size, modified)| { + ( + key.clone(), + Some(Object { + key: Some(key), + e_tag: Some(ETag::Strong(etag)), + last_modified: Some(modified), + size: size.and_then(|size| i64::try_from(size).ok()), + storage_class: Some(ObjectStorageClass::from_static("STANDARD")), + ..Default::default() + }), + None, + ) + }) + .chain(groups.into_iter().map(|prefix| { + ( + prefix.clone(), + None, + Some(CommonPrefix { + prefix: Some(prefix), + }), + ) + })) + .collect::>(); + projected.sort_by(|left, right| left.0.cmp(&right.0)); + let after = req + .input + .continuation_token + .as_deref() + .or(req.input.start_after.as_deref()); + if let Some(after) = after { + projected.retain(|(key, _, _)| key.as_str() > after); + } + let truncated = limit != 0 && projected.len() > limit; + projected.truncate(limit); + let next = truncated + .then(|| projected.last().map(|item| item.0.clone())) + .flatten(); + let contents = projected + .iter_mut() + .filter_map(|item| item.1.take()) + .collect::>(); + let common_prefixes = projected + .iter_mut() + .filter_map(|item| item.2.take()) + .collect::>(); + let mut contents = contents; + let mut common_prefixes = common_prefixes; + if url_encode { + for object in &mut contents { + object.key = encode_list_option(object.key.take(), true); + } + for group in &mut common_prefixes { + group.prefix = encode_list_option(group.prefix.take(), true); + } + } + Ok(S3Response::new(ListObjectsV2Output { + name: Some(req.input.bucket.clone()), + prefix: encode_list_option(req.input.prefix.clone(), url_encode), + max_keys: Some(max_keys), + key_count: Some( + i32::try_from(projected.len()).map_err(|_| s3_error!(InternalError))?, + ), + continuation_token: req.input.continuation_token.clone(), + next_continuation_token: next, + is_truncated: Some(truncated), + contents: (!contents.is_empty()).then_some(contents), + common_prefixes: (!common_prefixes.is_empty()).then_some(common_prefixes), + delimiter: encode_list_option(req.input.delimiter.clone(), url_encode), + encoding_type: req.input.encoding_type.clone(), + start_after: encode_list_option(req.input.start_after.clone(), url_encode), + ..Default::default() + })) + } + .await; + finish(operation, result).await + } +} + +async fn read_body(body: Option, declared: Option) -> S3Result { + use futures_util::StreamExt as _; + + let declared = declared + .map(|length| usize::try_from(length).map_err(|_| s3_error!(InvalidRequest))) + .transpose()?; + if declared.is_some_and(|length| length > MAX_SINGLE_OBJECT_BYTES) { + return Err(s3_error!(EntityTooLarge)); + } + let mut bytes = BytesMut::with_capacity(declared.unwrap_or(0).min(MAX_SINGLE_OBJECT_BYTES)); + let Some(mut body) = body else { + if declared.unwrap_or(0) != 0 { + return Err(s3_error!(IncompleteBody)); + } + return Ok(bytes.freeze()); + }; + while let Some(chunk) = body.next().await { + let chunk = chunk.map_err(|error| { + tracing::warn!(%error, "S3 request body failed"); + s3_error!(IncompleteBody) + })?; + if bytes.len().saturating_add(chunk.len()) > MAX_SINGLE_OBJECT_BYTES { + return Err(s3_error!(EntityTooLarge)); + } + bytes.extend_from_slice(&chunk); + } + if declared.is_some_and(|length| length != bytes.len()) { + return Err(s3_error!(IncompleteBody)); + } + Ok(bytes.freeze()) +} + +fn verify_content_md5(bytes: &[u8], expected: Option<&str>) -> S3Result<()> { + use md5::Digest as _; + + let Some(expected) = expected else { + return Ok(()); + }; + let actual = md5::Md5::digest(bytes); + let expected = base64::engine::general_purpose::STANDARD + .decode(expected) + .map_err(|_| s3_error!(InvalidDigest))?; + if expected.as_slice() != actual.as_slice() { + return Err(s3_error!(BadDigest)); + } + Ok(()) +} + +fn reject_put_extensions(input: &PutObjectInput) -> S3Result<()> { + if input.acl.is_some() + || input.bucket_key_enabled.is_some() + || input.expected_bucket_owner.is_some() + || input.grant_full_control.is_some() + || input.grant_read.is_some() + || input.grant_read_acp.is_some() + || input.grant_write_acp.is_some() + || input.if_match.is_some() + || input.if_none_match.is_some() + || input.object_lock_legal_hold_status.is_some() + || input.object_lock_mode.is_some() + || input.object_lock_retain_until_date.is_some() + || input.request_payer.is_some() + || input.server_side_encryption.is_some() + || input.sse_customer_algorithm.is_some() + || input.sse_customer_key.is_some() + || input.sse_customer_key_md5.is_some() + || input.ssekms_encryption_context.is_some() + || input.ssekms_key_id.is_some() + || input.storage_class.is_some() + || input.tagging.is_some() + || input.website_redirect_location.is_some() + || input.write_offset_bytes.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +#[derive(Clone)] +struct RequestChecksums { + algorithm: Option, + crc32: Option, + crc32c: Option, + crc64nvme: Option, + sha1: Option, + sha256: Option, +} + +impl From<&PutObjectInput> for RequestChecksums { + fn from(input: &PutObjectInput) -> Self { + Self { + algorithm: input.checksum_algorithm.clone(), + crc32: input.checksum_crc32.clone(), + crc32c: input.checksum_crc32c.clone(), + crc64nvme: input.checksum_crc64nvme.clone(), + sha1: input.checksum_sha1.clone(), + sha256: input.checksum_sha256.clone(), + } + } +} + +impl RequestChecksums { + fn merge_trailers(&mut self, trailers: Option<&s3s::TrailingHeaders>) -> S3Result<()> { + let Some(headers) = trailers.and_then(s3s::TrailingHeaders::take) else { + return Ok(()); + }; + merge_checksum_header(&mut self.crc32, &headers, "x-amz-checksum-crc32")?; + merge_checksum_header(&mut self.crc32c, &headers, "x-amz-checksum-crc32c")?; + merge_checksum_header(&mut self.crc64nvme, &headers, "x-amz-checksum-crc64nvme")?; + merge_checksum_header(&mut self.sha1, &headers, "x-amz-checksum-sha1")?; + merge_checksum_header(&mut self.sha256, &headers, "x-amz-checksum-sha256")?; + Ok(()) + } + + fn verify(&self, bytes: &[u8]) -> S3Result<()> { + use sha1::Digest as _; + + verify_base64_checksum( + self.crc32.as_deref(), + &u32::try_from(crc_fast::checksum( + crc_fast::CrcAlgorithm::Crc32IsoHdlc, + bytes, + )) + .map_err(|_| s3_error!(InternalError))? + .to_be_bytes(), + )?; + verify_base64_checksum( + self.crc32c.as_deref(), + &u32::try_from(crc_fast::checksum( + crc_fast::CrcAlgorithm::Crc32Iscsi, + bytes, + )) + .map_err(|_| s3_error!(InternalError))? + .to_be_bytes(), + )?; + verify_base64_checksum( + self.crc64nvme.as_deref(), + &crc_fast::checksum(crc_fast::CrcAlgorithm::Crc64Nvme, bytes).to_be_bytes(), + )?; + verify_base64_checksum(self.sha1.as_deref(), &sha1::Sha1::digest(bytes))?; + verify_base64_checksum(self.sha256.as_deref(), &sha2::Sha256::digest(bytes))?; + if let Some(algorithm) = &self.algorithm { + let supplied = match algorithm.as_str() { + ChecksumAlgorithm::CRC32 => self.crc32.is_some(), + ChecksumAlgorithm::CRC32C => self.crc32c.is_some(), + ChecksumAlgorithm::CRC64NVME => self.crc64nvme.is_some(), + ChecksumAlgorithm::SHA1 => self.sha1.is_some(), + ChecksumAlgorithm::SHA256 => self.sha256.is_some(), + _ => return Err(s3_error!(InvalidRequest, "Unsupported checksum algorithm")), + }; + if !supplied { + return Err(s3_error!(InvalidRequest, "Checksum value is missing")); + } + } + Ok(()) + } +} + +fn merge_checksum_header( + current: &mut Option, + headers: &http::HeaderMap, + name: &'static str, +) -> S3Result<()> { + let Some(value) = headers.get(name) else { + return Ok(()); + }; + let value = value + .to_str() + .map_err(|_| s3_error!(InvalidDigest))? + .to_owned(); + if current.as_ref().is_some_and(|current| current != &value) { + return Err(s3_error!(BadDigest)); + } + *current = Some(value); + Ok(()) +} + +fn verify_base64_checksum(expected: Option<&str>, actual: &[u8]) -> S3Result<()> { + let Some(expected) = expected else { + return Ok(()); + }; + let expected = base64::engine::general_purpose::STANDARD + .decode(expected) + .map_err(|_| s3_error!(InvalidDigest))?; + if expected.as_slice() != actual { + return Err(s3_error!(BadDigest)); + } + Ok(()) +} + +fn reject_delete_extensions(input: &DeleteObjectInput) -> S3Result<()> { + if input.bypass_governance_retention.is_some() + || input.expected_bucket_owner.is_some() + || input.if_match.is_some() + || input.if_match_last_modified_time.is_some() + || input.if_match_size.is_some() + || input.mfa.is_some() + || input.request_payer.is_some() + || input.version_id.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +fn reject_copy_extensions(input: &CopyObjectInput) -> S3Result<()> { + let directive_supported = input.metadata_directive.as_ref().is_none_or(|value| { + matches!( + value.as_str(), + MetadataDirective::COPY | MetadataDirective::REPLACE + ) + }); + if !directive_supported + || input.acl.is_some() + || input.bucket_key_enabled.is_some() + || input.checksum_algorithm.is_some() + || input.copy_source_sse_customer_algorithm.is_some() + || input.copy_source_sse_customer_key.is_some() + || input.copy_source_sse_customer_key_md5.is_some() + || input.expected_bucket_owner.is_some() + || input.expected_source_bucket_owner.is_some() + || input.grant_full_control.is_some() + || input.grant_read.is_some() + || input.grant_read_acp.is_some() + || input.grant_write_acp.is_some() + || input.object_lock_legal_hold_status.is_some() + || input.object_lock_mode.is_some() + || input.object_lock_retain_until_date.is_some() + || input.request_payer.is_some() + || input.server_side_encryption.is_some() + || input.sse_customer_algorithm.is_some() + || input.sse_customer_key.is_some() + || input.sse_customer_key_md5.is_some() + || input.ssekms_encryption_context.is_some() + || input.ssekms_key_id.is_some() + || input.storage_class.is_some() + || input.tagging.is_some() + || input.tagging_directive.is_some() + || input.website_redirect_location.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +fn reject_create_multipart_extensions(input: &CreateMultipartUploadInput) -> S3Result<()> { + if input.acl.is_some() + || input.bucket_key_enabled.is_some() + || input.checksum_algorithm.is_some() + || input.checksum_type.is_some() + || input.expected_bucket_owner.is_some() + || input.grant_full_control.is_some() + || input.grant_read.is_some() + || input.grant_read_acp.is_some() + || input.grant_write_acp.is_some() + || input.object_lock_legal_hold_status.is_some() + || input.object_lock_mode.is_some() + || input.object_lock_retain_until_date.is_some() + || input.request_payer.is_some() + || input.server_side_encryption.is_some() + || input.sse_customer_algorithm.is_some() + || input.sse_customer_key.is_some() + || input.sse_customer_key_md5.is_some() + || input.ssekms_encryption_context.is_some() + || input.ssekms_key_id.is_some() + || input.storage_class.is_some() + || input.tagging.is_some() + || input.website_redirect_location.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +fn reject_upload_part_extensions(input: &UploadPartInput) -> S3Result<()> { + if input.checksum_algorithm.is_some() + || input.checksum_crc32.is_some() + || input.checksum_crc32c.is_some() + || input.checksum_crc64nvme.is_some() + || input.checksum_sha1.is_some() + || input.checksum_sha256.is_some() + || input.expected_bucket_owner.is_some() + || input.request_payer.is_some() + || input.sse_customer_algorithm.is_some() + || input.sse_customer_key.is_some() + || input.sse_customer_key_md5.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +fn reject_complete_multipart_extensions(input: &CompleteMultipartUploadInput) -> S3Result<()> { + if input.checksum_crc32.is_some() + || input.checksum_crc32c.is_some() + || input.checksum_crc64nvme.is_some() + || input.checksum_sha1.is_some() + || input.checksum_sha256.is_some() + || input.checksum_type.is_some() + || input.expected_bucket_owner.is_some() + || input.if_match.is_some() + || input.if_none_match.is_some() + || input.mpu_object_size.is_some() + || input.request_payer.is_some() + || input.sse_customer_algorithm.is_some() + || input.sse_customer_key.is_some() + || input.sse_customer_key_md5.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +fn writable_branch<'a>( + repository: &Repository, + address: &'a namespace::ObjectAddress, +) -> S3Result<&'a str> { + let branch = address + .branch + .as_deref() + .ok_or_else(|| s3_error!(MethodNotAllowed, "Writes require a branch key"))?; + let short = branch.strip_prefix("refs/heads/").unwrap_or(branch); + if repository + .config + .protected_branches + .iter() + .any(|protected| protected == short) + { + return Err(s3_error!( + AccessDenied, + "The destination branch is protected" + )); + } + Ok(branch) +} + +fn copy_source_bucket_key(source: &CopySource) -> S3Result<(String, String)> { + match source { + CopySource::Bucket { + bucket, + key, + version_id: None, + } => Ok((bucket.to_string(), key.to_string())), + CopySource::Bucket { + version_id: Some(_), + .. + } => Err(s3_error!( + NotImplemented, + "Copying an object version is unsupported" + )), + CopySource::AccessPoint { .. } | CopySource::Outpost { .. } => Err(s3_error!( + NotImplemented, + "Copy access points are unsupported" + )), + } +} + +fn stored_to_pending( + value: &crate::attributes::ObjectAttributes, +) -> crate::attributes::PutAttributes { + crate::attributes::PutAttributes { + etag_override: None, + completion_upload_id: None, + cache_control: value.cache_control.clone(), + content_disposition: value.content_disposition.clone(), + content_encoding: value.content_encoding.clone(), + content_language: value.content_language.clone(), + content_type: value.content_type.clone(), + expires: value.expires.clone(), + metadata: value.metadata.clone(), + } +} + +impl Gateway { + async fn list_refs( + &self, + req: &S3Request, + repository: &Repository, + ) -> S3Result> { + let url_encode = list_url_encoding(req.input.encoding_type.as_ref())?; + let repo = self.open(repository).await?; + let mut keys = repo + .refs() + .entries + .iter() + .filter_map(|reference| reference.name.strip_prefix("refs/heads/")) + .map(|branch| { + format!( + "{}/", + percent_encoding::utf8_percent_encode( + branch, + percent_encoding::NON_ALPHANUMERIC + ) + ) + }) + .collect::>(); + keys.sort(); + let after = req + .input + .continuation_token + .as_deref() + .or(req.input.start_after.as_deref()); + if let Some(after) = after { + keys.retain(|key| key.as_str() > after); + } + let max_keys = req.input.max_keys.unwrap_or(1000); + if max_keys < 0 { + return Err(s3_error!(InvalidArgument)); + } + let max_keys = max_keys.min(1000); + let limit = usize::try_from(max_keys).map_err(|_| s3_error!(InvalidArgument))?; + let truncated = limit != 0 && keys.len() > limit; + keys.truncate(limit); + let next = truncated.then(|| keys.last().cloned()).flatten(); + let count = i32::try_from(keys.len()).map_err(|_| s3_error!(InternalError))?; + Ok(S3Response::new(ListObjectsV2Output { + name: Some(req.input.bucket.clone()), + prefix: encode_list_option(req.input.prefix.clone(), url_encode), + max_keys: Some(max_keys), + key_count: Some(count), + continuation_token: req.input.continuation_token.clone(), + next_continuation_token: next, + is_truncated: Some(truncated), + common_prefixes: Some( + keys.into_iter() + .map(|prefix| CommonPrefix { + prefix: Some(encode_list_value(&prefix, url_encode)), + }) + .collect(), + ), + delimiter: encode_list_option(req.input.delimiter.clone(), url_encode), + encoding_type: req.input.encoding_type.clone(), + start_after: encode_list_option(req.input.start_after.clone(), url_encode), + ..Default::default() + })) + } +} + +fn reject_get_extensions(input: &GetObjectInput) -> S3Result<()> { + if input.version_id.is_some() + || input.part_number.is_some() + || input.request_payer.is_some() + || input.sse_customer_algorithm.is_some() + || input.sse_customer_key.is_some() + || input.sse_customer_key_md5.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +fn reject_head_extensions(input: &HeadObjectInput) -> S3Result<()> { + if input.version_id.is_some() + || input.part_number.is_some() + || input.request_payer.is_some() + || input.sse_customer_algorithm.is_some() + || input.sse_customer_key.is_some() + || input.sse_customer_key_md5.is_some() + { + return Err(s3_error!(NotImplemented)); + } + Ok(()) +} + +fn evaluate_conditions( + if_match: Option<&ETagCondition>, + if_none_match: Option<&ETagCondition>, + if_modified_since: Option<&Timestamp>, + if_unmodified_since: Option<&Timestamp>, + etag: &str, + modified: &Timestamp, +) -> S3Result<()> { + let actual = ETag::Strong(etag.to_owned()); + if let Some(condition) = if_match { + let matches = match condition { + ETagCondition::Any => true, + ETagCondition::ETag(expected) => actual.strong_cmp(expected), + }; + if !matches { + return Err(s3_error!(PreconditionFailed)); + } + } else if if_unmodified_since.is_some_and(|expected| modified > expected) { + return Err(s3_error!(PreconditionFailed)); + } + if let Some(condition) = if_none_match { + let matches = match condition { + ETagCondition::Any => true, + ETagCondition::ETag(expected) => actual.weak_cmp(expected), + }; + if matches { + return Err(s3_error!(NotModified)); + } + } else if if_modified_since.is_some_and(|expected| modified <= expected) { + return Err(s3_error!(NotModified)); + } + Ok(()) +} + +async fn finish( + operation: crab_remote_git::OperationContext, + result: S3Result, +) -> S3Result { + match operation.finish(Ok(())).await { + Ok(()) => result, + Err(error) => Err(remote_error(error)), + } +} + +fn timestamp(seconds: i64) -> S3Result { + let seconds = u64::try_from(seconds).map_err(|_| s3_error!(InternalError))?; + UNIX_EPOCH + .checked_add(Duration::from_secs(seconds)) + .map(Timestamp::from) + .ok_or_else(|| s3_error!(InternalError)) +} + +fn list_url_encoding(encoding: Option<&EncodingType>) -> S3Result { + match encoding { + None => Ok(false), + Some(value) if value.as_str() == EncodingType::URL => Ok(true), + Some(_) => Err(s3_error!(InvalidArgument, "Unsupported list encoding type")), + } +} + +fn encode_list_option(value: Option, enabled: bool) -> Option { + value.map(|value| encode_list_value(&value, enabled)) +} + +fn encode_list_value(value: &str, enabled: bool) -> String { + if enabled { + percent_encoding::utf8_percent_encode(value, percent_encoding::NON_ALPHANUMERIC).to_string() + } else { + value.to_owned() + } +} + +fn namespace_error(error: namespace::NamespaceError) -> s3s::S3Error { + match error { + namespace::NamespaceError::MissingObject => s3_error!(NoSuchKey), + _ => s3_error!(InvalidArgument), + } +} + +fn remote_error(error: crab_remote_git::Error) -> s3s::S3Error { + match error { + crab_remote_git::Error::PathNotFound => s3_error!(NoSuchKey), + crab_remote_git::Error::Revision { .. } => s3_error!(NoSuchKey), + crab_remote_git::Error::Cancelled => s3_error!(RequestTimeout), + crab_remote_git::Error::LimitExceeded { .. } => s3_error!(SlowDown), + error => { + tracing::error!(error = ?error, "S3 repository read failed"); + s3_error!(InternalError) + } + } +} + +fn mutation_error(error: mutation::Error) -> s3s::S3Error { + match error { + mutation::Error::NotDirectory | mutation::Error::IsDirectory => { + s3_error!(InvalidObjectState) + } + mutation::Error::Cancelled => s3_error!(RequestTimeout), + mutation::Error::Write(crab_write::WriteError::RefChanged { .. }) => { + s3_error!( + OperationAborted, + "A conflicting branch write won; retry the request" + ) + } + mutation::Error::Coordination(crab_coordination::CoordinationError::PushLockHeld { + .. + }) => s3_error!( + OperationAborted, + "The destination branch is busy; retry the request" + ), + error => { + tracing::error!(error = ?error, "S3 repository mutation failed"); + s3_error!(InternalError) + } + } +} + +fn gateway_error(error: crate::Error) -> s3s::S3Error { + tracing::error!(error = ?error, "S3 gateway persistence failed"); + s3_error!(InternalError) +} + +pub(crate) fn md5_hex(bytes: &[u8]) -> String { + use md5::Digest as _; + + md5::Md5::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn multipart_etag(parts: &[crate::multipart::Part]) -> S3Result { + use md5::Digest as _; + + let mut binary = Vec::with_capacity(parts.len().saturating_mul(16)); + for part in parts { + if part.etag.len() != 32 { + return Err(s3_error!(InvalidPart)); + } + for index in (0..part.etag.len()).step_by(2) { + binary.push( + u8::from_str_radix(&part.etag[index..index + 2], 16) + .map_err(|_| s3_error!(InvalidPart))?, + ); + } + } + let digest = md5::Md5::digest(binary); + let hash: String = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + Ok(format!("{hash}-{}", parts.len())) +} + +fn now_seconds() -> S3Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .map_err(|_| s3_error!(InternalError)) +} + +fn multipart_error(error: crate::multipart::Error) -> s3s::S3Error { + match error { + crate::multipart::Error::NoSuchUpload + | crate::multipart::Error::NotOpen + | crate::multipart::Error::Identity => s3_error!(NoSuchUpload), + crate::multipart::Error::PartNumber | crate::multipart::Error::InvalidPart => { + s3_error!(InvalidPart) + } + crate::multipart::Error::InvalidPartOrder => s3_error!(InvalidPartOrder), + crate::multipart::Error::EntityTooSmall => s3_error!(EntityTooSmall), + crate::multipart::Error::EntityTooLarge => s3_error!(EntityTooLarge), + crate::multipart::Error::Conflict => s3_error!(OperationAborted), + error => { + tracing::error!(error = ?error, "S3 multipart state failed"); + s3_error!(InternalError) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn url_encoded_list_values_round_trip_reserved_and_unicode_bytes() { + let value = "main/special name(1)-雪.txt"; + let encoded = encode_list_value(value, true); + assert_eq!( + percent_encoding::percent_decode_str(&encoded) + .decode_utf8() + .unwrap(), + value + ); + assert!(encoded.contains("%2F")); + assert!(encoded.contains("%28")); + } + + #[test] + fn put_checksums_accept_all_supported_algorithms_and_reject_mismatch() { + use sha1::Digest as _; + + let body = b"123456789"; + let encode = |bytes: &[u8]| base64::engine::general_purpose::STANDARD.encode(bytes); + let checksums = RequestChecksums { + algorithm: Some(ChecksumAlgorithm::from_static(ChecksumAlgorithm::CRC32)), + crc32: Some(encode( + &u32::try_from(crc_fast::checksum( + crc_fast::CrcAlgorithm::Crc32IsoHdlc, + body, + )) + .unwrap() + .to_be_bytes(), + )), + crc32c: Some(encode( + &u32::try_from(crc_fast::checksum(crc_fast::CrcAlgorithm::Crc32Iscsi, body)) + .unwrap() + .to_be_bytes(), + )), + crc64nvme: Some(encode( + &crc_fast::checksum(crc_fast::CrcAlgorithm::Crc64Nvme, body).to_be_bytes(), + )), + sha1: Some(encode(&sha1::Sha1::digest(body))), + sha256: Some(encode(&sha2::Sha256::digest(body))), + }; + checksums.verify(body).unwrap(); + + let mut invalid = checksums; + invalid.sha256 = Some("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".to_owned()); + let error = invalid.verify(body).unwrap_err(); + assert_eq!(error.code().as_str(), "BadDigest"); + } +} diff --git a/crates/crab-s3-gateway/src/lib.rs b/crates/crab-s3-gateway/src/lib.rs new file mode 100644 index 000000000..70077de47 --- /dev/null +++ b/crates/crab-s3-gateway/src/lib.rs @@ -0,0 +1,49 @@ +//! S3 protocol composition for logical Crab repositories. + +mod attributes; +mod auth; +mod config; +mod gateway; +mod multipart; +mod mutation; +mod namespace; +mod repository; +mod server; + +pub use config::{Config, CredentialConfig, RepositoryAccess, RepositoryConfig, RepositoryMember}; +pub use server::{initialize, serve}; + +/// Gateway startup, configuration, and runtime failures. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid gateway configuration: {0}")] + Config(&'static str), + #[error("gateway configuration I/O failed")] + Io(#[from] std::io::Error), + #[error("invalid gateway TOML configuration")] + Toml(#[from] toml::de::Error), + #[error("object storage configuration failed")] + Storage(#[from] crab_storage::StorageError), + #[error("repository reader configuration failed")] + Remote(#[from] crab_remote_git::Error), + #[error("repository read publication failed")] + Write(#[from] crab_write::WriteError), + #[error("repository did not become readable before the publication deadline")] + ReadinessTimeout, + #[error("repository cache setup failed")] + Cache(#[from] crab_cache_store::CacheStoreError), + #[error("repository hydration failed")] + Read(#[from] crab_read::ReadError), + #[error("Git LFS hydration failed")] + Lfs(#[from] crab_lfs::LfsError), + #[error("S3 object attributes are corrupt")] + Attributes { + #[source] + source: serde_json::Error, + }, + #[error("gateway worker failed")] + Worker(#[from] tokio::task::JoinError), +} + +/// Gateway result retaining original sources. +pub type Result = std::result::Result; diff --git a/crates/crab-s3-gateway/src/main.rs b/crates/crab-s3-gateway/src/main.rs new file mode 100644 index 000000000..a5a8dd81c --- /dev/null +++ b/crates/crab-s3-gateway/src/main.rs @@ -0,0 +1,28 @@ +use std::path::PathBuf; + +use clap::Parser; + +#[derive(Debug, Parser)] +#[command(about = "Serve Crab repositories through the S3 protocol")] +struct Args { + #[arg(long, default_value = "crab-s3-gateway.toml")] + config: PathBuf, + /// Initialize configured repository prefixes and exit. + #[arg(long)] + initialize: bool, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init()?; + let args = Args::parse(); + let config = crab_s3_gateway::Config::read(&args.config)?; + if args.initialize { + crab_s3_gateway::initialize(config).await?; + } else { + crab_s3_gateway::serve(config).await?; + } + Ok(()) +} diff --git a/crates/crab-s3-gateway/src/multipart.rs b/crates/crab-s3-gateway/src/multipart.rs new file mode 100644 index 000000000..a94d6f6a4 --- /dev/null +++ b/crates/crab-s3-gateway/src/multipart.rs @@ -0,0 +1,467 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use crab_storage::ETag; +use serde::{Deserialize, Serialize}; + +use crate::{attributes::PutAttributes, gateway::Repository}; + +const VERSION: u32 = 1; +const MAX_RECORD_BYTES: u64 = 8 * 1024 * 1024; +const MAX_PARTS: usize = 10_000; + +pub(crate) type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("multipart upload does not exist")] + NoSuchUpload, + #[error("multipart upload is no longer open")] + NotOpen, + #[error("multipart upload identity does not match the request")] + Identity, + #[error("multipart part number is invalid")] + PartNumber, + #[error("multipart part is missing or has a different ETag")] + InvalidPart, + #[error("multipart parts are not strictly ascending")] + InvalidPartOrder, + #[error("a non-final multipart part is smaller than 5 MiB")] + EntityTooSmall, + #[error("the completed multipart object exceeds the gateway object limit")] + EntityTooLarge, + #[error("multipart state changed concurrently")] + Conflict, + #[error("multipart record is corrupt")] + Decode(#[from] serde_json::Error), + #[error("multipart storage failed")] + Storage(#[from] crab_storage::StorageError), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum State { + Open, + Completing, + Completed, + Aborted, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Part { + pub(crate) number: i32, + pub(crate) etag: String, + pub(crate) size: u64, + pub(crate) modified_seconds: u64, + path: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Session { + version: u32, + pub(crate) id: String, + pub(crate) bucket: String, + pub(crate) key: String, + pub(crate) branch: String, + pub(crate) path: String, + pub(crate) principal: String, + pub(crate) created_seconds: u64, + revision: u64, + state: State, + pub(crate) attributes: PutAttributes, + pub(crate) parts: BTreeMap, + selected_parts: Option>, + completion_etag: Option, +} + +pub(crate) struct Loaded { + pub(crate) session: Session, + etag: ETag, +} + +pub(crate) async fn create( + repository: &Repository, + bucket: &str, + key: &str, + branch: &str, + path: &str, + principal: &str, + attributes: PutAttributes, + now: u64, +) -> Result { + for _ in 0..8 { + let id = ulid::Ulid::new().to_string(); + let session = Session { + version: VERSION, + id: id.clone(), + bucket: bucket.to_owned(), + key: key.to_owned(), + branch: branch.to_owned(), + path: path.to_owned(), + principal: principal.to_owned(), + created_seconds: now, + revision: 0, + state: State::Open, + attributes: attributes.clone(), + parts: BTreeMap::new(), + selected_parts: None, + completion_etag: None, + }; + let bytes = serde_json::to_vec(&session)?; + match repository + .store + .create_strict(&state_path(repository, &id), Bytes::from(bytes)) + .await + { + Ok(()) => return Ok(session), + Err(crab_storage::StorageError::StateConflict { .. }) => {} + Err(error) => return Err(error.into()), + } + } + Err(Error::Conflict) +} + +pub(crate) async fn load(repository: &Repository, id: &str) -> Result { + validate_id(id)?; + let (bytes, etag) = repository + .store + .get_with_etag_bounded(&state_path(repository, id), MAX_RECORD_BYTES) + .await + .map_err(|error| match error { + crab_storage::StorageError::NotFound { .. } => Error::NoSuchUpload, + error => Error::Storage(error), + })?; + let session: Session = serde_json::from_slice(&bytes)?; + if session.version != VERSION || session.id != id { + return Err(Error::Identity); + } + Ok(Loaded { session, etag }) +} + +pub(crate) fn authorize(session: &Session, bucket: &str, key: &str, principal: &str) -> Result<()> { + if session.bucket != bucket || session.key != key || session.principal != principal { + return Err(Error::Identity); + } + Ok(()) +} + +pub(crate) async fn register_part( + repository: &Repository, + mut loaded: Loaded, + number: i32, + bytes: Bytes, + etag: String, + now: u64, +) -> Result { + if !(1..=10_000).contains(&number) { + return Err(Error::PartNumber); + } + if !matches!(loaded.session.state, State::Open) { + return Err(Error::NotOpen); + } + let path = format!("s3/multipart/parts/{}/{number}/{etag}", loaded.session.id); + repository + .store + .put_exact(&repository.layout.repo_path(&path), bytes.clone()) + .await?; + let part = Part { + number, + etag, + size: bytes.len() as u64, + modified_seconds: now, + path, + }; + loaded.session.parts.insert(number, part.clone()); + if loaded.session.parts.len() > MAX_PARTS { + return Err(Error::PartNumber); + } + loaded.session.revision = loaded.session.revision.saturating_add(1); + save(repository, &loaded).await?; + Ok(part) +} + +pub(crate) async fn freeze( + repository: &Repository, + mut loaded: Loaded, + selected: &[(i32, String)], + max_total_bytes: u64, +) -> Result<(Session, Vec)> { + if matches!(loaded.session.state, State::Completing) { + if loaded.session.selected_parts.as_deref() != Some(selected) { + return Err(Error::InvalidPart); + } + let parts = selected + .iter() + .map(|(number, _)| { + loaded + .session + .parts + .get(number) + .cloned() + .ok_or(Error::InvalidPart) + }) + .collect::>>()?; + return Ok((loaded.session, parts)); + } + if !matches!(loaded.session.state, State::Open) { + return Err(Error::NotOpen); + } + if selected.is_empty() || selected.len() > MAX_PARTS { + return Err(Error::InvalidPart); + } + let mut previous = 0; + let mut total_bytes = 0_u64; + let mut parts = Vec::with_capacity(selected.len()); + for (index, (number, etag)) in selected.iter().enumerate() { + if *number <= previous { + return Err(Error::InvalidPartOrder); + } + previous = *number; + let part = loaded.session.parts.get(number).ok_or(Error::InvalidPart)?; + if &part.etag != etag { + return Err(Error::InvalidPart); + } + if index + 1 != selected.len() && part.size < 5 * 1024 * 1024 { + return Err(Error::EntityTooSmall); + } + total_bytes = total_bytes + .checked_add(part.size) + .ok_or(Error::EntityTooLarge)?; + if total_bytes > max_total_bytes { + return Err(Error::EntityTooLarge); + } + parts.push(part.clone()); + } + loaded.session.state = State::Completing; + loaded.session.selected_parts = Some(selected.to_vec()); + loaded.session.revision = loaded.session.revision.saturating_add(1); + save(repository, &loaded).await?; + Ok((loaded.session, parts)) +} + +pub(crate) async fn part_bytes(repository: &Repository, part: &Part) -> Result { + let (bytes, _) = repository + .store + .get_with_etag_bounded(&repository.layout.repo_path(&part.path), part.size) + .await?; + if bytes.len() as u64 != part.size || crate::gateway::md5_hex(&bytes) != part.etag { + return Err(Error::InvalidPart); + } + Ok(bytes) +} + +pub(crate) async fn complete( + repository: &Repository, + mut loaded: Loaded, + etag: String, +) -> Result<()> { + if matches!(loaded.session.state, State::Completed) + && loaded.session.completion_etag.as_deref() == Some(&etag) + { + return Ok(()); + } + if !matches!(loaded.session.state, State::Completing) { + return Err(Error::NotOpen); + } + loaded.session.state = State::Completed; + loaded.session.completion_etag = Some(etag); + loaded.session.revision = loaded.session.revision.saturating_add(1); + save(repository, &loaded).await?; + if let Err(error) = cleanup_parts(repository, &loaded.session.id).await { + tracing::warn!(upload_id = %loaded.session.id, %error, "completed multipart part cleanup failed"); + } + Ok(()) +} + +pub(crate) fn completed_etag<'a>( + session: &'a Session, + selected: &[(i32, String)], +) -> Result> { + if !matches!(session.state, State::Completed) { + return Ok(None); + } + if session.selected_parts.as_deref() != Some(selected) { + return Err(Error::InvalidPart); + } + Ok(session.completion_etag.as_deref()) +} + +pub(crate) async fn abort(repository: &Repository, mut loaded: Loaded) -> Result<()> { + if !matches!(loaded.session.state, State::Open) { + return Err(Error::NotOpen); + } + loaded.session.state = State::Aborted; + loaded.session.revision = loaded.session.revision.saturating_add(1); + save(repository, &loaded).await?; + cleanup_parts(repository, &loaded.session.id).await?; + Ok(()) +} + +async fn cleanup_parts(repository: &Repository, id: &str) -> Result<()> { + let prefix = repository + .layout + .repo_path(&format!("s3/multipart/parts/{id}/")); + repository.store.delete_prefix(&prefix).await?; + Ok(()) +} + +pub(crate) async fn list(repository: &Repository) -> Result> { + let prefix = repository.layout.repo_path("s3/multipart/uploads/"); + let mut sessions = Vec::new(); + for object in repository.store.list_prefix(&prefix).await? { + if !object.location.as_ref().ends_with("/state.json") { + continue; + } + let (bytes, _) = repository + .store + .get_with_etag_bounded(&object.location, MAX_RECORD_BYTES) + .await?; + let session: Session = serde_json::from_slice(&bytes)?; + if session.version == VERSION && matches!(session.state, State::Open) { + sessions.push(session); + } + } + sessions.sort_by(|left, right| (&left.key, &left.id).cmp(&(&right.key, &right.id))); + Ok(sessions) +} + +async fn save(repository: &Repository, loaded: &Loaded) -> Result<()> { + let bytes = serde_json::to_vec(&loaded.session)?; + repository + .store + .update( + &state_path(repository, &loaded.session.id), + Bytes::from(bytes), + loaded.etag.clone(), + ) + .await + .map(drop) + .map_err(|error| match error { + crab_storage::StorageError::StateConflict { .. } => Error::Conflict, + error => Error::Storage(error), + }) +} + +fn state_path(repository: &Repository, id: &str) -> object_store::path::Path { + repository + .layout + .repo_path(&format!("s3/multipart/uploads/{id}/state.json")) +} + +fn validate_id(id: &str) -> Result<()> { + if id.len() != 26 || !id.bytes().all(|byte| byte.is_ascii_alphanumeric()) { + return Err(Error::NoSuchUpload); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use crate::{RepositoryAccess, RepositoryConfig, RepositoryMember}; + + async fn fixture() -> Repository { + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())); + let layout = crab_storage::StoreLayout::new(store.clone(), "multipart-test".to_owned()); + crab_write::initialize::initialize_repository(&store, &layout, "refs/heads/main") + .await + .unwrap(); + Repository::new( + RepositoryConfig { + name: "repo".to_owned(), + provider: crab_storage::StorageProviderKind::Local, + bucket: "memory".to_owned(), + prefix: "multipart-test".to_owned(), + default_branch: "main".to_owned(), + members: vec![RepositoryMember { + principal: "user".to_owned(), + access: RepositoryAccess::Write, + }], + protected_branches: vec![], + }, + store, + ) + .unwrap() + } + + #[tokio::test] + async fn parts_and_abort_survive_fresh_catalog_reads() { + let repository = fixture().await; + let session = create( + &repository, + "repo", + "main/file.bin", + "refs/heads/main", + "file.bin", + "user", + PutAttributes::default(), + 10, + ) + .await + .unwrap(); + let loaded = load(&repository, &session.id).await.unwrap(); + let body = Bytes::from_static(b"part bytes"); + let etag = crate::gateway::md5_hex(&body); + register_part(&repository, loaded, 1, body.clone(), etag.clone(), 11) + .await + .unwrap(); + + let reloaded = load(&repository, &session.id).await.unwrap(); + assert_eq!(reloaded.session.parts[&1].etag, etag); + assert_eq!( + part_bytes(&repository, &reloaded.session.parts[&1]) + .await + .unwrap(), + body + ); + assert_eq!(list(&repository).await.unwrap().len(), 1); + + abort(&repository, reloaded).await.unwrap(); + assert!(list(&repository).await.unwrap().is_empty()); + let terminal = load(&repository, &session.id).await.unwrap(); + assert!(matches!(terminal.session.state, State::Aborted)); + } + + #[tokio::test] + async fn identical_completion_can_resume_and_return_recorded_outcome() { + let repository = fixture().await; + let session = create( + &repository, + "repo", + "main/file.bin", + "refs/heads/main", + "file.bin", + "user", + PutAttributes::default(), + 10, + ) + .await + .unwrap(); + let body = Bytes::from_static(b"final part"); + let etag = crate::gateway::md5_hex(&body); + let loaded = load(&repository, &session.id).await.unwrap(); + register_part(&repository, loaded, 1, body, etag.clone(), 11) + .await + .unwrap(); + let selected = vec![(1, etag)]; + let loaded = load(&repository, &session.id).await.unwrap(); + freeze(&repository, loaded, &selected, 1024).await.unwrap(); + let loaded = load(&repository, &session.id).await.unwrap(); + freeze(&repository, loaded, &selected, 1024).await.unwrap(); + let loaded = load(&repository, &session.id).await.unwrap(); + complete(&repository, loaded, "result-etag".to_owned()) + .await + .unwrap(); + + let loaded = load(&repository, &session.id).await.unwrap(); + assert_eq!( + completed_etag(&loaded.session, &selected).unwrap(), + Some("result-etag") + ); + assert!(completed_etag(&loaded.session, &[(2, "other".to_owned())]).is_err()); + } +} diff --git a/crates/crab-s3-gateway/src/mutation.rs b/crates/crab-s3-gateway/src/mutation.rs new file mode 100644 index 000000000..4f8787eb7 --- /dev/null +++ b/crates/crab-s3-gateway/src/mutation.rs @@ -0,0 +1,870 @@ +use std::{ + collections::BTreeMap, + io::Write as _, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use bytes::Bytes; +use crab_coordination::{GcFenceHeartbeat, GcFenceLease, PushLock}; +use crab_metadata::{git_visibility, manifests::PackManifestEntry, ref_journal::RefJournalEdit}; +use crab_remote_git::{EntryMode, OperationKind, RemoteGitRepository, Revision}; +use gix_hash::ObjectId; +use gix_object::{Kind, WriteTo as _, bstr::BString, tree}; +use md5::Digest as _; +use tokio_util::sync::CancellationToken; + +use crate::{attributes, gateway::Repository}; + +const LOCK_TTL: Duration = Duration::from_secs(300); +const MAX_GENERATED_PACK_BYTES: u64 = 512 * 1024 * 1024; + +pub(crate) type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("object path resolves through a non-directory")] + NotDirectory, + #[error("object path names a directory")] + IsDirectory, + #[error("repository mutation was cancelled")] + Cancelled, + #[error("system clock is before the Unix epoch")] + Clock(#[from] std::time::SystemTimeError), + #[error("repository read failed")] + Remote(#[from] crab_remote_git::Error), + #[error("Git object encoding failed")] + Object(#[from] gix_object::encode::Error), + #[error("Git object hashing failed")] + Hash(#[from] gix_hash::hasher::Error), + #[error("temporary pack I/O failed")] + Io(#[from] std::io::Error), + #[error("generated pack validation failed")] + Pack(#[from] crab_git::incoming_pack::IncomingPackError), + #[error("generated pack preparation failed")] + Prepare(#[from] crab_git::incoming_pack::PreparePackError), + #[error("repository storage failed")] + Storage(#[from] crab_storage::StorageError), + #[error("repository metadata failed")] + Metadata(#[from] crab_metadata::error::MetadataError), + #[error("repository coordination failed")] + Coordination(#[from] crab_coordination::CoordinationError), + #[error("repository publication failed")] + Write(#[from] crab_write::WriteError), + #[error("mutation worker failed")] + Worker(#[from] tokio::task::JoinError), + #[error("S3 attribute persistence failed")] + Attributes(#[source] Box), +} + +impl From for Error { + fn from(error: crate::Error) -> Self { + Self::Attributes(Box::new(error)) + } +} + +#[derive(Clone, Debug)] +pub(crate) enum Change { + Put { + bytes: Bytes, + attributes: Box, + }, + Delete, +} + +#[derive(Clone, Debug)] +pub(crate) struct Outcome { + pub(crate) etag: Option, +} + +struct RefLease { + holder: String, + stop: CancellationToken, + worker: tokio::task::JoinHandle>, +} + +impl RefLease { + async fn acquire( + repository: &Repository, + branch: &str, + cancel: &CancellationToken, + ) -> Result { + check_cancelled(cancel)?; + let mut lock = PushLock::acquire_ref( + repository.store.inner(), + repository.layout.repo_prefix(), + branch, + LOCK_TTL, + ) + .await?; + let holder = lock.holder().to_owned(); + let stop = CancellationToken::new(); + let stopped = stop.clone(); + let cancel = cancel.clone(); + let worker = tokio::spawn(async move { + let result = crab_coordination::while_renewing(&mut lock, Some(&cancel), async { + stopped.cancelled().await; + Ok::<_, crab_coordination::CoordinationError>(()) + }) + .await; + result.and(lock.release().await) + }); + Ok(Self { + holder, + stop, + worker, + }) + } + + async fn release(self) { + self.stop.cancel(); + if let result @ (Err(_) | Ok(Err(_))) = self.worker.await { + tracing::warn!(?result, "S3 ref lease cleanup failed"); + } + } +} + +pub(crate) async fn apply( + repository: &Repository, + runtime: Arc, + options: crab_remote_git::RepositoryOptions, + branch: &str, + path: &crab_remote_git::GitPath, + change: Change, + principal: &str, + cancel: &CancellationToken, +) -> Result { + let lease = RefLease::acquire(repository, branch, cancel).await?; + let maintenance_runtime = Arc::clone(&runtime); + let mut fences = Vec::new(); + let result = async { + for domain in [ + repository.layout.global_prefix(), + repository.layout.repo_prefix(), + ] { + check_cancelled(cancel)?; + let fence = + GcFenceLease::acquire_writer(repository.store.inner(), domain, LOCK_TTL).await?; + let heartbeat = GcFenceHeartbeat::spawn(&fence, cancel.clone(), LOCK_TTL / 3); + fences.push((fence, heartbeat)); + } + Box::pin(apply_locked( + repository, + runtime, + options, + branch, + path, + change, + principal, + &lease.holder, + cancel, + )) + .await + } + .await; + for (fence, heartbeat) in fences.into_iter().rev() { + heartbeat.stop().await; + if let Err(error) = fence.release().await { + tracing::warn!(%error, "S3 GC fence cleanup failed"); + } + } + lease.release().await; + if result.is_ok() { + crate::repository::ensure_readable(repository, maintenance_runtime, options, cancel) + .await?; + } + result +} + +#[expect(clippy::too_many_arguments)] +async fn apply_locked( + repository: &Repository, + runtime: Arc, + options: crab_remote_git::RepositoryOptions, + branch: &str, + path: &crab_remote_git::GitPath, + change: Change, + principal: &str, + holder: &str, + cancel: &CancellationToken, +) -> Result { + check_cancelled(cancel)?; + let remote = crate::repository::open_current(repository, runtime, options, cancel).await?; + let old = remote + .refs() + .entries + .iter() + .find(|reference| reference.name == branch) + .map(|reference| reference.target); + let snapshot = crab_metadata::manifest_store::read_repository_snapshot( + &repository.store, + &repository.layout, + ) + .await?; + if snapshot.manifest.generation != remote.generation() + || snapshot.journal.refs.get(branch).map(String::as_str) + != old.as_ref().map(ToString::to_string).as_deref() + { + return Err(crab_write::WriteError::RefChanged { + ref_name: branch.to_owned(), + path: repository.layout.repo_prefix().to_owned(), + } + .into()); + } + let mut attribute_manifest = match old { + Some(old) => attributes::load(repository, old) + .await + .map_err(|error| Error::Attributes(Box::new(error)))?, + None => attributes::Manifest::default(), + }; + let operation = remote.operation(OperationKind::Repository, cancel).await?; + let built = build_commit( + &remote, + &operation, + old, + path, + change, + principal, + &attribute_manifest, + ) + .await; + let built = match operation.finish(Ok(())).await { + Ok(()) => built?, + Err(error) => return Err(error.into()), + }; + let built = match built { + Build::Noop(outcome) => return Ok(outcome), + Build::Commit(built) => *built, + }; + let (pack_owner, pack) = prepare_pack(built.objects.clone(), cancel).await?; + check_cancelled(cancel)?; + let pack_id = pack.content_hash().to_hex().to_string(); + repository + .store + .put_multipart_file_retry( + &repository.layout.pack_path(&pack_id), + pack.pack_path(), + pack.size(), + *pack.content_hash().as_bytes(), + 8 * 1024 * 1024, + cancel, + None, + ) + .await?; + for (source, target) in [ + ( + pack.index_path(), + repository.layout.pack_index_path(&pack_id), + ), + ( + pack.reverse_path(), + repository.layout.pack_reverse_index_path(&pack_id), + ), + ( + pack.kinds_path(), + repository.layout.pack_kind_metadata_path(&pack_id), + ), + ] { + check_cancelled(cancel)?; + repository + .store + .put_exact(&target, tokio::fs::read(source).await?.into()) + .await?; + } + let visible_objects = built + .objects + .iter() + .map(|(kind, bytes)| object_id(*kind, bytes).map(|oid| oid.to_string())) + .collect::, _>>()?; + let evidence = match old { + Some(old) => git_visibility::GitVisibilityEdit::from_delta_objects( + Some(old.to_string()), + built.commit.to_string(), + visible_objects, + vec![], + ), + None => git_visibility::GitVisibilityEdit::from_replacement_objects( + None, + built.commit.to_string(), + visible_objects, + ), + }; + let evidence_hash = + git_visibility::upload_edit(&repository.store, &repository.layout, &evidence).await?; + match built.attributes.clone() { + Some(attributes) => attribute_manifest.put(built.path.clone(), attributes), + None => attribute_manifest.remove(&built.path), + } + attributes::save(repository, built.commit, &attribute_manifest) + .await + .map_err(|error| Error::Attributes(Box::new(error)))?; + check_cancelled(cancel)?; + crab_write::journal::commit_edits( + &repository.store, + &repository.layout, + &snapshot, + vec![RefJournalEdit { + ref_name: branch.to_owned(), + old_oid: old.map(|oid| oid.to_string()), + new_oid: Some(built.commit.to_string()), + peeled_oid: None, + lock_holder: Some(holder.to_owned()), + visibility_evidence_hash: Some(evidence_hash), + }], + old.is_none().then(|| branch.to_owned()), + vec![PackManifestEntry { + pack_id: pack_id.clone(), + content_hash: pack_id, + size: pack.size(), + object_count: pack.object_count().into(), + ref_tips: vec![built.commit.to_string()], + }], + vec![], + crab_write::journal::CommitOptions::new(LOCK_TTL, cancel), + ) + .await?; + drop(pack); + drop(pack_owner); + Ok(Outcome { etag: built.etag }) +} + +struct BuiltCommit { + commit: ObjectId, + etag: Option, + objects: Vec<(Kind, Vec)>, + path: String, + attributes: Option, +} + +enum Build { + Noop(Outcome), + Commit(Box), +} + +#[derive(Default)] +struct TreeNode { + old_oid: Option, + files: BTreeMap, TreeLeaf>, + directories: BTreeMap, TreeNode>, +} + +struct TreeLeaf { + oid: ObjectId, + mode: EntryMode, +} + +async fn build_commit( + remote: &RemoteGitRepository, + operation: &crab_remote_git::OperationContext, + parent: Option, + path: &crab_remote_git::GitPath, + change: Change, + principal: &str, + attribute_manifest: &attributes::Manifest, +) -> Result { + let mut root = TreeNode::default(); + if let Some(parent) = parent { + let snapshot = remote + .snapshot(&Revision::Commit(parent), operation) + .await?; + root.old_oid = Some(snapshot.root_tree_oid()); + for entry in snapshot.list_tree_recursive(operation).await? { + insert_entry(&mut root, &entry)?; + } + } + let old = root.file(path)?.map(|leaf| (leaf.oid, leaf.mode)); + let path_string = std::str::from_utf8(path.as_bytes()) + .map_err(|_| std::io::Error::other("S3 object path is not UTF-8"))?; + let (etag, changed, pending_attributes) = match change { + Change::Put { bytes, attributes } => { + let oid = object_id(Kind::Blob, &bytes)?; + let digest = md5::Md5::digest(&bytes); + let etag = attributes + .etag_override + .clone() + .unwrap_or_else(|| digest.iter().map(|byte| format!("{byte:02x}")).collect()); + if attributes.completion_upload_id.is_some() + && old.is_some_and(|(old_oid, mode)| old_oid == oid && mode == EntryMode::Regular) + && attribute_manifest + .object(path_string, oid) + .is_some_and(|stored| stored.matches_pending(&attributes, &etag, bytes.len())) + { + return Ok(Build::Noop(Outcome { etag: Some(etag) })); + } + root.put(path, oid)?; + let changed = old + .filter(|(old_oid, _)| *old_oid == oid) + .map(|_| None) + .unwrap_or_else(|| Some(bytes.to_vec())); + ( + Some(etag), + changed, + Some((oid, attributes, bytes.len() as u64)), + ) + } + Change::Delete => { + if old.is_none() { + return Ok(Build::Noop(Outcome { etag: None })); + } + root.delete(path)?; + (None, None, None) + } + }; + let mut objects = Vec::new(); + if let Some(bytes) = changed { + objects.push((Kind::Blob, bytes)); + } + let tree = encode_node(&root, &mut objects)?; + let seconds = now_seconds()?; + let object_attributes = pending_attributes.map(|(oid, pending, size)| { + attributes::ObjectAttributes::new( + oid, + etag.clone().unwrap_or_default(), + size, + seconds, + *pending, + ) + }); + let commit_bytes = commit_bytes(tree, parent, principal, seconds); + let commit = object_id(Kind::Commit, &commit_bytes)?; + objects.push((Kind::Commit, commit_bytes)); + Ok(Build::Commit(Box::new(BuiltCommit { + commit, + etag, + objects, + path: path_string.to_owned(), + attributes: object_attributes, + }))) +} + +fn insert_entry(root: &mut TreeNode, entry: &crab_remote_git::TreeEntry) -> Result<()> { + let components = entry.path.components().collect::>(); + let (name, parents) = components.split_last().ok_or(Error::NotDirectory)?; + let mut node = root; + for component in parents { + node = node.directories.entry((*component).to_vec()).or_default(); + } + if entry.mode == EntryMode::Tree { + node.directories + .entry((*name).to_vec()) + .or_default() + .old_oid = Some(entry.oid); + } else { + node.files.insert( + (*name).to_vec(), + TreeLeaf { + oid: entry.oid, + mode: entry.mode, + }, + ); + } + Ok(()) +} + +impl TreeNode { + fn file(&self, path: &crab_remote_git::GitPath) -> Result> { + let components = path.components().collect::>(); + let (name, parents) = components.split_last().ok_or(Error::IsDirectory)?; + let mut node = self; + for component in parents { + if node.files.contains_key(*component) { + return Err(Error::NotDirectory); + } + let Some(next) = node.directories.get(*component) else { + return Ok(None); + }; + node = next; + } + if node.directories.contains_key(*name) { + return Err(Error::IsDirectory); + } + Ok(node.files.get(*name)) + } + + fn put(&mut self, path: &crab_remote_git::GitPath, oid: ObjectId) -> Result<()> { + let components = path.components().collect::>(); + let (name, parents) = components.split_last().ok_or(Error::IsDirectory)?; + let mut node = self; + for component in parents { + if node.files.contains_key(*component) { + return Err(Error::NotDirectory); + } + node = node.directories.entry((*component).to_vec()).or_default(); + } + if node.directories.contains_key(*name) { + return Err(Error::IsDirectory); + } + node.files.insert( + (*name).to_vec(), + TreeLeaf { + oid, + mode: EntryMode::Regular, + }, + ); + Ok(()) + } + + fn delete(&mut self, path: &crab_remote_git::GitPath) -> Result<()> { + let components = path.components().collect::>(); + let (name, parents) = components.split_last().ok_or(Error::IsDirectory)?; + delete_from(self, parents, name) + } +} + +fn delete_from(node: &mut TreeNode, parents: &[&[u8]], name: &[u8]) -> Result<()> { + let Some((component, rest)) = parents.split_first() else { + node.files.remove(name); + return Ok(()); + }; + if node.files.contains_key(*component) { + return Err(Error::NotDirectory); + } + let Some(child) = node.directories.get_mut(*component) else { + return Ok(()); + }; + delete_from(child, rest, name)?; + if child.files.is_empty() && child.directories.is_empty() { + node.directories.remove(*component); + } + Ok(()) +} + +fn encode_node(node: &TreeNode, objects: &mut Vec<(Kind, Vec)>) -> Result { + let mut entries = Vec::with_capacity(node.files.len() + node.directories.len()); + for (name, child) in &node.directories { + let oid = encode_node(child, objects)?; + entries.push(tree::Entry { + mode: tree::EntryKind::Tree.into(), + filename: BString::from(name.clone()), + oid, + }); + } + for (name, leaf) in &node.files { + let mode = match leaf.mode { + EntryMode::Regular => tree::EntryKind::Blob, + EntryMode::Executable => tree::EntryKind::BlobExecutable, + EntryMode::Symlink => tree::EntryKind::Link, + EntryMode::Submodule => tree::EntryKind::Commit, + EntryMode::Tree => return Err(Error::IsDirectory), + }; + entries.push(tree::Entry { + mode: mode.into(), + filename: BString::from(name.clone()), + oid: leaf.oid, + }); + } + entries.sort(); + let mut bytes = Vec::new(); + gix_object::Tree { entries }.write_to(&mut bytes)?; + let oid = object_id(Kind::Tree, &bytes)?; + if node.old_oid != Some(oid) { + objects.push((Kind::Tree, bytes)); + } + Ok(oid) +} + +fn commit_bytes( + tree: ObjectId, + parent: Option, + principal: &str, + seconds: u64, +) -> Vec { + let name: String = principal + .chars() + .filter(|character| !matches!(character, '<' | '>' | '\n' | '\r' | '\0')) + .take(160) + .collect(); + let name = if name.trim().is_empty() { + "Crab S3 user" + } else { + name.trim() + }; + let email = blake3::hash(principal.as_bytes()).to_hex(); + let parent = parent + .map(|oid| format!("parent {oid}\n")) + .unwrap_or_default(); + format!( + "tree {tree}\n{parent}author {name} <{email}@users.crab.invalid> {seconds} +0000\ncommitter {name} <{email}@users.crab.invalid> {seconds} +0000\n\nUpdate object through Crab S3 gateway\n" + ) + .into_bytes() +} + +fn object_id(kind: Kind, bytes: &[u8]) -> std::result::Result { + gix_object::compute_hash(gix_hash::Kind::Sha1, kind, bytes) +} + +async fn prepare_pack( + objects: Vec<(Kind, Vec)>, + cancel: &CancellationToken, +) -> Result<(tempfile::TempDir, crab_git::incoming_pack::PreparedPack)> { + let cancelled = Arc::new(AtomicBool::new(cancel.is_cancelled())); + let watched = cancel.clone(); + let flag = Arc::clone(&cancelled); + let watcher = tokio::spawn(async move { + watched.cancelled().await; + flag.store(true, Ordering::Release); + }); + let worker_flag = Arc::clone(&cancelled); + let result = tokio::task::spawn_blocking(move || { + let owner = tempfile::tempdir()?; + let input = owner.path().join("generated.pack"); + write_pack(&input, &objects)?; + let incoming = crab_git::incoming_pack::quarantine( + std::io::BufReader::new(std::fs::File::open(&input)?), + owner.path(), + crab_git::incoming_pack::ReceiveLimits { + max_pack_bytes: MAX_GENERATED_PACK_BYTES, + max_objects: 1_000_000, + max_object_bytes: 256 * 1024 * 1024, + max_inflated_bytes: MAX_GENERATED_PACK_BYTES, + max_delta_depth: 1, + }, + || worker_flag.load(Ordering::Acquire), + |_| Ok(None), + )?; + let prepared = incoming + .prepare(owner.path(), MAX_GENERATED_PACK_BYTES, &worker_flag)? + .ok_or_else(|| std::io::Error::other("generated pack contains no objects"))?; + Ok::<_, Error>((owner, prepared)) + }) + .await?; + watcher.abort(); + result +} + +fn write_pack(path: &std::path::Path, objects: &[(Kind, Vec)]) -> Result<()> { + let count = u32::try_from(objects.len()) + .map_err(|_| std::io::Error::other("too many generated Git objects"))?; + let mut bytes = b"PACK\0\0\0\x02".to_vec(); + bytes.extend_from_slice(&count.to_be_bytes()); + for (kind, data) in objects { + pack_header(*kind, data.len(), &mut bytes); + let mut encoder = + flate2::write::ZlibEncoder::new(&mut bytes, flate2::Compression::default()); + encoder.write_all(data)?; + encoder.finish()?; + } + let mut hasher = gix_hash::hasher(gix_hash::Kind::Sha1); + hasher.update(&bytes); + bytes.extend_from_slice(hasher.try_finalize()?.as_bytes()); + std::fs::write(path, bytes)?; + Ok(()) +} + +fn pack_header(kind: Kind, size: usize, output: &mut Vec) { + let kind = match kind { + Kind::Commit => 1, + Kind::Tree => 2, + Kind::Blob => 3, + Kind::Tag => 4, + }; + let mut remaining = size >> 4; + let mut byte = (kind << 4) | (size as u8 & 0x0f); + while remaining != 0 { + output.push(byte | 0x80); + byte = (remaining as u8) & 0x7f; + remaining >>= 7; + } + output.push(byte); +} + +fn now_seconds() -> Result { + Ok(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()) +} + +fn check_cancelled(cancel: &CancellationToken) -> Result<()> { + if cancel.is_cancelled() { + Err(Error::Cancelled) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{RepositoryAccess, RepositoryConfig}; + + async fn fixture() -> ( + Repository, + Arc, + CancellationToken, + ) { + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())); + let layout = crab_storage::StoreLayout::new(store.clone(), "s3-test".to_owned()); + crab_write::initialize::initialize_repository(&store, &layout, "refs/heads/main") + .await + .unwrap(); + let repository = Repository::new( + RepositoryConfig { + name: "repo".to_owned(), + provider: crab_storage::StorageProviderKind::Local, + bucket: "memory".to_owned(), + prefix: "s3-test".to_owned(), + default_branch: "main".to_owned(), + members: vec![crate::RepositoryMember { + principal: "user".to_owned(), + access: RepositoryAccess::Write, + }], + protected_branches: vec![], + }, + store, + ) + .unwrap(); + ( + repository, + Arc::new(crab_remote_git::RemoteGitRuntime::default()), + CancellationToken::new(), + ) + } + + async fn read( + repository: &Repository, + runtime: Arc, + cancel: &CancellationToken, + path: &str, + ) -> Bytes { + let remote = RemoteGitRepository::open( + repository.store.clone(), + repository.layout.clone(), + repository.identity.clone(), + runtime, + crab_remote_git::RepositoryOptions::default(), + cancel, + ) + .await + .unwrap(); + let operation = remote + .operation(OperationKind::Repository, cancel) + .await + .unwrap(); + let snapshot = remote + .snapshot(&Revision::parse("refs/heads/main").unwrap(), &operation) + .await + .unwrap(); + let blob = snapshot + .read_blob( + &crab_remote_git::GitPath::new(path.as_bytes().to_vec()).unwrap(), + &operation, + ) + .await + .unwrap(); + operation.finish(Ok(())).await.unwrap(); + blob.bytes + } + + async fn tip( + repository: &Repository, + runtime: Arc, + cancel: &CancellationToken, + ) -> ObjectId { + RemoteGitRepository::open( + repository.store.clone(), + repository.layout.clone(), + repository.identity.clone(), + runtime, + crab_remote_git::RepositoryOptions::default(), + cancel, + ) + .await + .unwrap() + .refs() + .entries + .iter() + .find(|reference| reference.name == "refs/heads/main") + .unwrap() + .target + } + + #[tokio::test(flavor = "multi_thread")] + async fn writes_initialize_branch_and_preserve_unrelated_files() { + let (repository, runtime, cancel) = fixture().await; + for (path, body) in [("a.txt", "alpha"), ("nested/b.txt", "bravo")] { + apply( + &repository, + Arc::clone(&runtime), + crab_remote_git::RepositoryOptions::default(), + "refs/heads/main", + &crab_remote_git::GitPath::new(path.as_bytes().to_vec()).unwrap(), + Change::Put { + bytes: Bytes::copy_from_slice(body.as_bytes()), + attributes: Box::new(attributes::PutAttributes::default()), + }, + "user", + &cancel, + ) + .await + .unwrap(); + } + assert_eq!( + read(&repository, Arc::clone(&runtime), &cancel, "a.txt").await, + "alpha" + ); + assert_eq!( + read(&repository, Arc::clone(&runtime), &cancel, "nested/b.txt").await, + "bravo" + ); + apply( + &repository, + Arc::clone(&runtime), + crab_remote_git::RepositoryOptions::default(), + "refs/heads/main", + &crab_remote_git::GitPath::new(b"a.txt".to_vec()).unwrap(), + Change::Delete, + "user", + &cancel, + ) + .await + .unwrap(); + assert_eq!( + read(&repository, Arc::clone(&runtime), &cancel, "nested/b.txt").await, + "bravo" + ); + runtime.shutdown().await; + } + + #[tokio::test(flavor = "multi_thread")] + async fn multipart_completion_token_makes_publication_retry_idempotent() { + let (repository, runtime, cancel) = fixture().await; + let path = crab_remote_git::GitPath::new(b"object.bin".to_vec()).unwrap(); + let change = Change::Put { + bytes: Bytes::from_static(b"multipart content"), + attributes: Box::new(attributes::PutAttributes { + etag_override: Some("multipart-etag-1".to_owned()), + completion_upload_id: Some("upload-id".to_owned()), + ..Default::default() + }), + }; + apply( + &repository, + Arc::clone(&runtime), + crab_remote_git::RepositoryOptions::default(), + "refs/heads/main", + &path, + change.clone(), + "user", + &cancel, + ) + .await + .unwrap(); + let first = tip(&repository, Arc::clone(&runtime), &cancel).await; + apply( + &repository, + Arc::clone(&runtime), + crab_remote_git::RepositoryOptions::default(), + "refs/heads/main", + &path, + change, + "user", + &cancel, + ) + .await + .unwrap(); + let second = tip(&repository, Arc::clone(&runtime), &cancel).await; + assert_eq!(first, second); + runtime.shutdown().await; + } +} diff --git a/crates/crab-s3-gateway/src/namespace.rs b/crates/crab-s3-gateway/src/namespace.rs new file mode 100644 index 000000000..77e5c3af1 --- /dev/null +++ b/crates/crab-s3-gateway/src/namespace.rs @@ -0,0 +1,140 @@ +use percent_encoding::percent_decode_str; + +const MAX_KEY_BYTES: usize = 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ObjectAddress { + pub reference: String, + pub branch: Option, + pub path: crab_remote_git::GitPath, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum NamespaceError { + MissingReference, + MissingObject, + InvalidReference, + InvalidPath, +} + +pub(crate) fn object_address(key: &str) -> Result { + if key.len() > MAX_KEY_BYTES { + return Err(NamespaceError::InvalidPath); + } + let (encoded_ref, path) = key.split_once('/').ok_or(NamespaceError::MissingObject)?; + if encoded_ref.is_empty() { + return Err(NamespaceError::MissingReference); + } + let reference = percent_decode_str(encoded_ref) + .decode_utf8() + .map_err(|_| NamespaceError::InvalidReference)?; + let (reference, branch) = resolve_reference(&reference)?; + validate_path(path)?; + let path = crab_remote_git::GitPath::new(path.as_bytes().to_vec()) + .map_err(|_| NamespaceError::InvalidPath)?; + Ok(ObjectAddress { + reference, + branch, + path, + }) +} + +pub(crate) fn listing_reference(prefix: &str) -> Result, NamespaceError> { + let Some((encoded_ref, path_prefix)) = prefix.split_once('/') else { + if prefix.is_empty() { + return Ok(None); + } + return Err(NamespaceError::MissingReference); + }; + if encoded_ref.is_empty() { + return Err(NamespaceError::MissingReference); + } + let reference = percent_decode_str(encoded_ref) + .decode_utf8() + .map_err(|_| NamespaceError::InvalidReference)?; + let (reference, _) = resolve_reference(&reference)?; + if prefix.len() > MAX_KEY_BYTES + || path_prefix.as_bytes().contains(&0) + || path_prefix.starts_with('/') + || path_prefix.contains("//") + { + return Err(NamespaceError::InvalidPath); + } + Ok(Some((reference, path_prefix.to_owned()))) +} + +fn resolve_reference(value: &str) -> Result<(String, Option), NamespaceError> { + if value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Ok((value.to_ascii_lowercase(), None)); + } + if let Some(name) = value.strip_prefix("refs/heads/") { + validate_branch(name)?; + return Ok((value.to_owned(), Some(value.to_owned()))); + } + if let Some(name) = value.strip_prefix("refs/tags/") { + validate_ref(value, name)?; + return Ok((value.to_owned(), None)); + } + validate_branch(value)?; + let reference = format!("refs/heads/{value}"); + Ok((reference.clone(), Some(reference))) +} + +fn validate_branch(value: &str) -> Result<(), NamespaceError> { + let reference = format!("refs/heads/{value}"); + validate_ref(&reference, value) +} + +fn validate_ref(reference: &str, name: &str) -> Result<(), NamespaceError> { + if name.is_empty() + || name.contains(['~', '^']) + || name.contains("@{") + || crab_git::validate_push_refname(reference).is_err() + { + return Err(NamespaceError::InvalidReference); + } + Ok(()) +} + +fn validate_path(path: &str) -> Result<(), NamespaceError> { + if path.is_empty() + || path.len() > MAX_KEY_BYTES + || path.starts_with('/') + || path.ends_with('/') + || path.contains("//") + || path + .bytes() + .any(|byte| byte == 0 || byte.is_ascii_control()) + || path.split('/').any(|component| { + component.is_empty() + || component.len() > 255 + || matches!(component, "." | "..") + || component.eq_ignore_ascii_case(".git") + }) + { + return Err(NamespaceError::InvalidPath); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encoded_branch_slash_is_decoded_once() { + let address = object_address("feature%2Fdata/a%2Fb").unwrap(); + assert_eq!(address.reference, "refs/heads/feature/data"); + assert_eq!(address.path.as_bytes(), b"a%2Fb"); + } + + #[test] + fn ancestry_and_lossy_paths_are_rejected() { + for key in ["main/a//b", "main/a/../b", "main/.git/config", "main/"] { + assert!(object_address(key).is_err(), "{key}"); + } + for key in ["main~/a", "main^/a", "main@{1}/a"] { + assert!(object_address(key).is_err(), "{key}"); + } + } +} diff --git a/crates/crab-s3-gateway/src/repository.rs b/crates/crab-s3-gateway/src/repository.rs new file mode 100644 index 000000000..35fdd7f96 --- /dev/null +++ b/crates/crab-s3-gateway/src/repository.rs @@ -0,0 +1,89 @@ +use std::{sync::Arc, time::Duration}; + +use crab_remote_git::{RemoteGitRepository, RemoteGitRuntime, RepositoryOptions}; +use tokio_util::sync::CancellationToken; + +use crate::gateway::Repository; + +const MAINTENANCE_TTL: Duration = Duration::from_secs(60); + +pub(crate) async fn open_current( + repository: &Repository, + runtime: Arc, + options: RepositoryOptions, + cancel: &CancellationToken, +) -> crate::Result { + let open = || { + RemoteGitRepository::open( + repository.store.clone(), + repository.layout.clone(), + repository.identity.clone(), + Arc::clone(&runtime), + options, + cancel, + ) + }; + match open().await { + Ok(remote) if remote.refs().is_empty() || remote.commit_graph_available() => { + return Ok(remote); + } + Ok(_) | Err(crab_remote_git::Error::RepositoryIndexing { .. }) => {} + Err(error) => return Err(error.into()), + } + ensure_readable(repository, Arc::clone(&runtime), options, cancel).await?; + open().await.map_err(Into::into) +} + +pub(crate) async fn ensure_readable( + repository: &Repository, + runtime: Arc, + options: RepositoryOptions, + cancel: &CancellationToken, +) -> crate::Result<()> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let store = repository.store.clone(); + let layout = repository.layout.clone(); + let identity = repository.identity.clone(); + let publication_runtime = Arc::clone(&runtime); + let publication_cancel = cancel.clone(); + // The publication future is deeply nested. A task boundary prevents its + // poll stack from accumulating with the request handler's read retry. + tokio::spawn(async move { + crab_write::generation::ensure_readable( + &store, + &layout, + &identity, + publication_runtime, + options, + MAINTENANCE_TTL, + &publication_cancel, + ) + .await + }) + .await??; + let opened = RemoteGitRepository::open( + repository.store.clone(), + repository.layout.clone(), + repository.identity.clone(), + Arc::clone(&runtime), + options, + cancel, + ) + .await; + match opened { + Ok(remote) if remote.refs().is_empty() || remote.commit_graph_available() => { + return Ok(()); + } + Ok(_) | Err(crab_remote_git::Error::RepositoryIndexing { .. }) => {} + Err(error) => return Err(error.into()), + } + if tokio::time::Instant::now() >= deadline { + return Err(crate::Error::ReadinessTimeout); + } + tokio::select! { + () = cancel.cancelled() => return Err(crab_remote_git::Error::Cancelled.into()), + () = tokio::time::sleep(Duration::from_millis(100)) => {} + } + } +} diff --git a/crates/crab-s3-gateway/src/server.rs b/crates/crab-s3-gateway/src/server.rs new file mode 100644 index 000000000..f812307db --- /dev/null +++ b/crates/crab-s3-gateway/src/server.rs @@ -0,0 +1,81 @@ +use hyper_util::{ + rt::{TokioExecutor, TokioIo}, + server::{conn::auto::Builder as ConnectionBuilder, graceful::GracefulShutdown}, +}; +use s3s::service::S3ServiceBuilder; +use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; + +use crate::{Config, Result, gateway::Gateway}; + +/// Serve the configured gateway until SIGINT or SIGTERM. +pub async fn serve(config: Config) -> Result<()> { + config.validate()?; + let listener = TcpListener::bind(config.listen).await?; + let cancellation = CancellationToken::new(); + let gateway = Gateway::new(config, cancellation.clone())?; + let mut builder = S3ServiceBuilder::new(gateway.clone()); + builder.set_auth(gateway.auth()); + let service = builder.build(); + let connections = ConnectionBuilder::new(TokioExecutor::new()); + let graceful = GracefulShutdown::new(); + tracing::info!(address = %listener.local_addr()?, "Crab S3 gateway listening"); + + loop { + let accepted = tokio::select! { + accepted = listener.accept() => Some(accepted?), + () = shutdown_signal() => None, + }; + let Some((socket, peer)) = accepted else { + break; + }; + let connection = connections.serve_connection(TokioIo::new(socket), service.clone()); + let connection = graceful.watch(connection.into_owned()); + tokio::spawn(async move { + if let Err(error) = connection.await { + tracing::warn!(%peer, %error, "S3 connection failed"); + } + }); + } + + cancellation.cancel(); + tokio::select! { + () = graceful.shutdown() => {} + () = tokio::time::sleep(std::time::Duration::from_secs(30)) => { + tracing::warn!("S3 connections exceeded graceful shutdown deadline"); + } + } + gateway.shutdown().await; + Ok(()) +} + +/// Initialize every configured repository without starting the listener. +pub async fn initialize(config: Config) -> Result<()> { + config.validate()?; + let gateway = Gateway::new(config, CancellationToken::new())?; + gateway.initialize_repositories().await?; + gateway.shutdown().await; + Ok(()) +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + let terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()); + match terminate { + Ok(mut terminate) => { + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = terminate.recv() => {} + } + } + Err(_) => { + let _ = tokio::signal::ctrl_c().await; + } + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} diff --git a/crates/crab-write/src/generation.rs b/crates/crab-write/src/generation.rs index 963c45532..f57b45a34 100644 --- a/crates/crab-write/src/generation.rs +++ b/crates/crab-write/src/generation.rs @@ -6,7 +6,8 @@ use std::{ }; use crab_coordination::{ - CoordinationError, GIT_OBJECT_LOCATOR_RESOURCE, PushLock, PushLockAcquireContext, + CoordinationError, GIT_GENERATION_OWNER_RESOURCE, GIT_OBJECT_LOCATOR_RESOURCE, + GcFenceHeartbeat, GcFenceLease, PushLock, PushLockAcquireContext, }; use crab_metadata::{ git_object_locator::{ @@ -32,6 +33,82 @@ use crate::{Result, WriteError, catalog::publish_inventory, finish_after_cleanup const COMMIT_GRAPH_BATCH_SIZE: usize = 512; +struct WriterFence { + lease: GcFenceLease, + heartbeat: GcFenceHeartbeat, +} + +impl WriterFence { + async fn acquire( + store: &Store, + domain: &str, + ttl: Duration, + cancel: &CancellationToken, + ) -> Result { + check_cancelled(cancel)?; + let lease = GcFenceLease::acquire_writer(store.inner(), domain, ttl).await?; + let heartbeat = GcFenceHeartbeat::spawn(&lease, cancel.clone(), ttl / 3); + Ok(Self { lease, heartbeat }) + } + + async fn release(self) -> Result<()> { + self.heartbeat.stop().await; + self.lease.release().await.map_err(Into::into) + } +} + +/// Elect one owner and publish all committed repository state needed by readers. +/// +/// Concurrent callers converge: a caller which loses owner election returns +/// successfully because the owner is responsible for the same canonical state. +/// Every acquired lease and GC fence is released before this function returns. +pub async fn ensure_readable( + store: &Store, + layout: &StoreLayout, + identity: &RepositoryIdentity, + runtime: Arc, + options: RepositoryOptions, + ttl: Duration, + cancel: &CancellationToken, +) -> Result<()> { + let mut context = PushLockAcquireContext::new(Arc::clone(store.inner())); + let mut owner = match context + .try_acquire_internal(layout.repo_prefix(), GIT_GENERATION_OWNER_RESOURCE, ttl) + .await + { + Ok(owner) => owner, + Err(CoordinationError::PushLockHeld { .. }) => return Ok(()), + Err(error) => return Err(error.into()), + }; + let result = crab_coordination::while_renewing(&mut owner, Some(cancel), async { + let global = WriterFence::acquire(store, layout.global_prefix(), ttl, cancel).await?; + let repo = match WriterFence::acquire(store, layout.repo_prefix(), ttl, cancel).await { + Ok(repo) => repo, + Err(error) => { + let _ = global.release().await; + return Err(error); + } + }; + let mut result = async { + let (manifest, _) = manifest_store::read_manifest(store, layout).await?; + let Some(manifest) = make_readable(store, layout, ttl, manifest.pusher, cancel).await? + else { + return Ok(()); + }; + maintain_commit_graph(store, layout, &manifest, identity, runtime, options, cancel) + .await + .map(drop) + } + .await; + for fence in [repo, global] { + result = result.and(fence.release().await); + } + result + }) + .await; + result.and(owner.release().await.map_err(Into::into)) +} + /// Complete the read path for already committed refs using their verified visibility evidence. /// /// The caller retains generation-owner election and global/repository GC writer From d00b6c961769688a708bfc7fddadf0f80a001657 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 8 Sep 2026 06:55:14 -0700 Subject: [PATCH 2/3] ci: register S3 gateway architecture policy --- crab/scripts/check-architecture-gates.py | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/crab/scripts/check-architecture-gates.py b/crab/scripts/check-architecture-gates.py index 705766234..629a4239e 100644 --- a/crab/scripts/check-architecture-gates.py +++ b/crab/scripts/check-architecture-gates.py @@ -1754,6 +1754,7 @@ "crab-lfs", "crab-metadata", "crab-read", + "crab-s3-gateway", "crab-storage", "crab-types", "crab-workflow", @@ -1764,11 +1765,17 @@ "crab-auth-server": {"crab-auth-receive", "crab-auth-view"}, "crab-cache-server": {"crab-cache-server"}, } -SERVER_PACKAGES = {"crab-auth-server", "crab-cache-server", "crab-http-server"} +SERVER_PACKAGES = { + "crab-auth-server", + "crab-cache-server", + "crab-http-server", + "crab-s3-gateway", +} ALLOWED_SERVER_DEV_FIXTURES = { "crab-http-server": set(), "crab-auth-server": set(), "crab-cache-server": {"crab", "crab-cache-store"}, + "crab-s3-gateway": set(), } WORKSPACE_DEPENDENCY_POLICY = { "crab-write": {"normal": {"crab-coordination", "crab-types", "crab-git", "crab-metadata", "crab-remote-git", "crab-storage", "crab-xet"}}, @@ -1857,6 +1864,21 @@ "crab-xet", }, }, + # The S3 gateway is a product composition boundary for protocol, repository, + # read, write, coordination, LFS, cache, and storage behavior. + "crab-s3-gateway": { + "normal": { + "crab-cache-store", + "crab-coordination", + "crab-git", + "crab-lfs", + "crab-metadata", + "crab-read", + "crab-remote-git", + "crab-storage", + "crab-write", + }, + }, "crab-remote-git": { "normal": {"crab-git", "crab-metadata", "crab-storage", "crab-xet"}, "dev": {"crab-metadata"}, @@ -1898,6 +1920,7 @@ "crab-metadata": "crates/crab-metadata", "crab-read": "crates/crab-read", "crab-remote-git": "crates/crab-remote-git", + "crab-s3-gateway": "crates/crab-s3-gateway", "crab-staging": "crates/crab-staging", "crab-storage": "crates/crab-storage", "crab-types": "crates/crab-types", From 1b549ed5acdb7e808fbb1019085c2a7636f0b814 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 8 Sep 2026 09:15:12 -0700 Subject: [PATCH 3/3] feat: stream large S3 objects --- Cargo.lock | 2 + crab/docs/architecture/s3-gateway-contract.md | 24 +- crates/crab-s3-gateway/Cargo.toml | 6 +- crates/crab-s3-gateway/README.md | 9 +- crates/crab-s3-gateway/src/attributes.rs | 6 +- crates/crab-s3-gateway/src/content.rs | 202 ++++++++ crates/crab-s3-gateway/src/gateway.rs | 481 +++++++++++++----- crates/crab-s3-gateway/src/lib.rs | 1 + crates/crab-s3-gateway/src/multipart.rs | 72 ++- crates/crab-s3-gateway/src/mutation.rs | 9 +- 10 files changed, 638 insertions(+), 174 deletions(-) create mode 100644 crates/crab-s3-gateway/src/content.rs diff --git a/Cargo.lock b/Cargo.lock index fba4232e5..5e9e78d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2428,6 +2428,8 @@ dependencies = [ "gix-hash", "gix-object", "http 1.4.1", + "http-body 1.0.1", + "http-body-util", "hyper-util", "md-5", "object_store", diff --git a/crab/docs/architecture/s3-gateway-contract.md b/crab/docs/architecture/s3-gateway-contract.md index 2e1bc8fcf..035caf33e 100644 --- a/crab/docs/architecture/s3-gateway-contract.md +++ b/crab/docs/architecture/s3-gateway-contract.md @@ -144,10 +144,10 @@ Supported operations: | `ListBuckets`, `HeadBucket` | Authorized logical repositories only; deterministic order | | `GetObject`, `HeadObject` | metadata, response overrides, RFC dates, ETag/date conditions, one byte range including open and suffix forms | | `ListObjects`, `ListObjectsV2` | prefix, delimiter `/`, marker/start-after, max keys, reusable keys, common prefixes | -| `PutObject` | body up to 256 MiB, `Content-MD5`, SigV4 payload hash, CRC32/CRC32C/CRC64NVME/SHA1/SHA256 checksums, metadata and standard content headers | +| `PutObject` | body up to 5 GiB, `Content-MD5`, SigV4 payload hash, CRC32/CRC32C/CRC64NVME/SHA1/SHA256 checksums, metadata and standard content headers | | `DeleteObject`, `DeleteObjects` | S3 missing-key success, per-key authorization/results, quiet mode, and at most 1000 XML entries | | `CopyObject` | pinned source, source conditions/range where defined, `COPY`/`REPLACE`, separately authorized destination | -| Multipart create/upload/copy/list/abort/complete | durable opaque sessions, part replacement, ordered selection, 10,000-part and 256 MiB completed-object limits, restart and multi-instance retry | +| Multipart create/upload/copy/list/abort/complete | durable opaque sessions, part replacement, ordered selection, 10,000 parts, 5 GiB per part, 50 TB completed objects, restart and multi-instance retry | Modeled unsupported request headers and query parameters are rejected rather than ignored. Multi-range GET returns `InvalidRange`. `versionId` returns @@ -204,11 +204,10 @@ persists the terminal state, then synchronously removes part objects. Completion persists its terminal outcome before best-effort part cleanup; a cleanup failure does not erase the completed outcome. -Limits: 10,000 parts per upload, 256 MiB per logical object read, single PUT, -part, and final multipart object, and 1000 results per multipart listing page. -Unknown-length streams are counted as they arrive. Larger objects require later -streaming reader and content-writer implementations; the gateway returns -`EntityTooLarge` instead of risking unbounded memory or a non-durable upload. +Limits follow the S3 general-purpose bucket contract: 5 GiB per single PUT or +multipart part, 10,000 parts per upload, 50 TB per completed multipart object, +and 1000 results per multipart listing page. Unknown-length streams are counted +as they arrive. Requests beyond an operation's S3 limit return `EntityTooLarge`. ## Error and response contract @@ -232,10 +231,13 @@ boundary without credentials or request bodies. | Admission capacity limit | `SlowDown` | | Corrupt/unavailable committed data | `InternalError` | -The initial implementation materializes each bounded object before starting a -response, so backend failures still return an S3 error rather than truncating a -successful response stream. Successful writes are returned only after their -committed outcome is durable and read-ready. +Request bodies and multipart completion are streamed through bounded memory to +temporary storage while checksums are computed. Objects above the inline Git +threshold are stored through Crab's verified LFS content path; the committed Git +blob is the canonical LFS pointer and the S3 attribute record retains the logical +size and ETag. GET streams LFS content directly and reconstructs Crab pointers to +temporary storage before opening the response. Successful writes are returned +only after their committed outcome is durable and read-ready. ## Repository extension API diff --git a/crates/crab-s3-gateway/Cargo.toml b/crates/crab-s3-gateway/Cargo.toml index d81d87311..e473333f8 100644 --- a/crates/crab-s3-gateway/Cargo.toml +++ b/crates/crab-s3-gateway/Cargo.toml @@ -27,6 +27,8 @@ futures-util = { workspace = true } gix-hash = { workspace = true, features = ["sha1"] } gix-object = { workspace = true } http = "1" +http-body = "1" +http-body-util = "0.1" hyper-util = { version = "0.1", features = ["server-auto", "server-graceful", "http1", "http2", "tokio"] } md-5 = { workspace = true } object_store = { workspace = true } @@ -39,8 +41,8 @@ sha2 = "0.10" tempfile = { workspace = true } thiserror = { workspace = true } time = { workspace = true } -tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } -tokio-util = { workspace = true, features = ["rt"] } +tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio-util = { workspace = true, features = ["io", "rt"] } toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/crates/crab-s3-gateway/README.md b/crates/crab-s3-gateway/README.md index bdbb961c4..5955cd160 100644 --- a/crates/crab-s3-gateway/README.md +++ b/crates/crab-s3-gateway/README.md @@ -14,10 +14,11 @@ The initial client-compatible surface includes bucket listing/head, object GET/HEAD/PUT/DELETE/COPY, V1/V2 object listing, multi-delete, and durable multipart create/upload/copy/list/abort/complete. GET/HEAD support conditions and a single byte range; PUT validates Content-MD5 and the standard S3 checksum -headers. Path-style addressing is required. Object and completed multipart -payloads are limited to 256 MiB while the canonical streaming writer remains -future work. The complete frozen surface and deliberate exclusions are in the -protocol contract linked below. +headers. Path-style addressing is required. Single PUTs and multipart parts +support up to 5 GiB, and multipart completion supports S3's 50 TB object limit. +Large payloads use bounded-memory spooling and Crab's verified LFS content path. +The complete frozen surface and deliberate exclusions are in the protocol +contract linked below. ## Build and run diff --git a/crates/crab-s3-gateway/src/attributes.rs b/crates/crab-s3-gateway/src/attributes.rs index 15a20eb03..2c0695534 100644 --- a/crates/crab-s3-gateway/src/attributes.rs +++ b/crates/crab-s3-gateway/src/attributes.rs @@ -16,6 +16,8 @@ pub(crate) struct PutAttributes { pub(crate) etag_override: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) completion_upload_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) logical_size: Option, pub(crate) cache_control: Option, pub(crate) content_disposition: Option, pub(crate) content_encoding: Option, @@ -74,9 +76,9 @@ impl ObjectAttributes { } } - pub(crate) fn matches_pending(&self, pending: &PutAttributes, etag: &str, size: usize) -> bool { + pub(crate) fn matches_pending(&self, pending: &PutAttributes, etag: &str, size: u64) -> bool { self.etag == etag - && usize::try_from(self.size).ok() == Some(size) + && self.size == size && self.completion_upload_id == pending.completion_upload_id && self.cache_control == pending.cache_control && self.content_disposition == pending.content_disposition diff --git a/crates/crab-s3-gateway/src/content.rs b/crates/crab-s3-gateway/src/content.rs new file mode 100644 index 000000000..772e37aab --- /dev/null +++ b/crates/crab-s3-gateway/src/content.rs @@ -0,0 +1,202 @@ +use std::path::{Path, PathBuf}; + +use bytes::Bytes; +use futures_util::StreamExt as _; +use s3s::dto::StreamingBlob; +use tokio::io::{AsyncWriteExt as _, BufWriter}; + +pub(crate) const MAX_PUT_OBJECT_BYTES: u64 = 5 * 1024 * 1024 * 1024; +pub(crate) const MAX_MULTIPART_OBJECT_BYTES: u64 = 50_000_000_000_000; +pub(crate) const MAX_MULTIPART_PART_BYTES: u64 = 5 * 1024 * 1024 * 1024; +pub(crate) const INLINE_GIT_BLOB_BYTES: u64 = 64 * 1024 * 1024; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("request body exceeds the operation limit")] + TooLarge, + #[error("request body length does not match Content-Length")] + Incomplete, + #[error("request body stream failed")] + Body(#[source] s3s::StdError), + #[error("content spool I/O failed")] + Io(#[from] std::io::Error), +} + +pub(crate) struct Digests { + pub(crate) md5: [u8; 16], + pub(crate) sha1: [u8; 20], + pub(crate) sha256: [u8; 32], + pub(crate) crc32: u32, + pub(crate) crc32c: u32, + pub(crate) crc64nvme: u64, + pub(crate) blake3: [u8; 32], +} + +pub(crate) struct Spool { + _directory: tempfile::TempDir, + path: PathBuf, + pub(crate) size: u64, + pub(crate) digests: Digests, +} + +impl Spool { + pub(crate) fn path(&self) -> &Path { + &self.path + } + + pub(crate) async fn bytes(&self) -> Result { + Ok(Bytes::from(tokio::fs::read(&self.path).await?)) + } +} + +pub(crate) struct SpoolWriter { + directory: tempfile::TempDir, + path: PathBuf, + file: BufWriter, + size: u64, + md5: md5::Md5, + sha1: sha1::Sha1, + sha256: sha2::Sha256, + crc32: crc_fast::Digest, + crc32c: crc_fast::Digest, + crc64nvme: crc_fast::Digest, + blake3: blake3::Hasher, +} + +impl SpoolWriter { + pub(crate) async fn new() -> Result { + let directory = tempfile::tempdir()?; + let path = directory.path().join("content"); + let file = BufWriter::new(tokio::fs::File::create(&path).await?); + Ok(Self { + directory, + path, + file, + size: 0, + md5: md5::Md5::default(), + sha1: sha1::Sha1::default(), + sha256: sha2::Sha256::default(), + crc32: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc), + crc32c: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32Iscsi), + crc64nvme: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc64Nvme), + blake3: blake3::Hasher::new(), + }) + } + + pub(crate) async fn write(&mut self, bytes: &[u8], max_bytes: u64) -> Result<(), Error> { + let chunk_size = u64::try_from(bytes.len()).map_err(|_| Error::TooLarge)?; + self.size = self.size.checked_add(chunk_size).ok_or(Error::TooLarge)?; + if self.size > max_bytes { + return Err(Error::TooLarge); + } + self.file.write_all(bytes).await?; + md5::Digest::update(&mut self.md5, bytes); + sha1::Digest::update(&mut self.sha1, bytes); + sha2::Digest::update(&mut self.sha256, bytes); + self.crc32.update(bytes); + self.crc32c.update(bytes); + self.crc64nvme.update(bytes); + self.blake3.update(bytes); + Ok(()) + } + + pub(crate) async fn finish(mut self) -> Result { + self.file.flush().await?; + drop(self.file); + Ok(Spool { + _directory: self.directory, + path: self.path, + size: self.size, + digests: Digests { + md5: md5::Digest::finalize(self.md5).into(), + sha1: sha1::Digest::finalize(self.sha1).into(), + sha256: sha2::Digest::finalize(self.sha256).into(), + crc32: u32::try_from(self.crc32.finalize()).map_err(|_| Error::TooLarge)?, + crc32c: u32::try_from(self.crc32c.finalize()).map_err(|_| Error::TooLarge)?, + crc64nvme: self.crc64nvme.finalize(), + blake3: *self.blake3.finalize().as_bytes(), + }, + }) + } +} + +pub(crate) async fn spool_body( + body: Option, + declared: Option, + max_bytes: u64, +) -> Result { + let declared = declared + .map(|length| u64::try_from(length).map_err(|_| Error::Incomplete)) + .transpose()?; + if declared.is_some_and(|length| length > max_bytes) { + return Err(Error::TooLarge); + } + let mut writer = SpoolWriter::new().await?; + if let Some(mut body) = body { + while let Some(chunk) = body.next().await { + writer + .write(&chunk.map_err(Error::Body)?, max_bytes) + .await?; + } + } + if declared.is_some_and(|length| length != writer.size) { + return Err(Error::Incomplete); + } + writer.finish().await +} + +pub(crate) fn md5_hex(digest: &[u8; 16]) -> String { + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn operation_limits_match_s3_general_purpose_buckets() { + assert_eq!( + ( + MAX_PUT_OBJECT_BYTES, + MAX_MULTIPART_PART_BYTES, + MAX_MULTIPART_OBJECT_BYTES, + ), + ( + 5 * 1024 * 1024 * 1024, + 5 * 1024 * 1024 * 1024, + 50_000_000_000_000, + ) + ); + } + + #[tokio::test] + async fn spool_writer_hashes_chunks_and_enforces_limit() { + use sha1::Digest as _; + + let mut writer = SpoolWriter::new().await.unwrap(); + writer.write(b"streamed ", 16).await.unwrap(); + writer.write(b"content", 16).await.unwrap(); + let spool = writer.finish().await.unwrap(); + let expected_md5: [u8; 16] = ::digest(b"streamed content").into(); + let expected_sha1: [u8; 20] = sha1::Sha1::digest(b"streamed content").into(); + let expected_sha256: [u8; 32] = sha2::Sha256::digest(b"streamed content").into(); + assert_eq!( + ( + spool.size, + spool.digests.md5, + spool.digests.sha1, + spool.digests.sha256, + ), + (16, expected_md5, expected_sha1, expected_sha256) + ); + } + + #[tokio::test] + async fn spool_writer_rejects_content_over_the_operation_limit() { + let mut writer = SpoolWriter::new().await.unwrap(); + assert!(matches!( + writer.write(b"too large", 8).await, + Err(Error::TooLarge) + )); + } +} diff --git a/crates/crab-s3-gateway/src/gateway.rs b/crates/crab-s3-gateway/src/gateway.rs index ff8df6912..8cd60c7e6 100644 --- a/crates/crab-s3-gateway/src/gateway.rs +++ b/crates/crab-s3-gateway/src/gateway.rs @@ -1,11 +1,12 @@ use std::{ collections::BTreeMap, + pin::Pin, sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; use base64::Engine as _; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use crab_cache_store::{CacheConfig, CachingStore}; use crab_git::pointer_detect::PointerKind; use crab_remote_git::{ @@ -13,14 +14,13 @@ use crab_remote_git::{ RepositoryIdentity, RepositoryOptions, Revision, }; use crab_storage::{StorageProviderKind, Store, StoreLayout, build_static_env_store}; +use futures_util::StreamExt as _; use s3s::{S3, S3Request, S3Response, S3Result, dto::*, s3_error}; use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; use crate::{Config, RepositoryAccess, RepositoryConfig, auth::GatewayAuth, mutation, namespace}; -const MAX_SINGLE_OBJECT_BYTES: usize = 256 * 1024 * 1024; - pub(crate) struct Repository { pub(crate) config: RepositoryConfig, pub(crate) store: Store, @@ -74,13 +74,115 @@ pub(crate) struct Gateway { } struct ReadObject { - bytes: Bytes, + content: ReadContent, size: u64, etag: String, modified: Timestamp, attributes: Option, } +#[derive(Clone)] +enum ReadContent { + Ordinary(Bytes), + CrabPointer(Bytes), + LfsPointer(crab_git::LfsPointer), +} + +type ContentStream = + Pin> + Send + 'static>>; + +impl ReadContent { + async fn stream( + self, + repository: &Repository, + range: std::ops::Range, + ) -> S3Result { + use futures_util::{StreamExt as _, TryStreamExt as _}; + use tokio::io::{AsyncReadExt as _, AsyncSeekExt as _}; + + match self { + Self::Ordinary(bytes) => { + let start = usize::try_from(range.start).map_err(|_| s3_error!(InvalidRange))?; + let end = usize::try_from(range.end).map_err(|_| s3_error!(InvalidRange))?; + Ok(Box::pin(futures_util::stream::once(async move { + Ok(bytes.slice(start..end)) + }))) + } + Self::LfsPointer(pointer) => { + let (_, actual, stream) = repository + .lfs + .get_stream(&pointer.oid, pointer.size, Some(range.clone())) + .await + .map_err(|error| gateway_error(error.into()))?; + if actual != range { + return Err(s3_error!(InvalidObjectState)); + } + Ok(Box::pin( + stream.map_err(|error| Box::new(error) as s3s::StdError), + )) + } + Self::CrabPointer(pointer_bytes) => { + let PointerKind::Crab(pointer) = crab_git::classify(&pointer_bytes) else { + return Err(s3_error!(InvalidObjectState)); + }; + let directory = tempfile::tempdir().map_err(|error| gateway_error(error.into()))?; + let path = directory.path().join("content"); + repository + .hydrator + .reconstruct_to_path(&pointer, &path) + .await + .map_err(|error| gateway_error(error.into()))?; + let mut file = tokio::fs::File::open(path) + .await + .map_err(|error| gateway_error(error.into()))?; + file.seek(std::io::SeekFrom::Start(range.start)) + .await + .map_err(|error| gateway_error(error.into()))?; + let reader = tokio_util::io::ReaderStream::new(file.take(range.end - range.start)); + let stream = futures_util::stream::try_unfold( + (reader, directory), + |(mut reader, directory)| async move { + match reader.next().await { + Some(Ok(bytes)) => Ok(Some((bytes, (reader, directory)))), + Some(Err(error)) => Err(error), + None => Ok(None), + } + }, + ) + .map_err(|error| Box::new(error) as s3s::StdError); + Ok(Box::pin(stream)) + } + } + } + + async fn spool( + self, + repository: &Repository, + range: std::ops::Range, + max_bytes: u64, + ) -> S3Result { + use futures_util::StreamExt as _; + + let mut stream = self.stream(repository, range).await?; + let mut writer = crate::content::SpoolWriter::new() + .await + .map_err(content_error)?; + while let Some(chunk) = stream.next().await { + writer + .write( + &chunk.map_err(|error| { + tracing::warn!(%error, "S3 source object stream failed"); + s3_error!(InternalError) + })?, + max_bytes, + ) + .await + .map_err(content_error)?; + } + writer.finish().await.map_err(content_error) + } +} + impl Gateway { pub(crate) fn new(config: Config, cancellation: CancellationToken) -> crate::Result { let auth = GatewayAuth::load(&config)?; @@ -227,14 +329,19 @@ impl Gateway { .and_then(|value| i64::try_from(value.modified_seconds).ok()) .unwrap_or(commit.committer.seconds); let modified = timestamp(modified_seconds)?; - let bytes = materialize_blob(repository, blob).await?; - let size = u64::try_from(bytes.len()).map_err(|_| s3_error!(InternalError))?; - let etag = attributes - .as_ref() - .map(|value| value.etag.clone()) - .unwrap_or_else(|| md5_hex(&bytes)); + let (content, size) = classify_blob(blob)?; + let etag = match attributes.as_ref() { + Some(value) => value.etag.clone(), + None => match &content { + ReadContent::Ordinary(bytes) => md5_hex(bytes), + ReadContent::CrabPointer(_) | ReadContent::LfsPointer(_) => { + let spool = content.clone().spool(repository, 0..size, u64::MAX).await?; + crate::content::md5_hex(&spool.digests.md5) + } + }, + }; Ok(ReadObject { - bytes, + content, size, etag, modified, @@ -278,36 +385,26 @@ fn build_store(entry: &RepositoryConfig) -> crate::Result { Ok(build_static_env_store(&entry.bucket, entry.provider)?) } -async fn materialize_blob(repository: &Repository, blob: crab_remote_git::Blob) -> S3Result { +fn classify_blob(blob: crab_remote_git::Blob) -> S3Result<(ReadContent, u64)> { let physical_size = u64::try_from(blob.bytes.len()).map_err(|_| s3_error!(EntityTooLarge))?; let logical_size = blob.metadata.logical_size.unwrap_or(physical_size); - if logical_size > MAX_SINGLE_OBJECT_BYTES as u64 { + if logical_size > crate::content::MAX_MULTIPART_OBJECT_BYTES { return Err(s3_error!(EntityTooLarge)); } - match blob.metadata.classification { - ContentClassification::OrdinaryGit => Ok(blob.bytes), - ContentClassification::CrabPointer => Ok(Bytes::from( - repository - .hydrator - .reconstruct_from_pointer(&blob.bytes) - .await - .map_err(|error| gateway_error(error.into()))?, - )), + let content = match blob.metadata.classification { + ContentClassification::OrdinaryGit => ReadContent::Ordinary(blob.bytes), + ContentClassification::CrabPointer => ReadContent::CrabPointer(blob.bytes), ContentClassification::LfsPointer => { let PointerKind::Lfs(pointer) = crab_git::classify(&blob.bytes) else { return Err(s3_error!(InvalidObjectState)); }; - let bytes = repository - .lfs - .verify(&pointer.oid) - .await - .map_err(|error| gateway_error(error.into()))?; - if bytes.len() as u64 != pointer.size { + if pointer.size != logical_size { return Err(s3_error!(InvalidObjectState)); } - Ok(bytes) + ReadContent::LfsPointer(pointer) } - } + }; + Ok((content, logical_size)) } fn provider_name(provider: StorageProviderKind) -> &'static str { @@ -319,6 +416,36 @@ fn provider_name(provider: StorageProviderKind) -> &'static str { } } +async fn mutation_bytes(repository: &Repository, spool: &crate::content::Spool) -> S3Result { + mutation_bytes_with_inline_limit(repository, spool, crate::content::INLINE_GIT_BLOB_BYTES).await +} + +async fn mutation_bytes_with_inline_limit( + repository: &Repository, + spool: &crate::content::Spool, + inline_limit: u64, +) -> S3Result { + if spool.size <= inline_limit { + return spool.bytes().await.map_err(content_error); + } + // The LFS object is content-addressed and uploaded before its pointer commit. + // Crab's GC grace period protects this brief publication window and cleans an + // orphan if the later ref mutation fails. + repository + .lfs + .put_stream_with_size(&spool.digests.sha256, Some(spool.size), spool.path()) + .await + .map_err(|error| gateway_error(error.into()))?; + Ok(Bytes::from( + crab_git::LfsPointer { + oid: spool.digests.sha256, + size: spool.size, + extensions: Vec::new(), + } + .serialize(), + )) +} + #[async_trait::async_trait] impl S3 for Gateway { async fn list_buckets( @@ -421,26 +548,21 @@ impl S3 for Gateway { .as_ref() .map(|range| range.check(object.size)) .transpose()?; - let (bytes, content_range) = match checked { + let (range, content_range) = match checked { Some(range) => { - let start = usize::try_from(range.start).map_err(|_| s3_error!(InvalidRange))?; - let end = usize::try_from(range.end).map_err(|_| s3_error!(InvalidRange))?; - ( - object.bytes.slice(start..end), - Some(format!( - "bytes {}-{}/{}", - range.start, - range.end - 1, - object.size - )), - ) + let header = format!("bytes {}-{}/{}", range.start, range.end - 1, object.size); + (range.start..range.end, Some(header)) } - None => (object.bytes, None), + None => (0..object.size, None), }; - let content_length = i64::try_from(bytes.len()).map_err(|_| s3_error!(InternalError))?; + let content_length = + i64::try_from(range.end - range.start).map_err(|_| s3_error!(InternalError))?; + let body = object.content.stream(repository, range).await?; + let body = + http_body_util::StreamBody::new(body.map(|result| result.map(http_body::Frame::data))); let output = GetObjectOutput { accept_ranges: Some("bytes".to_owned()), - body: Some(StreamingBlob::from(s3s::Body::from(bytes))), + body: Some(StreamingBlob::from(s3s::Body::http_body_unsync(body))), content_length: Some(content_length), content_range, content_type: req @@ -585,10 +707,18 @@ impl S3 for Gateway { let content_md5 = req.input.content_md5.clone(); let mut checksums = RequestChecksums::from(&req.input); let trailing_headers = req.trailing_headers.clone(); - let body = read_body(req.input.body, content_length).await?; + let spool = crate::content::spool_body( + req.input.body, + content_length, + crate::content::MAX_PUT_OBJECT_BYTES, + ) + .await + .map_err(content_error)?; checksums.merge_trailers(trailing_headers.as_ref())?; - verify_content_md5(&body, content_md5.as_deref())?; - checksums.verify(&body)?; + verify_content_md5(&spool.digests.md5, content_md5.as_deref())?; + checksums.verify(&spool.digests)?; + let etag = crate::content::md5_hex(&spool.digests.md5); + let bytes = mutation_bytes(repository, &spool).await?; let outcome = mutation::apply( repository, Arc::clone(&self.runtime), @@ -599,10 +729,11 @@ impl S3 for Gateway { .ok_or_else(|| s3_error!(MethodNotAllowed))?, &address.path, mutation::Change::Put { - bytes: body, + bytes, attributes: Box::new(crate::attributes::PutAttributes { - etag_override: None, + etag_override: Some(etag), completion_upload_id: None, + logical_size: Some(spool.size), cache_control: req.input.cache_control, content_disposition: req.input.content_disposition, content_encoding: req.input.content_encoding, @@ -788,9 +919,21 @@ impl S3 for Gateway { &source_object.etag, &source_object.modified, )?; + let source_size = source_object.size; + if source_size > crate::content::MAX_PUT_OBJECT_BYTES { + return Err(s3_error!(EntityTooLarge)); + } + let spool = source_object + .content + .spool( + source_repository, + 0..source_size, + crate::content::MAX_PUT_OBJECT_BYTES, + ) + .await?; let (repository, address, principal) = self.writable_address(&req, &req.input.bucket, &req.input.key)?; - let attributes = if req + let mut attributes = if req .input .metadata_directive .as_ref() @@ -799,6 +942,7 @@ impl S3 for Gateway { crate::attributes::PutAttributes { etag_override: None, completion_upload_id: None, + logical_size: None, cache_control: req.input.cache_control, content_disposition: req.input.content_disposition, content_encoding: req.input.content_encoding, @@ -814,6 +958,9 @@ impl S3 for Gateway { .map(stored_to_pending) .unwrap_or_default() }; + attributes.etag_override = Some(crate::content::md5_hex(&spool.digests.md5)); + attributes.logical_size = Some(spool.size); + let bytes = mutation_bytes(repository, &spool).await?; let outcome = mutation::apply( repository, Arc::clone(&self.runtime), @@ -824,7 +971,7 @@ impl S3 for Gateway { .ok_or_else(|| s3_error!(MethodNotAllowed))?, &address.path, mutation::Change::Put { - bytes: source_object.bytes, + bytes, attributes: Box::new(attributes), }, &principal, @@ -877,6 +1024,7 @@ impl S3 for Gateway { crate::attributes::PutAttributes { etag_override: None, completion_upload_id: None, + logical_size: None, cache_control: req.input.cache_control, content_disposition: req.input.content_disposition, content_encoding: req.input.content_encoding, @@ -920,16 +1068,23 @@ impl S3 for Gateway { .map_err(multipart_error)?; let content_length = req.input.content_length; let content_md5 = req.input.content_md5.clone(); - let body = read_body(req.input.body, content_length).await?; - verify_content_md5(&body, content_md5.as_deref())?; - let etag = md5_hex(&body); + let spool = crate::content::spool_body( + req.input.body, + content_length, + crate::content::MAX_MULTIPART_PART_BYTES, + ) + .await + .map_err(content_error)?; + verify_content_md5(&spool.digests.md5, content_md5.as_deref())?; + let etag = crate::content::md5_hex(&spool.digests.md5); crate::multipart::register_part( repository, loaded, req.input.part_number, - body, + &spool, etag.clone(), now_seconds()?, + &self.cancellation, ) .await .map_err(multipart_error)?; @@ -970,16 +1125,25 @@ impl S3 for Gateway { &source.etag, &source.modified, )?; - let bytes = match req.input.copy_source_range.as_deref() { + let range = match req.input.copy_source_range.as_deref() { Some(value) => { let range = Range::parse(value).map_err(|_| s3_error!(InvalidArgument))?; let range = range.check(source.size)?; - let start = usize::try_from(range.start).map_err(|_| s3_error!(InvalidRange))?; - let end = usize::try_from(range.end).map_err(|_| s3_error!(InvalidRange))?; - source.bytes.slice(start..end) + range.start..range.end } - None => source.bytes, + None => 0..source.size, }; + if range.end - range.start > crate::content::MAX_MULTIPART_PART_BYTES { + return Err(s3_error!(EntityTooLarge)); + } + let spool = source + .content + .spool( + source_repository, + range, + crate::content::MAX_MULTIPART_PART_BYTES, + ) + .await?; let repository = self.repository(&req, &req.input.bucket, RepositoryAccess::Write)?; let principal = self.principal(&req)?.to_owned(); let loaded = crate::multipart::load(repository, &req.input.upload_id) @@ -992,14 +1156,15 @@ impl S3 for Gateway { &principal, ) .map_err(multipart_error)?; - let etag = md5_hex(&bytes); + let etag = crate::content::md5_hex(&spool.digests.md5); crate::multipart::register_part( repository, loaded, req.input.part_number, - bytes, + &spool, etag.clone(), now_seconds()?, + &self.cancellation, ) .await .map_err(multipart_error)?; @@ -1065,22 +1230,50 @@ impl S3 for Gateway { repository, loaded, &selected, - MAX_SINGLE_OBJECT_BYTES as u64, + crate::content::MAX_MULTIPART_OBJECT_BYTES, ) .await .map_err(multipart_error)?; - let mut body = BytesMut::new(); + let mut writer = crate::content::SpoolWriter::new() + .await + .map_err(content_error)?; for part in &parts { - let bytes = crate::multipart::part_bytes(repository, part) + use md5::Digest as _; + + let mut stream = crate::multipart::part_stream(repository, part) .await .map_err(multipart_error)?; - body.extend_from_slice(&bytes); + let mut digest = md5::Md5::new(); + let mut size = 0_u64; + while let Some(chunk) = stream.next().await { + let chunk = chunk + .map_err(|error| multipart_error(crate::multipart::Error::Storage(error)))?; + size = size + .checked_add(chunk.len() as u64) + .ok_or_else(|| s3_error!(EntityTooLarge))?; + digest.update(&chunk); + writer + .write(&chunk, crate::content::MAX_MULTIPART_OBJECT_BYTES) + .await + .map_err(content_error)?; + } + let actual = digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + if size != part.size || actual != part.etag { + return Err(s3_error!(InvalidPart)); + } } + let spool = writer.finish().await.map_err(content_error)?; let etag = multipart_etag(&parts)?; let mut attributes = session.attributes.clone(); attributes.etag_override = Some(etag.clone()); attributes.completion_upload_id = Some(session.id.clone()); + attributes.logical_size = Some(spool.size); let address = namespace::object_address(&session.key).map_err(namespace_error)?; + let bytes = mutation_bytes(repository, &spool).await?; mutation::apply( repository, Arc::clone(&self.runtime), @@ -1088,7 +1281,7 @@ impl S3 for Gateway { &session.branch, &address.path, mutation::Change::Put { - bytes: body.freeze(), + bytes, attributes: Box::new(attributes), }, &principal, @@ -1436,7 +1629,6 @@ impl S3 for Gateway { .read_blob(&entry.path, &operation) .await .map_err(remote_error)?; - let logical_size = blob.metadata.logical_size; let attributes = attribute_manifest.object(path, entry.oid); let modified = match attributes { Some(attributes) => timestamp( @@ -1445,11 +1637,15 @@ impl S3 for Gateway { )?, None => commit_modified.clone(), }; + let (content, logical_size) = classify_blob(blob)?; let etag = match attributes { Some(attributes) => attributes.etag.clone(), - None => md5_hex(&materialize_blob(repository, blob).await?), + None => { + let spool = content.spool(repository, 0..logical_size, u64::MAX).await?; + crate::content::md5_hex(&spool.digests.md5) + } }; - keys.push((key, etag, logical_size, modified)); + keys.push((key, etag, Some(logical_size), modified)); } keys.sort_by(|left, right| left.0.cmp(&right.0)); let max_keys = req.input.max_keys.unwrap_or(1000); @@ -1552,49 +1748,14 @@ impl S3 for Gateway { } } -async fn read_body(body: Option, declared: Option) -> S3Result { - use futures_util::StreamExt as _; - - let declared = declared - .map(|length| usize::try_from(length).map_err(|_| s3_error!(InvalidRequest))) - .transpose()?; - if declared.is_some_and(|length| length > MAX_SINGLE_OBJECT_BYTES) { - return Err(s3_error!(EntityTooLarge)); - } - let mut bytes = BytesMut::with_capacity(declared.unwrap_or(0).min(MAX_SINGLE_OBJECT_BYTES)); - let Some(mut body) = body else { - if declared.unwrap_or(0) != 0 { - return Err(s3_error!(IncompleteBody)); - } - return Ok(bytes.freeze()); - }; - while let Some(chunk) = body.next().await { - let chunk = chunk.map_err(|error| { - tracing::warn!(%error, "S3 request body failed"); - s3_error!(IncompleteBody) - })?; - if bytes.len().saturating_add(chunk.len()) > MAX_SINGLE_OBJECT_BYTES { - return Err(s3_error!(EntityTooLarge)); - } - bytes.extend_from_slice(&chunk); - } - if declared.is_some_and(|length| length != bytes.len()) { - return Err(s3_error!(IncompleteBody)); - } - Ok(bytes.freeze()) -} - -fn verify_content_md5(bytes: &[u8], expected: Option<&str>) -> S3Result<()> { - use md5::Digest as _; - +fn verify_content_md5(actual: &[u8; 16], expected: Option<&str>) -> S3Result<()> { let Some(expected) = expected else { return Ok(()); }; - let actual = md5::Md5::digest(bytes); let expected = base64::engine::general_purpose::STANDARD .decode(expected) .map_err(|_| s3_error!(InvalidDigest))?; - if expected.as_slice() != actual.as_slice() { + if expected.as_slice() != actual { return Err(s3_error!(BadDigest)); } Ok(()) @@ -1666,33 +1827,12 @@ impl RequestChecksums { Ok(()) } - fn verify(&self, bytes: &[u8]) -> S3Result<()> { - use sha1::Digest as _; - - verify_base64_checksum( - self.crc32.as_deref(), - &u32::try_from(crc_fast::checksum( - crc_fast::CrcAlgorithm::Crc32IsoHdlc, - bytes, - )) - .map_err(|_| s3_error!(InternalError))? - .to_be_bytes(), - )?; - verify_base64_checksum( - self.crc32c.as_deref(), - &u32::try_from(crc_fast::checksum( - crc_fast::CrcAlgorithm::Crc32Iscsi, - bytes, - )) - .map_err(|_| s3_error!(InternalError))? - .to_be_bytes(), - )?; - verify_base64_checksum( - self.crc64nvme.as_deref(), - &crc_fast::checksum(crc_fast::CrcAlgorithm::Crc64Nvme, bytes).to_be_bytes(), - )?; - verify_base64_checksum(self.sha1.as_deref(), &sha1::Sha1::digest(bytes))?; - verify_base64_checksum(self.sha256.as_deref(), &sha2::Sha256::digest(bytes))?; + fn verify(&self, digests: &crate::content::Digests) -> S3Result<()> { + verify_base64_checksum(self.crc32.as_deref(), &digests.crc32.to_be_bytes())?; + verify_base64_checksum(self.crc32c.as_deref(), &digests.crc32c.to_be_bytes())?; + verify_base64_checksum(self.crc64nvme.as_deref(), &digests.crc64nvme.to_be_bytes())?; + verify_base64_checksum(self.sha1.as_deref(), &digests.sha1)?; + verify_base64_checksum(self.sha256.as_deref(), &digests.sha256)?; if let Some(algorithm) = &self.algorithm { let supplied = match algorithm.as_str() { ChecksumAlgorithm::CRC32 => self.crc32.is_some(), @@ -1915,6 +2055,7 @@ fn stored_to_pending( crate::attributes::PutAttributes { etag_override: None, completion_upload_id: None, + logical_size: Some(value.size), cache_control: value.cache_control.clone(), content_disposition: value.content_disposition.clone(), content_encoding: value.content_encoding.clone(), @@ -2138,6 +2279,20 @@ fn gateway_error(error: crate::Error) -> s3s::S3Error { s3_error!(InternalError) } +fn content_error(error: crate::content::Error) -> s3s::S3Error { + match error { + crate::content::Error::TooLarge => s3_error!(EntityTooLarge), + crate::content::Error::Incomplete | crate::content::Error::Body(_) => { + tracing::warn!(%error, "S3 request body failed"); + s3_error!(IncompleteBody) + } + crate::content::Error::Io(_) => { + tracing::error!(error = ?error, "S3 content spool failed"); + s3_error!(InternalError) + } + } +} + pub(crate) fn md5_hex(bytes: &[u8]) -> String { use md5::Digest as _; @@ -2211,8 +2366,53 @@ mod tests { assert!(encoded.contains("%28")); } - #[test] - fn put_checksums_accept_all_supported_algorithms_and_reject_mismatch() { + #[tokio::test] + async fn content_above_inline_threshold_is_stored_as_streamable_lfs() { + use futures_util::TryStreamExt as _; + + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())); + let repository = Repository::new( + RepositoryConfig { + name: "repo".to_owned(), + provider: StorageProviderKind::Local, + bucket: "memory".to_owned(), + prefix: "large-content-test".to_owned(), + default_branch: "main".to_owned(), + members: Vec::new(), + protected_branches: Vec::new(), + }, + store, + ) + .unwrap(); + let content = b"content larger than the test inline limit"; + let mut writer = crate::content::SpoolWriter::new().await.unwrap(); + writer.write(content, u64::MAX).await.unwrap(); + let spool = writer.finish().await.unwrap(); + + let pointer_bytes = mutation_bytes_with_inline_limit(&repository, &spool, 8) + .await + .unwrap(); + let PointerKind::Lfs(pointer) = crab_git::classify(&pointer_bytes) else { + panic!("expected an LFS pointer"); + }; + let (_, _, stream) = repository + .lfs + .get_stream(&pointer.oid, pointer.size, None) + .await + .unwrap(); + let actual = stream + .try_fold(Vec::new(), |mut bytes, chunk| async move { + bytes.extend_from_slice(&chunk); + Ok(bytes) + }) + .await + .unwrap(); + + assert_eq!(actual, content); + } + + #[tokio::test] + async fn put_checksums_accept_all_supported_algorithms_and_reject_mismatch() { use sha1::Digest as _; let body = b"123456789"; @@ -2238,11 +2438,14 @@ mod tests { sha1: Some(encode(&sha1::Sha1::digest(body))), sha256: Some(encode(&sha2::Sha256::digest(body))), }; - checksums.verify(body).unwrap(); + let mut writer = crate::content::SpoolWriter::new().await.unwrap(); + writer.write(body, u64::MAX).await.unwrap(); + let spool = writer.finish().await.unwrap(); + checksums.verify(&spool.digests).unwrap(); let mut invalid = checksums; invalid.sha256 = Some("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".to_owned()); - let error = invalid.verify(body).unwrap_err(); + let error = invalid.verify(&spool.digests).unwrap_err(); assert_eq!(error.code().as_str(), "BadDigest"); } } diff --git a/crates/crab-s3-gateway/src/lib.rs b/crates/crab-s3-gateway/src/lib.rs index 70077de47..9a2e5e78f 100644 --- a/crates/crab-s3-gateway/src/lib.rs +++ b/crates/crab-s3-gateway/src/lib.rs @@ -3,6 +3,7 @@ mod attributes; mod auth; mod config; +mod content; mod gateway; mod multipart; mod mutation; diff --git a/crates/crab-s3-gateway/src/multipart.rs b/crates/crab-s3-gateway/src/multipart.rs index a94d6f6a4..65717a51e 100644 --- a/crates/crab-s3-gateway/src/multipart.rs +++ b/crates/crab-s3-gateway/src/multipart.rs @@ -28,7 +28,7 @@ pub(crate) enum Error { InvalidPartOrder, #[error("a non-final multipart part is smaller than 5 MiB")] EntityTooSmall, - #[error("the completed multipart object exceeds the gateway object limit")] + #[error("the completed multipart object exceeds the S3 object limit")] EntityTooLarge, #[error("multipart state changed concurrently")] Conflict, @@ -151,9 +151,10 @@ pub(crate) async fn register_part( repository: &Repository, mut loaded: Loaded, number: i32, - bytes: Bytes, + spool: &crate::content::Spool, etag: String, now: u64, + cancel: &tokio_util::sync::CancellationToken, ) -> Result { if !(1..=10_000).contains(&number) { return Err(Error::PartNumber); @@ -164,12 +165,20 @@ pub(crate) async fn register_part( let path = format!("s3/multipart/parts/{}/{number}/{etag}", loaded.session.id); repository .store - .put_exact(&repository.layout.repo_path(&path), bytes.clone()) + .put_multipart_file_retry( + &repository.layout.repo_path(&path), + spool.path(), + spool.size, + spool.digests.blake3, + 8 * 1024 * 1024, + cancel, + None, + ) .await?; let part = Part { number, etag, - size: bytes.len() as u64, + size: spool.size, modified_seconds: now, path, }; @@ -182,6 +191,24 @@ pub(crate) async fn register_part( Ok(part) } +pub(crate) async fn part_stream( + repository: &Repository, + part: &Part, +) -> Result< + impl futures_util::Stream> + + Send + + 'static, +> { + let (metadata, range, stream) = repository + .store + .get_stream(&repository.layout.repo_path(&part.path), None) + .await?; + if metadata.size != part.size || range != (0..part.size) { + return Err(Error::InvalidPart); + } + Ok(stream) +} + pub(crate) async fn freeze( repository: &Repository, mut loaded: Loaded, @@ -241,6 +268,7 @@ pub(crate) async fn freeze( Ok((loaded.session, parts)) } +#[cfg(test)] pub(crate) async fn part_bytes(repository: &Repository, part: &Part) -> Result { let (bytes, _) = repository .store @@ -388,6 +416,12 @@ mod tests { .unwrap() } + async fn spool(bytes: &[u8]) -> crate::content::Spool { + let mut writer = crate::content::SpoolWriter::new().await.unwrap(); + writer.write(bytes, u64::MAX).await.unwrap(); + writer.finish().await.unwrap() + } + #[tokio::test] async fn parts_and_abort_survive_fresh_catalog_reads() { let repository = fixture().await; @@ -406,9 +440,18 @@ mod tests { let loaded = load(&repository, &session.id).await.unwrap(); let body = Bytes::from_static(b"part bytes"); let etag = crate::gateway::md5_hex(&body); - register_part(&repository, loaded, 1, body.clone(), etag.clone(), 11) - .await - .unwrap(); + let spool = spool(&body).await; + register_part( + &repository, + loaded, + 1, + &spool, + etag.clone(), + 11, + &tokio_util::sync::CancellationToken::new(), + ) + .await + .unwrap(); let reloaded = load(&repository, &session.id).await.unwrap(); assert_eq!(reloaded.session.parts[&1].etag, etag); @@ -444,9 +487,18 @@ mod tests { let body = Bytes::from_static(b"final part"); let etag = crate::gateway::md5_hex(&body); let loaded = load(&repository, &session.id).await.unwrap(); - register_part(&repository, loaded, 1, body, etag.clone(), 11) - .await - .unwrap(); + let spool = spool(&body).await; + register_part( + &repository, + loaded, + 1, + &spool, + etag.clone(), + 11, + &tokio_util::sync::CancellationToken::new(), + ) + .await + .unwrap(); let selected = vec![(1, etag)]; let loaded = load(&repository, &session.id).await.unwrap(); freeze(&repository, loaded, &selected, 1024).await.unwrap(); diff --git a/crates/crab-s3-gateway/src/mutation.rs b/crates/crab-s3-gateway/src/mutation.rs index 4f8787eb7..315c90463 100644 --- a/crates/crab-s3-gateway/src/mutation.rs +++ b/crates/crab-s3-gateway/src/mutation.rs @@ -382,6 +382,7 @@ async fn build_commit( Change::Put { bytes, attributes } => { let oid = object_id(Kind::Blob, &bytes)?; let digest = md5::Md5::digest(&bytes); + let logical_size = attributes.logical_size.unwrap_or(bytes.len() as u64); let etag = attributes .etag_override .clone() @@ -390,7 +391,7 @@ async fn build_commit( && old.is_some_and(|(old_oid, mode)| old_oid == oid && mode == EntryMode::Regular) && attribute_manifest .object(path_string, oid) - .is_some_and(|stored| stored.matches_pending(&attributes, &etag, bytes.len())) + .is_some_and(|stored| stored.matches_pending(&attributes, &etag, logical_size)) { return Ok(Build::Noop(Outcome { etag: Some(etag) })); } @@ -399,11 +400,7 @@ async fn build_commit( .filter(|(old_oid, _)| *old_oid == oid) .map(|_| None) .unwrap_or_else(|| Some(bytes.to_vec())); - ( - Some(etag), - changed, - Some((oid, attributes, bytes.len() as u64)), - ) + (Some(etag), changed, Some((oid, attributes, logical_size))) } Change::Delete => { if old.is_none() {