feat: add S3 gateway service

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "crates/crab-metadata",
     "crates/crab-read",
     "crates/crab-remote-git",
+    "crates/crab-s3-gateway",
     "crates/crab-staging",
     "crates/crab-storage",
     "crates/crab-types",
@@ -43,6 +44,7 @@ crab-lfs = { path = "crates/crab-lfs", default-features = false }
 crab-metadata = { path = "crates/crab-metadata", default-features = false }
 crab-read = { path = "crates/crab-read", default-features = false }
 crab-remote-git = { path = "crates/crab-remote-git", default-features = false }
+crab-s3-gateway = { path = "crates/crab-s3-gateway", default-features = false }
 crab-staging = { path = "crates/crab-staging", default-features = false }
 crab-storage = { path = "crates/crab-storage", default-features = false }
 crab-types = { path = "crates/crab-types", default-features = false }

crab/docs/architecture/crab-s3-gateway.md

@@ -1,11 +1,14 @@
 # Crab S3 gateway: executable design and phased implementation plan
 
-Status: implementation plan; all phases below are pending. Gateway crate and
-protocol support are not implemented. Proposed product defaults require the
-phase-0 decision record before dependent implementation.
+Status: initial gateway implemented in `crates/crab-s3-gateway`; broader
+cross-client, cross-provider, failure-injection, and deployment qualification
+remains a release gate. The frozen delivered surface and deliberate limits are
+recorded in `s3-gateway-contract.md`.
 Depends on the [SDK delivery plan](crab-sdk.md), especially remote writes,
 publication recovery and backend qualification. This document does not mark any
-SDK or gateway capability delivered.
+SDK capability delivered beyond the shared contracts used by the gateway.
+Where this phased plan retains proposed or future work, the frozen contract is
+the authority for the currently delivered behavior.
 
 ## Outcome and ownership
 
@@ -81,19 +84,18 @@ and parent components; Git trees also cannot contain both file `a` and file
 lossless object-key representation, including folder markers and Git-client
 round trips. No silent normalization, dropped markers or false full-key parity.
 
-## Versioning and write visibility: decisions pending
+## Versioning and write visibility
 
-Recommended initial model: each successful object mutation publishes a commit;
+Selected initial model: each successful object mutation publishes a commit;
 multipart parts remain invisible until completion. Reads pin a commit per
 request. Writable branches advance through expected-OID publication, while tags
 and commit snapshots are read-only. Unrelated concurrent file writes must not
 overwrite each other; bounded re-preparation must preserve object preconditions.
 Reauthorize and recheck conditions at publication, not just upload admission.
 
-The alternative is durable uncommitted branch state plus explicit commit.
+The rejected initial alternative is durable uncommitted branch state plus explicit commit.
 That requires shared overlay/read/commit semantics beyond the current SDK plan;
 it must not be introduced as an invisible gateway-only second repository state.
-The product choice is pending user input.
 
 Repository versioning and AWS object versioning are separate contracts. Full S3
 versioning additionally needs `GetBucketVersioning`, `PutBucketVersioning`,
@@ -1569,4 +1571,4 @@ because its code exists, and no later phase waives an earlier missing proof.
 - [GET conditions and ranges](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html)
 - [PUT metadata and preconditions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html)
 - [Multi-delete results](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html)
-- [Multipart completion and embedded errors](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html)
+- [Multipart completion and embedded errors](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html)

crab/scripts/check-architecture-gates.py

@@ -1754,6 +1754,7 @@
     "crab-lfs",
     "crab-metadata",
     "crab-read",
+    "crab-s3-gateway",
     "crab-storage",
     "crab-types",
     "crab-workflow",
@@ -1764,11 +1765,17 @@
     "crab-auth-server": {"crab-auth-receive", "crab-auth-view"},
     "crab-cache-server": {"crab-cache-server"},
 }
-SERVER_PACKAGES = {"crab-auth-server", "crab-cache-server", "crab-http-server"}
+SERVER_PACKAGES = {
+    "crab-auth-server",
+    "crab-cache-server",
+    "crab-http-server",
+    "crab-s3-gateway",
+}
 ALLOWED_SERVER_DEV_FIXTURES = {
     "crab-http-server": set(),
     "crab-auth-server": set(),
     "crab-cache-server": {"crab", "crab-cache-store"},
+    "crab-s3-gateway": set(),
 }
 WORKSPACE_DEPENDENCY_POLICY = {
     "crab-write": {"normal": {"crab-coordination", "crab-types", "crab-git", "crab-metadata", "crab-remote-git", "crab-storage", "crab-xet"}},
@@ -1857,6 +1864,21 @@
             "crab-xet",
         },
     },
+    # The S3 gateway is a product composition boundary for protocol, repository,
+    # read, write, coordination, LFS, cache, and storage behavior.
+    "crab-s3-gateway": {
+        "normal": {
+            "crab-cache-store",
+            "crab-coordination",
+            "crab-git",
+            "crab-lfs",
+            "crab-metadata",
+            "crab-read",
+            "crab-remote-git",
+            "crab-storage",
+            "crab-write",
+        },
+    },
     "crab-remote-git": {
         "normal": {"crab-git", "crab-metadata", "crab-storage", "crab-xet"},
         "dev": {"crab-metadata"},
@@ -1898,6 +1920,7 @@
     "crab-metadata": "crates/crab-metadata",
     "crab-read": "crates/crab-read",
     "crab-remote-git": "crates/crab-remote-git",
+    "crab-s3-gateway": "crates/crab-s3-gateway",
     "crab-staging": "crates/crab-staging",
     "crab-storage": "crates/crab-storage",
     "crab-types": "crates/crab-types",

crates/crab-http-server/src/maintenance.rs

@@ -1,9 +1,5 @@
 use std::{sync::Arc, time::Duration};
 
-use crab_coordination::{
-    CoordinationError, GIT_GENERATION_OWNER_RESOURCE, GcFenceHeartbeat, GcFenceLease,
-    PushLockAcquireContext,
-};
 use crab_remote_git::{RemoteGitRuntime, RepositoryIdentity, RepositoryOptions};
 use crab_storage::{Store, StoreLayout};
 use crab_write::{Result, WriteError};
@@ -13,27 +9,6 @@ use tokio_util::sync::CancellationToken;
 const LEASE_TTL: Duration = Duration::from_secs(60);
 const PASS_BUDGET: Duration = Duration::from_secs(3 * 60);
 
-struct WriterFence {
-    lease: GcFenceLease,
-    heartbeat: GcFenceHeartbeat,
-}
-
-impl WriterFence {
-    async fn acquire(store: &Store, domain: &str, cancel: &CancellationToken) -> Result<Self> {
-        if cancel.is_cancelled() {
-            return Err(WriteError::Cancelled);
-        }
-        let lease = GcFenceLease::acquire_writer(store.inner(), domain, LEASE_TTL).await?;
-        let heartbeat = GcFenceHeartbeat::spawn(&lease, cancel.clone(), LEASE_TTL / 3);
-        Ok(Self { lease, heartbeat })
-    }
-
-    async fn release(self) -> Result<()> {
-        self.heartbeat.stop().await;
-        self.lease.release().await.map_err(Into::into)
-    }
-}
-
 async fn publish(
     store: &Store,
     layout: &StoreLayout<Store>,
@@ -42,57 +17,10 @@ async fn publish(
     options: RepositoryOptions,
     cancel: &CancellationToken,
 ) -> Result<()> {
-    let mut context = PushLockAcquireContext::new(Arc::clone(store.inner()));
-    let mut owner = match context
-        .try_acquire_internal(
-            layout.repo_prefix(),
-            GIT_GENERATION_OWNER_RESOURCE,
-            LEASE_TTL,
-        )
-        .await
-    {
-        Ok(owner) => owner,
-        // Another server or CLI owner is already responsible for publication.
-        Err(CoordinationError::PushLockHeld { .. }) => return Ok(()),
-        Err(error) => return Err(error.into()),
-    };
-    let result = crab_coordination::while_renewing(&mut owner, Some(cancel), async {
-        let global = WriterFence::acquire(store, layout.global_prefix(), cancel).await?;
-        let repo = match WriterFence::acquire(store, layout.repo_prefix(), cancel).await {
-            Ok(repo) => repo,
-            Err(error) => {
-                let _ = global.release().await;
-                return Err(error);
-            }
-        };
-        let mut result = async {
-            let (manifest, _) = crab_metadata::manifest_store::read_manifest(store, layout).await?;
-            let Some(manifest) = crab_write::generation::make_readable(
-                store,
-                layout,
-                LEASE_TTL,
-                manifest.pusher,
-                cancel,
-            )
-            .await?
-            else {
-                return Ok(());
-            };
-            crab_write::generation::maintain_commit_graph(
-                store, layout, &manifest, identity, runtime, options, cancel,
-            )
-            .await?;
-            Ok(())
-        }
-        .await;
-        // Release both domains even if publication or an earlier release failed.
-        for fence in [repo, global] {
-            result = result.and(fence.release().await);
-        }
-        result
-    })
-    .await;
-    result.and(owner.release().await.map_err(Into::into))
+    crab_write::generation::ensure_readable(
+        store, layout, identity, runtime, options, LEASE_TTL, cancel,
+    )
+    .await
 }
 
 pub(crate) async fn run(

crates/crab-s3-gateway/Cargo.toml

@@ -0,0 +1,49 @@
+[package]
+name = "crab-s3-gateway"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+description = "S3-compatible gateway for Crab repositories."
+publish = false
+
+[dependencies]
+async-trait = { workspace = true }
+base64 = "0.22"
+bytes = { workspace = true }
+blake3 = { workspace = true }
+clap = { version = "4", features = ["derive"] }
+crab-coordination = { workspace = true, features = ["object-store-lock"] }
+crab-cache-store = { workspace = true }
+crab-lfs = { workspace = true }
+crab-metadata = { workspace = true, features = ["storage"] }
+crab-remote-git = { workspace = true }
+crab-read = { workspace = true }
+crab-git = { workspace = true }
+crab-storage = { workspace = true }
+crab-write = { workspace = true }
+crc-fast = "1"
+flate2 = { workspace = true }
+futures-util = { workspace = true }
+gix-hash = { workspace = true, features = ["sha1"] }
+gix-object = { workspace = true }
+http = "1"
+http-body = "1"
+http-body-util = "0.1"
+hyper-util = { version = "0.1", features = ["server-auto", "server-graceful", "http1", "http2", "tokio"] }
+md-5 = { workspace = true }
+object_store = { workspace = true }
+percent-encoding = "2"
+s3s = "=0.14.1"
+serde = { workspace = true }
+serde_json = { workspace = true }
+sha1 = "0.10"
+sha2 = "0.10"
+tempfile = { workspace = true }
+thiserror = { workspace = true }
+time = { workspace = true }
+tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
+tokio-util = { workspace = true, features = ["io", "rt"] }
+toml = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+ulid = "1"

crates/crab-s3-gateway/Dockerfile

@@ -0,0 +1,15 @@
+FROM rust:1.90-bookworm AS builder
+WORKDIR /src
+COPY . .
+RUN cargo build --locked --release -p crab-s3-gateway
+
+FROM debian:bookworm-slim
+RUN apt-get update \
+    && apt-get install --yes --no-install-recommends ca-certificates tini \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd --create-home --uid 10001 crab
+COPY --from=builder /src/target/release/crab-s3-gateway /usr/local/bin/crab-s3-gateway
+USER crab
+EXPOSE 8080
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/crab-s3-gateway"]
+CMD ["--config", "/etc/crab/s3-gateway.toml"]

crates/crab-s3-gateway/README.md

@@ -0,0 +1,46 @@
+# Crab S3 gateway
+
+`crab-s3-gateway` presents configured Crab repositories as S3 buckets. Existing
+S3 clients use their normal endpoint, region, access-key, and secret-key
+configuration. Object keys use `REF/path`, for example
+`s3://my-repository/main/data/model.bin`.
+
+The gateway accepts S3 SigV4 and SigV2 authentication through `s3s`. It maps
+each access key to a Crab principal and authorizes that principal against the
+logical repository catalog. Gateway credentials are independent of the cloud
+credentials used for the backing object store.
+
+The initial client-compatible surface includes bucket listing/head, object
+GET/HEAD/PUT/DELETE/COPY, V1/V2 object listing, multi-delete, and durable
+multipart create/upload/copy/list/abort/complete. GET/HEAD support conditions
+and a single byte range; PUT validates Content-MD5 and the standard S3 checksum
+headers. Path-style addressing is required. Single PUTs and multipart parts
+support up to 5 GiB, and multipart completion supports S3's 50 TB object limit.
+Large payloads use bounded-memory spooling and Crab's verified LFS content path.
+The complete frozen surface and deliberate exclusions are in the protocol
+contract linked below.
+
+## Build and run
+
+```sh
+cargo build --release -p crab-s3-gateway --locked
+crab-s3-gateway --config /etc/crab/s3-gateway.toml --initialize
+crab-s3-gateway --config /etc/crab/s3-gateway.toml
+```
+
+`--initialize` creates missing canonical Crab metadata only for empty configured
+prefixes, then exits. It is safe to run repeatedly. Normal serving never
+initializes or converts repository storage.
+
+The backing provider uses Crab's existing environment credential chain. Set
+the usual AWS, GCP, or Azure credentials for the selected provider. For an
+S3-compatible endpoint, `AWS_ENDPOINT_URL_S3`, `AWS_ALLOW_HTTP`, and
+`AWS_VIRTUAL_HOSTED_STYLE_REQUEST` are supported by the shared storage layer.
+
+See `s3-gateway.example.toml` for configuration and
+`crab/docs/architecture/s3-gateway-contract.md` for the protocol contract.
+Terminate with SIGTERM or SIGINT for graceful connection draining.
+
+Production deployments should bind to a private listener and terminate TLS at
+an ingress, load balancer, or service mesh. Do not expose the plain HTTP
+listener beyond a trusted network boundary.

crates/crab-s3-gateway/s3-gateway.example.toml

@@ -0,0 +1,19 @@
+listen = "127.0.0.1:8080"
+region = "us-east-1"
+
+[[credentials]]
+access_key = "replace-with-issued-access-key"
+secret_key_file = "/run/secrets/crab-s3-secret"
+principal = "service-account:analytics"
+
+[[repositories]]
+name = "analytics-repository"
+provider = "s3"
+bucket = "physical-storage-bucket"
+prefix = "repositories/analytics"
+default_branch = "main"
+protected_branches = ["release"]
+
+[[repositories.members]]
+principal = "service-account:analytics"
+access = "write"