diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5cd8ec94f..560eaeda86 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,8 @@ on: required: true type: choice options: + - capture-protocol + - capture-viewer - core - viewer - editor @@ -128,7 +130,7 @@ jobs: # peerDeps sync below must use shell vars, not env indirection. declare -A NEW_VERSIONS - for pkg in core viewer editor nodes mcp ifc-converter cli; do + for pkg in capture-protocol core viewer capture-viewer editor nodes mcp ifc-converter cli; do if [ "$TARGET" = "$pkg" ] || [ "$TARGET" = "all" ]; then CUR=$(jq -r '.version' packages/$pkg/package.json) NEW=$(bump_version "$CUR") @@ -141,26 +143,77 @@ jobs: fi done - # Sync inter-package references in peerDependencies and devDependencies. + # Sync inter-package references in dependencies, peerDependencies, and devDependencies. # Anything that references a bumped @pascal-app/* package is updated to ^NEW. - for pkg in core viewer editor nodes mcp ifc-converter cli; do + for pkg in capture-protocol core viewer capture-viewer editor nodes mcp ifc-converter cli; do FILE=packages/$pkg/package.json - for dep in core viewer editor nodes mcp ifc-converter cli; do + for dep in capture-protocol core viewer capture-viewer editor nodes mcp ifc-converter cli; do VAL="${NEW_VERSIONS[$dep]}" [ -z "$VAL" ] && continue jq --arg name "@pascal-app/$dep" --arg v "^$VAL" ' - if .peerDependencies[$name] then .peerDependencies[$name] = $v else . end + if .dependencies[$name] then .dependencies[$name] = $v else . end + | if .peerDependencies[$name] then .peerDependencies[$name] = $v else . end | if .devDependencies[$name] then .devDependencies[$name] = $v else . end ' "$FILE" > tmp.json && mv tmp.json "$FILE" done done echo "=== @pascal-app/* refs after sync ===" - for pkg in core viewer editor nodes mcp ifc-converter cli; do + for pkg in capture-protocol core viewer capture-viewer editor nodes mcp ifc-converter cli; do echo "--- packages/$pkg/package.json ---" - jq '{ peerDependencies: (.peerDependencies // {} | with_entries(select(.key | startswith("@pascal-app/")))), devDependencies: (.devDependencies // {} | with_entries(select(.key | startswith("@pascal-app/")))) }' packages/$pkg/package.json + jq '{ dependencies: (.dependencies // {} | with_entries(select(.key | startswith("@pascal-app/")))), peerDependencies: (.peerDependencies // {} | with_entries(select(.key | startswith("@pascal-app/")))), devDependencies: (.devDependencies // {} | with_entries(select(.key | startswith("@pascal-app/")))) }' packages/$pkg/package.json done + # Version and dependency ranges changed after the frozen install. + # Refresh the lockfile so the release commit remains reproducible. + bun install + + # A single-package release may depend on another package introduced + # by this monorepo. Refuse to publish an uninstallable package when + # that dependency is not part of this run and is absent from npm. + RELEASE_PACKAGES="capture-protocol core viewer capture-viewer editor nodes mcp ifc-converter cli" + if [ "$TARGET" != "all" ]; then + FILE="packages/$TARGET/package.json" + while IFS=$'\t' read -r DEP RANGE; do + SLUG="${DEP#@pascal-app/}" + case " $RELEASE_PACKAGES " in + *" $SLUG "*) + if ! npm view "$DEP@$RANGE" version >/dev/null 2>&1; then + echo "Missing required published dependency: $DEP@$RANGE" + echo "Release $SLUG first or use the all-package release." + exit 1 + fi + ;; + esac + done < <( + jq -r ' + [(.dependencies // {}), (.peerDependencies // {})] + | add + | to_entries[] + | select(.key | startswith("@pascal-app/")) + | [.key, .value] + | @tsv + ' "$FILE" + ) + fi + + - name: Build & publish capture protocol + if: inputs.package == 'capture-protocol' || inputs.package == 'all' + working-directory: packages/capture-protocol + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + bun run build + if [ "${{ inputs.dry-run }}" = "true" ]; then + echo "🏜️ Dry run — would publish @pascal-app/capture-protocol@$CAPTURE_PROTOCOL_VERSION" + npm publish --dry-run --access public --tag "$NPM_TAG" + elif npm view "@pascal-app/capture-protocol@$CAPTURE_PROTOCOL_VERSION" version >/dev/null 2>&1; then + echo "📦 @pascal-app/capture-protocol@$CAPTURE_PROTOCOL_VERSION is already published; continuing release recovery" + else + npm publish --access public --tag "$NPM_TAG" + echo "📦 Published @pascal-app/capture-protocol@$CAPTURE_PROTOCOL_VERSION" + fi + - name: Validate portable editor runtime if: inputs.package == 'cli' || inputs.package == 'all' env: @@ -206,6 +259,23 @@ jobs: echo "📦 Published @pascal-app/viewer@$VIEWER_VERSION" fi + - name: Build & publish capture viewer + if: inputs.package == 'capture-viewer' || inputs.package == 'all' + working-directory: packages/capture-viewer + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + bun run build + if [ "${{ inputs.dry-run }}" = "true" ]; then + echo "🏜️ Dry run — would publish @pascal-app/capture-viewer@$CAPTURE_VIEWER_VERSION" + npm publish --dry-run --access public --tag "$NPM_TAG" + elif npm view "@pascal-app/capture-viewer@$CAPTURE_VIEWER_VERSION" version >/dev/null 2>&1; then + echo "📦 @pascal-app/capture-viewer@$CAPTURE_VIEWER_VERSION is already published; continuing release recovery" + else + npm publish --access public --tag "$NPM_TAG" + echo "📦 Published @pascal-app/capture-viewer@$CAPTURE_VIEWER_VERSION" + fi + - name: Publish editor if: inputs.package == 'editor' || inputs.package == 'all' working-directory: packages/editor @@ -298,6 +368,10 @@ jobs: PKGS="" TAGS="" + if [ -n "$CAPTURE_PROTOCOL_VERSION" ]; then + PKGS="$PKGS @pascal-app/capture-protocol@$CAPTURE_PROTOCOL_VERSION" + TAGS="$TAGS @pascal-app/capture-protocol@$CAPTURE_PROTOCOL_VERSION" + fi if [ -n "$CORE_VERSION" ]; then PKGS="$PKGS @pascal-app/core@$CORE_VERSION" TAGS="$TAGS @pascal-app/core@$CORE_VERSION" @@ -306,6 +380,10 @@ jobs: PKGS="$PKGS @pascal-app/viewer@$VIEWER_VERSION" TAGS="$TAGS @pascal-app/viewer@$VIEWER_VERSION" fi + if [ -n "$CAPTURE_VIEWER_VERSION" ]; then + PKGS="$PKGS @pascal-app/capture-viewer@$CAPTURE_VIEWER_VERSION" + TAGS="$TAGS @pascal-app/capture-viewer@$CAPTURE_VIEWER_VERSION" + fi if [ -n "$EDITOR_VERSION" ]; then PKGS="$PKGS @pascal-app/editor@$EDITOR_VERSION" TAGS="$TAGS @pascal-app/editor@$EDITOR_VERSION" diff --git a/README.md b/README.md index 79371cef35..cae0e87e14 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,12 @@ troubleshooting. ## Using Published Packages The viewer runtime and built-in node definitions are separate packages. Install the full built-in -viewer set, then load the built-in plugin once before mounting ``: +viewer set, then load the built-in plugin once before mounting ``. Capture sessions are an +optional transport-neutral extension: ```bash npm install @pascal-app/core @pascal-app/viewer @pascal-app/editor @pascal-app/nodes +npm install @pascal-app/capture-protocol @pascal-app/capture-viewer ``` ```typescript @@ -57,6 +59,8 @@ editor/ ├── packages/ │ ├── core/ # Schemas, scene state, and registry contracts │ ├── viewer/ # 3D rendering runtime and shared systems +│ ├── capture-protocol/ # Static/live capture-session contracts +│ ├── capture-viewer/ # Capture source runtime and reference renderers │ ├── editor/ # Editing tools and UI components │ ├── nodes/ # Built-in node definitions, renderers, and systems │ ├── cli/ # Persistent local editor installer and process manager @@ -70,6 +74,8 @@ editor/ |---------|---------------| | **@pascal-app/core** | Node schemas, scene state (Zustand), registry contracts, spatial queries, and event bus | | **@pascal-app/viewer** | 3D rendering via React Three Fiber, shared render systems, default camera/controls, and post-processing | +| **@pascal-app/capture-protocol** | Versioned capture manifests, normalized streams, and transport-neutral static/live sources | +| **@pascal-app/capture-viewer** | Viewer child runtime and reference model, device-motion, and point-cloud layers | | **@pascal-app/editor** | Editing tools, panels, selection, and direct-manipulation UI | | **@pascal-app/nodes** | Built-in registry plugin with node definitions, renderers, geometry, and systems | | **@pascal-app/cli** | Installs and manages a versioned standalone editor runtime and persistent local data | diff --git a/bun.lock b/bun.lock index ea48e4eb09..fa38b54b99 100644 --- a/bun.lock +++ b/bun.lock @@ -99,6 +99,45 @@ "typescript": "7.0.2", }, }, + "packages/capture-protocol": { + "name": "@pascal-app/capture-protocol", + "version": "1.0.0-beta.4", + "dependencies": { + "zod": "^4.3.5", + }, + "devDependencies": { + "@pascal/typescript-config": "*", + "@types/bun": "^1.3.0", + "typescript": "6.0.3", + }, + }, + "packages/capture-viewer": { + "name": "@pascal-app/capture-viewer", + "version": "1.0.0-beta.4", + "devDependencies": { + "@pascal-app/capture-protocol": "^1.0.0-beta.4", + "@pascal-app/core": "^1.0.0-beta.4", + "@pascal-app/viewer": "^1.0.0-beta.4", + "@pascal/typescript-config": "*", + "@react-three/drei": "^10.7.7", + "@react-three/fiber": "^9.5.0", + "@types/bun": "^1.3.0", + "@types/react": "^19.2.2", + "@types/three": "^0.184.0", + "react": "^19.2.4", + "three": "^0.185.0", + "typescript": "6.0.3", + }, + "peerDependencies": { + "@pascal-app/capture-protocol": "^1.0.0-beta.4", + "@pascal-app/core": "^1.0.0-beta.4", + "@pascal-app/viewer": "^1.0.0-beta.4", + "@react-three/drei": "^10", + "@react-three/fiber": "^9", + "react": "^18 || ^19", + "three": "^0.185", + }, + }, "packages/cli": { "name": "@pascal-app/cli", "version": "1.0.0-beta.1", @@ -118,6 +157,7 @@ "name": "@pascal-app/core", "version": "1.0.0-beta.5", "dependencies": { + "@pascal-app/capture-protocol": "^1.0.0-beta.4", "dedent": "^1.7.1", "idb-keyval": "^6.2.2", "mitt": "^3.0.1", @@ -539,7 +579,7 @@ "@mediapipe/tasks-vision": ["@mediapipe/tasks-vision@0.10.17", "", {}, "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg=="], - "@mint/pascal-plugin": ["@mint/pascal-plugin@github:mintdotgg/mint-pascal-plugin#902c546", { "peerDependencies": { "@pascal-app/core": ">=0.9.2 <1.0.0 || >=1.0.0-beta.1 <1.0.0", "@pascal-app/editor": ">=0.9.2 <1.0.0 || >=1.0.0-beta.1 <1.0.0", "@pascal-app/viewer": ">=0.9.2 <1.0.0 || >=1.0.0-beta.1 <1.0.0", "react": "^18 || ^19", "three": "^0.185" } }, "mintdotgg-mint-pascal-plugin-902c546"], + "@mint/pascal-plugin": ["@mint/pascal-plugin@github:mintdotgg/mint-pascal-plugin#902c546", { "peerDependencies": { "@pascal-app/core": ">=0.9.2 <1.0.0 || >=1.0.0-beta.1 <1.0.0", "@pascal-app/editor": ">=0.9.2 <1.0.0 || >=1.0.0-beta.1 <1.0.0", "@pascal-app/viewer": ">=0.9.2 <1.0.0 || >=1.0.0-beta.1 <1.0.0", "react": "^18 || ^19", "three": "^0.185" } }, "mintdotgg-mint-pascal-plugin-902c546", "sha512-/itUH9r9OIP8ZPrklW8iWe6B2SDOtlL1m8r9hlVM4Rw5josYtDdkDKQ37FbanvxxuUKcQnukHbtxWe3svRaCrQ=="], "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], @@ -725,6 +765,10 @@ "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.69.0", "", { "os": "win32", "cpu": "x64" }, "sha512-w8SOXv3mT9Fi6jY8OXdXCfnvX/3KNLXGNr4HEz2TA7S4Mv/PYAOmpB8y/ge40mxvBMgGNaSaaDwZpAsQn7HtWA=="], + "@pascal-app/capture-protocol": ["@pascal-app/capture-protocol@workspace:packages/capture-protocol"], + + "@pascal-app/capture-viewer": ["@pascal-app/capture-viewer@workspace:packages/capture-viewer"], + "@pascal-app/cli": ["@pascal-app/cli@workspace:packages/cli"], "@pascal-app/core": ["@pascal-app/core@workspace:packages/core"], @@ -739,11 +783,11 @@ "@pascal-app/nodes": ["@pascal-app/nodes@workspace:packages/nodes"], - "@pascal-app/plugin-bones": ["@pascal-app/plugin-bones@github:pascalorg/plugin-bones#5c2650e", { "peerDependencies": { "@pascal-app/core": ">=0.9.1 <1", "@pascal-app/editor": ">=0.9.1 <1", "@pascal-app/viewer": ">=0.9.1 <1", "@react-three/fiber": "^9", "react": "^18 || ^19", "three": "^0.185", "zod": "^4", "zustand": "^5" } }, "pascalorg-plugin-bones-5c2650e"], + "@pascal-app/plugin-bones": ["@pascal-app/plugin-bones@github:pascalorg/plugin-bones#5c2650e", { "peerDependencies": { "@pascal-app/core": ">=0.9.1 <1", "@pascal-app/editor": ">=0.9.1 <1", "@pascal-app/viewer": ">=0.9.1 <1", "@react-three/fiber": "^9", "react": "^18 || ^19", "three": "^0.185", "zod": "^4", "zustand": "^5" } }, "pascalorg-plugin-bones-5c2650e", "sha512-yzr+QfPb1vYb9n3lVq0weih2Nzv/Ve2IcTt3WskTzFhXRanYdfz3F+n9BtgnKxrzhsALNbkp3ltZVlqJyXCGgg=="], - "@pascal-app/plugin-streetscape": ["@pascal-app/plugin-streetscape@github:sudhir9297/streetscape-pascal-plugin#1c04ec9", { "peerDependencies": { "@pascal-app/core": ">=0.9.1 <1", "@pascal-app/editor": ">=0.9.1 <1", "@pascal-app/viewer": ">=0.9.1 <1", "@react-three/fiber": "^9", "react": "^18 || ^19", "three": "^0.185", "zod": "^4", "zustand": "^5" } }, "sudhir9297-streetscape-pascal-plugin-1c04ec9"], + "@pascal-app/plugin-streetscape": ["@pascal-app/plugin-streetscape@github:sudhir9297/streetscape-pascal-plugin#1c04ec9", { "peerDependencies": { "@pascal-app/core": ">=0.9.1 <1", "@pascal-app/editor": ">=0.9.1 <1", "@pascal-app/viewer": ">=0.9.1 <1", "@react-three/fiber": "^9", "react": "^18 || ^19", "three": "^0.185", "zod": "^4", "zustand": "^5" } }, "sudhir9297-streetscape-pascal-plugin-1c04ec9", "sha512-X7Zg7wi0ghZRcTtbH5LF6xye493uSZ2ft3AmAQBtguBCU6VCwCexA7pdKDr5AnVxX/JRj2s6JSd/OX4aIZ9Y6Q=="], - "@pascal-app/plugin-trees": ["@pascal-app/plugin-trees@github:pascalorg/plugin-trees#56d978c", { "dependencies": { "@dgreenheck/ez-tree": "^1.1.0" }, "peerDependencies": { "@pascal-app/core": ">=0.9.1 <1", "@pascal-app/editor": ">=0.9.1 <1", "@pascal-app/viewer": ">=0.9.1 <1", "@react-three/fiber": "^9", "react": "^18 || ^19", "three": "^0.185", "zod": "^4", "zustand": "^5" } }, "pascalorg-plugin-trees-56d978c"], + "@pascal-app/plugin-trees": ["@pascal-app/plugin-trees@github:pascalorg/plugin-trees#56d978c", { "dependencies": { "@dgreenheck/ez-tree": "^1.1.0" }, "peerDependencies": { "@pascal-app/core": ">=0.9.1 <1", "@pascal-app/editor": ">=0.9.1 <1", "@pascal-app/viewer": ">=0.9.1 <1", "@react-three/fiber": "^9", "react": "^18 || ^19", "three": "^0.185", "zod": "^4", "zustand": "^5" } }, "pascalorg-plugin-trees-56d978c", "sha512-16VzWot1oadvxCPqsRwMJbaP0a3u5FESFy7F7++pY5wAAFq9JTFwzHvKAsllrY5TZ5TJ7Y5vSqvbmQk8sy8HaA=="], "@pascal-app/viewer": ["@pascal-app/viewer@workspace:packages/viewer"], @@ -1471,7 +1515,7 @@ "is-number-object": ["is-number-object@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3", "has-tostringtag": "^1.0.2" } }, "sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw=="], - "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], + "is-promise": ["is-promise@2.2.2", "", {}, "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ=="], "is-regex": ["is-regex@1.2.1", "", { "dependencies": { "call-bound": "^1.0.2", "gopd": "^1.2.0", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g=="], @@ -2117,8 +2161,6 @@ "postcss/nanoid": ["nanoid@3.3.17", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g=="], - "promise-worker-transferable/is-promise": ["is-promise@2.2.2", "", {}, "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ=="], - "react-doctor/agent-install": ["agent-install@0.0.5", "", { "dependencies": { "@iarna/toml": "^2.2.5", "commander": "^14.0.0", "jsonc-parser": "^3.3.1", "picocolors": "^1.1.1", "prompts": "^2.4.2", "yaml": "^2.8.3" }, "bin": { "agent-install": "bin/agent-install.mjs" } }, "sha512-nHlms9BkP8ZiY79HrwCGiA2DcNaXrAaJrCM/BEqQ7MEsSKyCk+2A76xPGylIfASZSZE0SaU3T0bNSg4rBPIJAQ=="], "react-doctor/eslint-plugin-react-hooks": ["eslint-plugin-react-hooks@7.1.1", "", { "dependencies": { "@babel/core": "^7.24.4", "@babel/parser": "^7.24.4", "hermes-parser": "^0.25.1", "zod": "^3.25.0 || ^4.0.0", "zod-validation-error": "^3.5.0 || ^4.0.0" }, "peerDependencies": { "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" } }, "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g=="], @@ -2127,6 +2169,8 @@ "react-scan/commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], + "router/is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], + "sharp/semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], "three-stdlib/fflate": ["fflate@0.6.10", "", {}, "sha512-IQrh3lEPM93wVCEczc9SaAOvkmcoQn/G8Bo1e8ZPlY3X3bnAxWaBdvTdvM1hP62iZp0BXWDy4vTAy4fF0+Dlpg=="], diff --git a/packages/capture-protocol/README.md b/packages/capture-protocol/README.md new file mode 100644 index 0000000000..a569059558 --- /dev/null +++ b/packages/capture-protocol/README.md @@ -0,0 +1,30 @@ +# `@pascal-app/capture-protocol` + +Transport-neutral capture-session contracts for Pascal viewers and hosts. + +The package contains versioned static manifests, a normalized session descriptor, packet headers +for incremental data, and a `CaptureSource` interface that can be backed by HTTP, WebSocket, +WebRTC, local files, or an in-memory producer. It does not contain authentication, persistence, +React, Three.js, or a canonical network transport. + +```ts +import { + createHttpCaptureSource, + type CaptureSessionLocator, +} from '@pascal-app/capture-protocol' + +const locator: CaptureSessionLocator = { + sessionId: 'capture_123', + manifestUrl: '/api/captures/capture_123/manifest', +} + +const source = createHttpCaptureSource(locator, { credentials: 'include' }) +const descriptor = await source.describe() +``` + +For live producers, use `PushCaptureSource` directly or implement `CaptureSource.subscribe()` with +the same descriptor and packet event contract. + +The package exports its TypeScript source under the `react-native` condition so Metro can consume +the workspace package from a clean checkout. Web and Node consumers continue to use the compiled +ES module output. diff --git a/packages/capture-protocol/package.json b/packages/capture-protocol/package.json new file mode 100644 index 0000000000..2c3184b39f --- /dev/null +++ b/packages/capture-protocol/package.json @@ -0,0 +1,50 @@ +{ + "name": "@pascal-app/capture-protocol", + "version": "1.0.0-beta.4", + "description": "Transport-neutral capture-session manifests and live stream sources for Pascal", + "type": "module", + "main": "./dist/index.js", + "types": "./src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "react-native": "./src/index.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsc --build", + "dev": "tsgo --build --watch", + "test": "bun test src", + "prepublishOnly": "npm run build" + }, + "dependencies": { + "zod": "^4.3.5" + }, + "devDependencies": { + "@pascal/typescript-config": "*", + "@types/bun": "^1.3.0", + "typescript": "6.0.3" + }, + "keywords": [ + "3d", + "capture", + "point-cloud", + "sensor-fusion", + "streaming" + ], + "repository": { + "type": "git", + "url": "https://github.com/pascalorg/editor.git", + "directory": "packages/capture-protocol" + }, + "license": "MIT", + "homepage": "https://github.com/pascalorg/editor/tree/main/packages/capture-protocol#readme", + "bugs": "https://github.com/pascalorg/editor/issues" +} diff --git a/packages/capture-protocol/src/index.ts b/packages/capture-protocol/src/index.ts new file mode 100644 index 0000000000..097ae99055 --- /dev/null +++ b/packages/capture-protocol/src/index.ts @@ -0,0 +1,47 @@ +export { + ArkitDeviceMotionTrajectorySchema, + ArkitPointCloudPayloadSchema, + ArkitSurfaceMeshPayloadSchema, + type CaptureArtifactReference, + CaptureArtifactReferenceSchema, + type CaptureClock, + CaptureClockSchema, + type CaptureCoordinateFrame, + CaptureCoordinateFrameSchema, + type CaptureSessionDescriptor, + CaptureSessionDescriptorSchema, + type CaptureSessionLocator, + CaptureSessionLocatorSchema, + type CaptureSessionManifest, + CaptureSessionManifestSchema, + type CaptureSessionManifestV1, + CaptureSessionManifestV1Schema, + type CaptureSessionManifestV2, + CaptureSessionManifestV2Schema, + type CaptureStreamDescriptor, + CaptureStreamDescriptorSchema, + CaptureTimeRangeSchema, + captureLayerKey, + captureStreamLabel, + DeviceMotionSampleSchema, + type DeviceMotionTrajectoryPayload, + DeviceMotionTrajectorySchema, + normalizeCaptureSessionManifest, + type PointCloudPayload, + PointCloudPayloadSchema, + type SurfaceMeshPayload, + SurfaceMeshPayloadSchema, +} from './schema' +export { + type CaptureArtifactResolution, + type CaptureSource, + type CaptureSourceEvent, + type CaptureSourceResolver, + type CaptureStreamPacket, + CaptureStreamPacketSchema, + type CaptureSubscriptionOptions, + createHttpCaptureSource, + type HttpCaptureSourceOptions, + PushCaptureSource, + type PushCaptureSourceOptions, +} from './source' diff --git a/packages/capture-protocol/src/schema.test.ts b/packages/capture-protocol/src/schema.test.ts new file mode 100644 index 0000000000..bd44354b74 --- /dev/null +++ b/packages/capture-protocol/src/schema.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, test } from 'bun:test' +import { + CaptureSessionManifestV2Schema, + captureLayerKey, + normalizeCaptureSessionManifest, +} from './schema' + +const identity = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] + +describe('capture manifests', () => { + test('normalizes the Community v1 manifest into extensible streams', () => { + const descriptor = normalizeCaptureSessionManifest({ + schemaVersion: 1, + sessionId: 'capture_123', + projectId: 'project_123', + streams: { + roomModel: { + kind: 'room-model', + mediaType: 'model/vnd.usdz+zip', + url: 'https://cdn.pascal.app/room.usdz', + }, + deviceMotion: { + kind: 'device-motion', + trajectory: { + coordinateSystem: 'arkit-world', + samples: [ + { segment: 0, timestamp: 0, transform: identity }, + { segment: 0, timestamp: 1, transform: identity }, + ], + }, + }, + pointCloud: { + kind: 'point-cloud', + points: { + coordinateSystem: 'arkit-world', + positions: [0, 0, 0, 1, 1, 1], + }, + }, + surfaceMesh: { + kind: 'surface-mesh', + mesh: { + version: 1, + coordinateSystem: 'arkit-world', + representation: 'quantized-indexed-triangle-mesh', + appearance: 'camera-vertex-color', + vertexCount: 3, + faceCount: 1, + boundsMin: [0, 0, 0], + boundsMax: [1, 1, 0], + positionEncoding: 'uint16x3-base64-little-endian', + colorEncoding: 'uint8x3-base64-srgb', + indexEncoding: 'uint16x3-base64-little-endian', + positions: 'AAAAAAAAAAAAAAAAAAAAAAAA', + colors: '////////////', + indices: 'AAABAAIA', + }, + }, + }, + }) + + expect(descriptor.streams.map(captureLayerKey)).toEqual([ + 'model', + 'deviceMotion', + 'pointCloud', + 'surfaceMesh', + ]) + expect(descriptor.streams[0]?.artifact?.uri).toBe('https://cdn.pascal.app/room.usdz') + expect(descriptor.streams[2]?.inline).toMatchObject({ + coordinateSystem: 'arkit-world', + positions: [0, 0, 0, 1, 1, 1], + }) + expect(descriptor.streams[3]?.inline).toMatchObject({ + appearance: 'camera-vertex-color', + faceCount: 1, + }) + }) + + test('keeps unknown v2 stream kinds without a protocol release', () => { + const manifest = CaptureSessionManifestV2Schema.parse({ + schemaVersion: 2, + sessionId: 'capture_123', + state: 'live', + streams: [ + { + id: 'wifi-rtt', + kind: 'wifi-ranging', + availability: 'live', + }, + ], + }) + + expect(normalizeCaptureSessionManifest(manifest).streams[0]?.kind).toBe('wifi-ranging') + }) + + test('preserves the exact ARKit coordinate system required by v1', () => { + expect(() => + normalizeCaptureSessionManifest({ + schemaVersion: 1, + sessionId: 'capture_123', + projectId: 'project_123', + streams: { + deviceMotion: { + kind: 'device-motion', + trajectory: { + coordinateSystem: 'unknown', + samples: [ + { segment: 0, timestamp: 0, transform: identity }, + { segment: 0, timestamp: 1, transform: identity }, + ], + }, + }, + }, + }), + ).toThrow() + }) + + test('rejects oversized or structurally inconsistent surface meshes', () => { + const surfaceMesh = { + version: 1, + coordinateSystem: 'arkit-world', + representation: 'quantized-indexed-triangle-mesh', + appearance: 'camera-vertex-color', + vertexCount: 3, + faceCount: 1, + boundsMin: [0, 0, 0], + boundsMax: [1, 1, 0], + positionEncoding: 'uint16x3-base64-little-endian', + colorEncoding: 'uint8x3-base64-srgb', + indexEncoding: 'uint16x3-base64-little-endian', + positions: 'AAAAAAAAAAAAAAAAAAAAAAAA', + colors: '////////////', + indices: 'AAABAAIA', + } + const manifest = (mesh: unknown) => ({ + schemaVersion: 1, + sessionId: 'capture_123', + projectId: 'project_123', + streams: { surfaceMesh: { kind: 'surface-mesh', mesh } }, + }) + + expect(() => + normalizeCaptureSessionManifest(manifest({ ...surfaceMesh, faceCount: 6_001 })), + ).toThrow() + expect(() => + normalizeCaptureSessionManifest(manifest({ ...surfaceMesh, positions: 'AAAA' })), + ).toThrow('decoded bytes') + expect(() => + normalizeCaptureSessionManifest(manifest({ ...surfaceMesh, indices: 'AAABAP//' })), + ).toThrow('existing vertex') + }) + + test('rejects non-finite capture geometry', () => { + expect(() => + normalizeCaptureSessionManifest({ + schemaVersion: 1, + sessionId: 'capture_123', + projectId: 'project_123', + streams: { + pointCloud: { + kind: 'point-cloud', + points: { + coordinateSystem: 'arkit-world', + positions: [0, 0, Number.POSITIVE_INFINITY], + }, + }, + }, + }), + ).toThrow() + + expect(() => + normalizeCaptureSessionManifest({ + schemaVersion: 1, + sessionId: 'capture_123', + projectId: 'project_123', + streams: { + surfaceMesh: { + kind: 'surface-mesh', + mesh: { + version: 1, + coordinateSystem: 'arkit-world', + representation: 'quantized-indexed-triangle-mesh', + appearance: 'camera-vertex-color', + vertexCount: 3, + faceCount: 1, + boundsMin: [0, 0, Number.NaN], + boundsMax: [1, 1, 0], + positionEncoding: 'uint16x3-base64-little-endian', + colorEncoding: 'uint8x3-base64-srgb', + indexEncoding: 'uint16x3-base64-little-endian', + positions: 'AAAAAAAAAAAAAAAAAAAAAAAA', + colors: '////////////', + indices: 'AAABAAIA', + }, + }, + }, + }), + ).toThrow() + }) + + test('rejects duplicate stream IDs and backwards time ranges', () => { + expect(() => + normalizeCaptureSessionManifest({ + schemaVersion: 2, + sessionId: 'capture_123', + streams: [ + { id: 'points', kind: 'point-cloud' }, + { id: 'points', kind: 'point-cloud' }, + ], + }), + ).toThrow('Duplicate capture streams id') + + expect(() => + normalizeCaptureSessionManifest({ + schemaVersion: 2, + sessionId: 'capture_123', + streams: [ + { + id: 'video', + kind: 'video', + artifact: { + id: 'video', + mediaType: 'video/mp4', + timeRange: { start: 2, end: 1 }, + }, + }, + ], + }), + ).toThrow('must end at or after') + }) +}) diff --git a/packages/capture-protocol/src/schema.ts b/packages/capture-protocol/src/schema.ts new file mode 100644 index 0000000000..7ff16ea261 --- /dev/null +++ b/packages/capture-protocol/src/schema.ts @@ -0,0 +1,394 @@ +import { z } from 'zod' + +const MetadataSchema = z.record(z.string(), z.unknown()) + +export const CaptureSessionLocatorSchema = z.object({ + sessionId: z.string().min(1), + manifestUrl: z.string().min(1).optional(), + schemaVersion: z.number().int().positive().optional(), + revisionId: z.string().min(1).optional(), +}) + +export const DeviceMotionSampleSchema = z.object({ + segment: z.number().int().nonnegative(), + timestamp: z.number().nonnegative(), + transform: z.array(z.number()).length(16), +}) + +export const DeviceMotionTrajectorySchema = z.object({ + coordinateSystem: z.string().min(1), + samples: z.array(DeviceMotionSampleSchema).min(2), +}) + +export const ArkitDeviceMotionTrajectorySchema = DeviceMotionTrajectorySchema.extend({ + coordinateSystem: z.literal('arkit-world'), +}) + +export const PointCloudPayloadSchema = z + .object({ + coordinateSystem: z.string().min(1), + positions: z.array(z.number().finite()).min(3), + colors: z.array(z.number().finite()).optional(), + }) + .superRefine((payload, context) => { + if (payload.positions.length % 3 !== 0) { + context.addIssue({ + code: 'custom', + message: 'Point-cloud positions must contain XYZ triples.', + path: ['positions'], + }) + } + if (payload.colors && payload.colors.length !== payload.positions.length) { + context.addIssue({ + code: 'custom', + message: 'Point-cloud colors must match the positions array length.', + path: ['colors'], + }) + } + }) + +export const ArkitPointCloudPayloadSchema = PointCloudPayloadSchema.safeExtend({ + coordinateSystem: z.literal('arkit-world'), +}) + +const MAX_SURFACE_MESH_VERTICES = 65_535 +const MAX_SURFACE_MESH_FACES = 6_000 + +export const SurfaceMeshPayloadSchema = z + .object({ + version: z.literal(1), + coordinateSystem: z.string().min(1), + representation: z.literal('quantized-indexed-triangle-mesh'), + appearance: z.literal('camera-vertex-color'), + vertexCount: z.number().int().positive().max(MAX_SURFACE_MESH_VERTICES), + faceCount: z.number().int().positive().max(MAX_SURFACE_MESH_FACES), + boundsMin: z.array(z.number().finite()).length(3), + boundsMax: z.array(z.number().finite()).length(3), + positionEncoding: z.literal('uint16x3-base64-little-endian'), + colorEncoding: z.literal('uint8x3-base64-srgb'), + indexEncoding: z.literal('uint16x3-base64-little-endian'), + positions: z.string().min(1).max(524_280), + colors: z.string().min(1).max(262_140), + indices: z.string().min(1).max(48_000), + }) + .superRefine((payload, context) => { + if (payload.vertexCount > payload.faceCount * 3) { + context.addIssue({ + code: 'custom', + message: 'Surface meshes cannot contain more than three vertices per face.', + path: ['vertexCount'], + }) + } + for (let axis = 0; axis < 3; axis += 1) { + if ((payload.boundsMax[axis] ?? 0) < (payload.boundsMin[axis] ?? 0)) { + context.addIssue({ + code: 'custom', + message: 'Surface-mesh maximum bounds must not be below minimum bounds.', + path: ['boundsMax', axis], + }) + } + } + + const positionBytes = decodeBase64(payload.positions) + const colorBytes = decodeBase64(payload.colors) + const indexBytes = decodeBase64(payload.indices) + validateSurfaceMeshByteLength(positionBytes, payload.vertexCount * 3 * 2, 'positions', context) + validateSurfaceMeshByteLength(colorBytes, payload.vertexCount * 3, 'colors', context) + validateSurfaceMeshByteLength(indexBytes, payload.faceCount * 3 * 2, 'indices', context) + + if (indexBytes?.byteLength === payload.faceCount * 3 * 2) { + const indices = new DataView(indexBytes.buffer, indexBytes.byteOffset, indexBytes.byteLength) + for (let offset = 0; offset < indexBytes.byteLength; offset += 2) { + if (indices.getUint16(offset, true) >= payload.vertexCount) { + context.addIssue({ + code: 'custom', + message: 'Surface-mesh indices must reference an existing vertex.', + path: ['indices'], + }) + break + } + } + } + }) + +export const ArkitSurfaceMeshPayloadSchema = SurfaceMeshPayloadSchema.safeExtend({ + coordinateSystem: z.literal('arkit-world'), +}) + +export const CaptureTimeRangeSchema = z + .object({ + start: z.number().nonnegative(), + end: z.number().nonnegative(), + }) + .refine((range) => range.end >= range.start, { + message: 'Capture time ranges must end at or after they start.', + path: ['end'], + }) + +export const CaptureArtifactReferenceSchema = z.object({ + id: z.string().min(1), + uri: z.string().min(1).optional(), + mediaType: z.string().min(1), + byteLength: z.number().int().nonnegative().optional(), + sha256: z.string().min(1).optional(), + frameId: z.string().min(1).optional(), + timeRange: CaptureTimeRangeSchema.optional(), + metadata: MetadataSchema.optional(), +}) + +export const CaptureStreamDescriptorSchema = z.object({ + id: z.string().min(1), + kind: z.string().min(1), + role: z.string().min(1).optional(), + availability: z.enum(['pending', 'live', 'ready', 'failed']).default('ready'), + frameId: z.string().min(1).optional(), + clockId: z.string().min(1).optional(), + artifact: CaptureArtifactReferenceSchema.optional(), + inline: z.unknown().optional(), + metadata: MetadataSchema.optional(), +}) + +export const CaptureClockSchema = z.object({ + id: z.string().min(1), + timebase: z.enum(['seconds', 'milliseconds', 'microseconds', 'nanoseconds']), + epoch: z.string().min(1).optional(), +}) + +export const CaptureCoordinateFrameSchema = z.object({ + id: z.string().min(1), + parentId: z.string().min(1).optional(), + convention: z.string().min(1), + transform: z.array(z.number()).length(16).optional(), +}) + +export const CaptureSessionManifestV1Schema = z.object({ + schemaVersion: z.literal(1), + sessionId: z.string().min(1), + projectId: z.string().min(1), + streams: z.object({ + roomModel: z + .object({ + kind: z.literal('room-model'), + mediaType: z.literal('model/vnd.usdz+zip'), + url: z.string().min(1), + }) + .optional(), + deviceMotion: z + .object({ + kind: z.literal('device-motion'), + trajectory: ArkitDeviceMotionTrajectorySchema, + }) + .optional(), + pointCloud: z + .object({ + kind: z.literal('point-cloud'), + points: ArkitPointCloudPayloadSchema, + }) + .optional(), + surfaceMesh: z + .object({ + kind: z.literal('surface-mesh'), + mesh: ArkitSurfaceMeshPayloadSchema, + }) + .optional(), + }), +}) + +export const CaptureSessionManifestV2Schema = z + .object({ + schemaVersion: z.literal(2), + sessionId: z.string().min(1), + projectId: z.string().min(1).optional(), + revisionId: z.string().min(1).optional(), + state: z.enum(['live', 'finalizing', 'ready', 'failed']).default('ready'), + clocks: z.array(CaptureClockSchema).default([]), + coordinateFrames: z.array(CaptureCoordinateFrameSchema).default([]), + streams: z.array(CaptureStreamDescriptorSchema), + metadata: MetadataSchema.optional(), + }) + .superRefine(validateUniqueSessionIds) + +export const CaptureSessionManifestSchema = z.union([ + CaptureSessionManifestV1Schema, + CaptureSessionManifestV2Schema, +]) + +export const CaptureSessionDescriptorSchema = z + .object({ + schemaVersion: z.number().int().positive(), + sessionId: z.string().min(1), + projectId: z.string().min(1).optional(), + revisionId: z.string().min(1).optional(), + state: z.enum(['live', 'finalizing', 'ready', 'failed']), + clocks: z.array(CaptureClockSchema), + coordinateFrames: z.array(CaptureCoordinateFrameSchema), + streams: z.array(CaptureStreamDescriptorSchema), + metadata: MetadataSchema.optional(), + }) + .superRefine(validateUniqueSessionIds) + +export type CaptureArtifactReference = z.infer +export type CaptureClock = z.infer +export type CaptureCoordinateFrame = z.infer +export type CaptureSessionDescriptor = z.infer +export type CaptureSessionLocator = z.infer +export type CaptureSessionManifest = z.infer +export type CaptureSessionManifestV1 = z.infer +export type CaptureSessionManifestV2 = z.infer +export type CaptureStreamDescriptor = z.infer +export type DeviceMotionTrajectoryPayload = z.infer +export type PointCloudPayload = z.infer +export type SurfaceMeshPayload = z.infer + +export function normalizeCaptureSessionManifest(value: unknown): CaptureSessionDescriptor { + const manifest = CaptureSessionManifestSchema.parse(value) + if (manifest.schemaVersion === 2) return CaptureSessionDescriptorSchema.parse(manifest) + + const streams: CaptureStreamDescriptor[] = [] + if (manifest.streams.roomModel) { + streams.push({ + id: 'room-model', + kind: manifest.streams.roomModel.kind, + role: 'model', + availability: 'ready', + artifact: { + id: `${manifest.sessionId}:room-model`, + mediaType: manifest.streams.roomModel.mediaType, + uri: manifest.streams.roomModel.url, + }, + }) + } + if (manifest.streams.deviceMotion) { + streams.push({ + id: 'device-motion', + kind: manifest.streams.deviceMotion.kind, + role: 'deviceMotion', + availability: 'ready', + inline: manifest.streams.deviceMotion.trajectory, + }) + } + if (manifest.streams.pointCloud) { + streams.push({ + id: 'point-cloud', + kind: manifest.streams.pointCloud.kind, + role: 'pointCloud', + availability: 'ready', + inline: manifest.streams.pointCloud.points, + }) + } + if (manifest.streams.surfaceMesh) { + streams.push({ + id: 'surface-mesh', + kind: manifest.streams.surfaceMesh.kind, + role: 'surfaceMesh', + availability: 'ready', + inline: manifest.streams.surfaceMesh.mesh, + }) + } + + return CaptureSessionDescriptorSchema.parse({ + schemaVersion: manifest.schemaVersion, + sessionId: manifest.sessionId, + projectId: manifest.projectId, + state: 'ready', + clocks: [], + coordinateFrames: [], + streams, + }) +} + +export function captureLayerKey(stream: CaptureStreamDescriptor): string { + if (stream.role) return stream.role + if (stream.kind === 'room-model') return 'model' + if (stream.kind === 'device-motion') return 'deviceMotion' + if (stream.kind === 'point-cloud') return 'pointCloud' + if (stream.kind === 'surface-mesh') return 'surfaceMesh' + if (stream.kind === 'gaussian-splat') return 'splat' + return stream.kind +} + +export function captureStreamLabel(stream: CaptureStreamDescriptor): string { + const key = captureLayerKey(stream) + if (key === 'model') return '3D model' + if (key === 'deviceMotion') return 'Device motion' + if (key === 'pointCloud') return 'Point cloud' + if (key === 'surfaceMesh') return 'Surface mesh' + if (key === 'splat') return 'Gaussian splat' + return key + .replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/[-_]+/g, ' ') + .replace(/^./, (value) => value.toUpperCase()) +} + +function validateSurfaceMeshByteLength( + bytes: Uint8Array | null, + expectedLength: number, + path: 'colors' | 'indices' | 'positions', + context: { addIssue(issue: { code: 'custom'; message: string; path: string[] }): void }, +): void { + if (bytes?.byteLength === expectedLength) return + context.addIssue({ + code: 'custom', + message: `Surface-mesh ${path} must contain exactly ${expectedLength} decoded bytes.`, + path: [path], + }) +} + +function decodeBase64(value: string): Uint8Array | null { + if ( + value.length % 4 !== 0 || + !/^(?:[A-Za-z\d+/]{4})*(?:[A-Za-z\d+/]{2}==|[A-Za-z\d+/]{3}=)?$/.test(value) + ) { + return null + } + const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + const padding = value.endsWith('==') ? 2 : value.endsWith('=') ? 1 : 0 + const output = new Uint8Array((value.length / 4) * 3 - padding) + let outputIndex = 0 + for (let index = 0; index < value.length; index += 4) { + const a = alphabet.indexOf(value[index] ?? '') + const b = alphabet.indexOf(value[index + 1] ?? '') + const c = value[index + 2] === '=' ? 0 : alphabet.indexOf(value[index + 2] ?? '') + const d = value[index + 3] === '=' ? 0 : alphabet.indexOf(value[index + 3] ?? '') + const bits = a * 262_144 + b * 4096 + c * 64 + d + if (outputIndex < output.length) output[outputIndex++] = Math.floor(bits / 65_536) % 256 + if (outputIndex < output.length) output[outputIndex++] = Math.floor(bits / 256) % 256 + if (outputIndex < output.length) output[outputIndex++] = bits % 256 + } + return output +} + +function validateUniqueSessionIds( + value: { + clocks: Array<{ id: string }> + coordinateFrames: Array<{ id: string }> + streams: Array<{ id: string }> + }, + context: { + addIssue(issue: { code: 'custom'; message: string; path: Array }): void + }, +): void { + validateUniqueIds(value.streams, 'streams', context) + validateUniqueIds(value.clocks, 'clocks', context) + validateUniqueIds(value.coordinateFrames, 'coordinateFrames', context) +} + +function validateUniqueIds( + values: Array<{ id: string }>, + path: string, + context: { + addIssue(issue: { code: 'custom'; message: string; path: Array }): void + }, +): void { + const seen = new Set() + values.forEach((value, index) => { + if (seen.has(value.id)) { + context.addIssue({ + code: 'custom', + message: `Duplicate capture ${path} id: ${value.id}`, + path: [path, index, 'id'], + }) + } + seen.add(value.id) + }) +} diff --git a/packages/capture-protocol/src/source.test.ts b/packages/capture-protocol/src/source.test.ts new file mode 100644 index 0000000000..e5f493f6d4 --- /dev/null +++ b/packages/capture-protocol/src/source.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, test } from 'bun:test' +import type { CaptureSessionDescriptor } from './schema' +import { createHttpCaptureSource, PushCaptureSource } from './source' + +const descriptor: CaptureSessionDescriptor = { + schemaVersion: 2, + sessionId: 'capture_123', + state: 'live', + clocks: [], + coordinateFrames: [], + streams: [ + { id: 'points', kind: 'point-cloud', role: 'pointCloud', availability: 'live' }, + { id: 'motion', kind: 'device-motion', role: 'deviceMotion', availability: 'live' }, + ], +} + +describe('PushCaptureSource', () => { + test('filters live packets by stream and closes the iterator', async () => { + const source = new PushCaptureSource(descriptor) + const iterator = source.subscribe({ streamIds: ['points'] })[Symbol.asyncIterator]() + + source.publishPacket({ + protocolVersion: 1, + sessionId: descriptor.sessionId, + streamId: 'motion', + generation: 0, + sequence: 0, + timestamp: 0, + payload: {}, + }) + source.publishPacket({ + protocolVersion: 1, + sessionId: descriptor.sessionId, + streamId: 'points', + generation: 0, + sequence: 1, + timestamp: 0.1, + payload: { positions: [0, 0, 0] }, + }) + + expect((await iterator.next()).value).toMatchObject({ + type: 'packet', + packet: { streamId: 'points', sequence: 1 }, + }) + + source.close() + expect((await iterator.next()).value).toEqual({ type: 'closed' }) + expect((await iterator.next()).done).toBe(true) + }) + + test('rejects packets from another session', () => { + const source = new PushCaptureSource(descriptor) + expect(() => + source.publishPacket({ + protocolVersion: 1, + sessionId: 'capture_other', + streamId: 'points', + generation: 0, + sequence: 0, + timestamp: 0, + payload: {}, + }), + ).toThrow('does not belong') + }) + + test('rejects packets for undeclared streams', () => { + const source = new PushCaptureSource(descriptor) + expect(() => + source.publishPacket({ + protocolVersion: 1, + sessionId: descriptor.sessionId, + streamId: 'typo', + generation: 0, + sequence: 0, + timestamp: 0, + payload: {}, + }), + ).toThrow('unknown stream') + }) + + test('bounds slow-subscriber queues and keeps the newest packets', async () => { + const source = new PushCaptureSource(descriptor, { maxQueuedEventsPerSubscriber: 2 }) + const iterator = source.subscribe({ streamIds: ['points'] })[Symbol.asyncIterator]() + for (let sequence = 0; sequence < 4; sequence += 1) { + source.publishPacket({ + protocolVersion: 1, + sessionId: descriptor.sessionId, + streamId: 'points', + generation: 0, + sequence, + timestamp: sequence, + payload: {}, + }) + } + + expect((await iterator.next()).value).toMatchObject({ packet: { sequence: 2 } }) + expect((await iterator.next()).value).toMatchObject({ packet: { sequence: 3 } }) + }) + + test('cancellation clears queued packets', async () => { + const source = new PushCaptureSource(descriptor) + const iterator = source.subscribe()[Symbol.asyncIterator]() + source.publishPacket({ + protocolVersion: 1, + sessionId: descriptor.sessionId, + streamId: 'points', + generation: 0, + sequence: 0, + timestamp: 0, + payload: {}, + }) + + await iterator.return?.() + expect((await iterator.next()).done).toBe(true) + }) + + test('does not expose mutable descriptor identity', async () => { + const source = new PushCaptureSource(descriptor) + const described = await source.describe() + described.sessionId = 'mutated' + + expect((await source.describe()).sessionId).toBe(descriptor.sessionId) + }) + + test('isolates live event payloads between subscribers', async () => { + const source = new PushCaptureSource(descriptor) + const first = source.subscribe()[Symbol.asyncIterator]() + const second = source.subscribe()[Symbol.asyncIterator]() + source.publishPacket({ + protocolVersion: 1, + sessionId: descriptor.sessionId, + streamId: 'points', + generation: 0, + sequence: 0, + timestamp: 0, + payload: { positions: [0, 0, 0] }, + }) + + const firstEvent = (await first.next()).value + if (firstEvent?.type !== 'packet') throw new Error('Expected a packet event.') + const firstPayload = firstEvent.packet.payload as { positions: number[] } + firstPayload.positions[0] = 99 + + const secondEvent = (await second.next()).value + expect(secondEvent).toMatchObject({ + type: 'packet', + packet: { payload: { positions: [0, 0, 0] } }, + }) + }) +}) + +describe('createHttpCaptureSource', () => { + test('resolves relative artifacts against an absolute manifest URL', async () => { + const source = createHttpCaptureSource( + { + sessionId: 'capture_123', + manifestUrl: 'https://example.com/captures/capture_123/manifest.json', + }, + { + fetch: (async () => + new Response( + JSON.stringify({ + schemaVersion: 1, + sessionId: 'capture_123', + projectId: 'project_123', + streams: {}, + }), + )) as typeof fetch, + }, + ) + + await source.describe() + await expect( + source.resolveArtifact?.({ id: 'model', mediaType: 'model/gltf-binary', uri: 'room.glb' }), + ).resolves.toEqual({ url: 'https://example.com/captures/capture_123/room.glb' }) + }) + + test('enforces locator schema and revision pins', async () => { + const source = createHttpCaptureSource( + { + sessionId: 'capture_123', + manifestUrl: 'https://example.com/manifest.json', + revisionId: 'revision_expected', + schemaVersion: 2, + }, + { + fetch: (async () => + new Response( + JSON.stringify({ + schemaVersion: 2, + sessionId: 'capture_123', + revisionId: 'revision_other', + streams: [], + }), + )) as typeof fetch, + }, + ) + + await expect(source.describe()).rejects.toThrow('revision does not match') + }) + + test('deduplicates static manifest requests across consumers', async () => { + let requests = 0 + const source = createHttpCaptureSource( + { sessionId: 'capture_123', manifestUrl: 'https://example.com/manifest.json' }, + { + fetch: (async () => { + requests += 1 + return new Response( + JSON.stringify({ + schemaVersion: 1, + sessionId: 'capture_123', + projectId: 'project_123', + streams: {}, + }), + ) + }) as typeof fetch, + }, + ) + + await Promise.all([source.describe(), source.describe()]) + expect(requests).toBe(1) + }) +}) diff --git a/packages/capture-protocol/src/source.ts b/packages/capture-protocol/src/source.ts new file mode 100644 index 0000000000..21c140f774 --- /dev/null +++ b/packages/capture-protocol/src/source.ts @@ -0,0 +1,322 @@ +import { z } from 'zod' +import { + type CaptureArtifactReference, + type CaptureSessionDescriptor, + CaptureSessionDescriptorSchema, + type CaptureSessionLocator, + CaptureSessionLocatorSchema, + normalizeCaptureSessionManifest, +} from './schema' + +export const CaptureStreamPacketSchema = z.object({ + protocolVersion: z.literal(1), + sessionId: z.string().min(1), + streamId: z.string().min(1), + generation: z.number().int().nonnegative(), + sequence: z.number().int().nonnegative(), + timestamp: z.number().nonnegative(), + frameId: z.string().min(1).optional(), + keyframe: z.boolean().optional(), + bounds: z + .tuple([z.number(), z.number(), z.number(), z.number(), z.number(), z.number()]) + .optional(), + payload: z.unknown(), +}) + +export type CaptureStreamPacket = z.infer + +export type CaptureSourceEvent = + | { type: 'descriptor'; descriptor: CaptureSessionDescriptor } + | { type: 'packet'; packet: CaptureStreamPacket } + | { type: 'closed' } + +export type CaptureArtifactResolution = { + url: string + dispose?: () => void +} + +export type CaptureSubscriptionOptions = { + signal?: AbortSignal + streamIds?: readonly string[] +} + +export interface CaptureSource { + describe(signal?: AbortSignal): Promise + resolveArtifact?( + artifact: CaptureArtifactReference, + signal?: AbortSignal, + ): Promise + subscribe?(options?: CaptureSubscriptionOptions): AsyncIterable +} + +export type CaptureSourceResolver = ( + locator: CaptureSessionLocator, +) => CaptureSource | Promise + +export type HttpCaptureSourceOptions = { + credentials?: RequestCredentials + fetch?: typeof globalThis.fetch + headers?: HeadersInit + manifestUrl?: (locator: CaptureSessionLocator) => string + resolveArtifact?: ( + artifact: CaptureArtifactReference, + signal?: AbortSignal, + ) => Promise +} + +export type PushCaptureSourceOptions = { + maxQueuedEventsPerSubscriber?: number +} + +export function createHttpCaptureSource( + locatorInput: CaptureSessionLocator, + options: HttpCaptureSourceOptions = {}, +): CaptureSource { + const locator = CaptureSessionLocatorSchema.parse(locatorInput) + const manifestUrl = locator.manifestUrl ?? options.manifestUrl?.(locator) + if (!manifestUrl) throw new Error(`Capture session ${locator.sessionId} has no manifest URL.`) + let descriptorPromise: Promise | null = null + + const loadDescriptor = async (): Promise => { + const fetcher = options.fetch ?? globalThis.fetch + const response = await fetcher(manifestUrl, { + credentials: options.credentials, + headers: options.headers, + }) + if (!response.ok) throw new Error(`Capture session ${locator.sessionId} is unavailable.`) + const descriptor = normalizeCaptureSessionManifest(await response.json()) + if (descriptor.sessionId !== locator.sessionId) { + throw new Error(`Capture manifest session does not match ${locator.sessionId}.`) + } + if (locator.schemaVersion && descriptor.schemaVersion !== locator.schemaVersion) { + throw new Error(`Capture manifest schema does not match version ${locator.schemaVersion}.`) + } + if (locator.revisionId && descriptor.revisionId !== locator.revisionId) { + throw new Error(`Capture manifest revision does not match ${locator.revisionId}.`) + } + return descriptor + } + + return { + describe(signal) { + descriptorPromise ??= loadDescriptor().catch((cause: unknown) => { + descriptorPromise = null + throw cause + }) + return waitForPromise(descriptorPromise, signal).then(cloneCaptureDescriptor) + }, + async resolveArtifact(artifact, signal) { + if (options.resolveArtifact) return options.resolveArtifact(artifact, signal) + if (!artifact.uri) throw new Error(`Capture artifact ${artifact.id} has no URI.`) + return { url: resolveArtifactUri(artifact.uri, manifestUrl) } + }, + } +} + +function waitForPromise(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise + if (signal.aborted) return Promise.reject(abortError()) + return new Promise((resolve, reject) => { + const onAbort = () => reject(abortError()) + signal.addEventListener('abort', onAbort, { once: true }) + void promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + }) +} + +function abortError(): Error { + const error = new Error('The capture request was aborted.') + error.name = 'AbortError' + return error +} + +function resolveArtifactUri(uri: string, manifestUrl: string): string { + try { + const base = + typeof globalThis.location === 'undefined' + ? new URL(manifestUrl) + : new URL(manifestUrl, globalThis.location.href) + return new URL(uri, base).toString() + } catch { + return uri + } +} + +type Subscriber = { + iterator: EventIterator + streamIds: Set | null +} + +export class PushCaptureSource implements CaptureSource { + #closed = false + #descriptor: CaptureSessionDescriptor + #maxQueuedEventsPerSubscriber: number + #subscribers = new Set() + + constructor(descriptor: CaptureSessionDescriptor, options: PushCaptureSourceOptions = {}) { + this.#descriptor = cloneCaptureDescriptor(descriptor) + this.#maxQueuedEventsPerSubscriber = Math.max(1, options.maxQueuedEventsPerSubscriber ?? 32) + } + + async describe(): Promise { + return cloneCaptureDescriptor(this.#descriptor) + } + + async resolveArtifact(artifact: CaptureArtifactReference): Promise { + if (!artifact.uri) throw new Error(`Capture artifact ${artifact.id} has no URI.`) + return { url: artifact.uri } + } + + subscribe(options: CaptureSubscriptionOptions = {}): AsyncIterable { + let cleanupAbort = () => {} + let subscriber: Subscriber + const iterator = new EventIterator(() => { + cleanupAbort() + this.#subscribers.delete(subscriber) + }, this.#maxQueuedEventsPerSubscriber) + subscriber = { + iterator, + streamIds: options.streamIds ? new Set(options.streamIds) : null, + } + this.#subscribers.add(subscriber) + if (this.#closed) { + iterator.close({ type: 'closed' }) + } else if (options.signal) { + if (options.signal.aborted) iterator.finish() + else { + const onAbort = () => iterator.finish() + options.signal.addEventListener('abort', onAbort, { once: true }) + cleanupAbort = () => options.signal?.removeEventListener('abort', onAbort) + } + } + return iterator + } + + updateDescriptor(descriptor: CaptureSessionDescriptor): void { + if (this.#closed) return + const nextDescriptor = cloneCaptureDescriptor(descriptor) + if (nextDescriptor.sessionId !== this.#descriptor.sessionId) { + throw new Error('A capture source cannot change session identity.') + } + this.#descriptor = nextDescriptor + this.#publish({ type: 'descriptor', descriptor: cloneCaptureDescriptor(nextDescriptor) }) + } + + publishPacket(packetInput: CaptureStreamPacket): void { + if (this.#closed) return + const packet = CaptureStreamPacketSchema.parse(packetInput) + if (packet.sessionId !== this.#descriptor.sessionId) { + throw new Error(`Capture packet does not belong to ${this.#descriptor.sessionId}.`) + } + if (!this.#descriptor.streams.some((stream) => stream.id === packet.streamId)) { + throw new Error(`Capture packet references unknown stream ${packet.streamId}.`) + } + this.#publish({ type: 'packet', packet }) + } + + close(): void { + if (this.#closed) return + this.#closed = true + for (const subscriber of this.#subscribers) { + subscriber.iterator.close({ type: 'closed' }) + } + this.#subscribers.clear() + } + + #publish(event: CaptureSourceEvent): void { + for (const subscriber of this.#subscribers) { + if ( + event.type === 'packet' && + subscriber.streamIds && + !subscriber.streamIds.has(event.packet.streamId) + ) { + continue + } + subscriber.iterator.push(cloneCaptureSourceEvent(event)) + } + } +} + +class EventIterator implements AsyncIterableIterator { + #done = false + #maxQueuedEvents: number + #onFinish: () => void + #queue: CaptureSourceEvent[] = [] + #waiters: Array<(result: IteratorResult) => void> = [] + + constructor(onFinish: () => void, maxQueuedEvents: number) { + this.#onFinish = onFinish + this.#maxQueuedEvents = maxQueuedEvents + } + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this + } + + next(): Promise> { + const event = this.#queue.shift() + if (event) return Promise.resolve({ done: false, value: event }) + if (this.#done) return Promise.resolve({ done: true, value: undefined }) + return new Promise((resolve) => this.#waiters.push(resolve)) + } + + return(): Promise> { + this.finish() + return Promise.resolve({ done: true, value: undefined }) + } + + push(event: CaptureSourceEvent): void { + if (this.#done) return + const waiter = this.#waiters.shift() + if (waiter) waiter({ done: false, value: event }) + else { + this.#queue.push(event) + this.#trimQueue() + } + } + + close(event: CaptureSourceEvent): void { + if (this.#done) return + this.#done = true + this.#queue = [] + const waiter = this.#waiters.shift() + if (waiter) waiter({ done: false, value: event }) + else this.#queue.push(event) + for (const pending of this.#waiters.splice(0)) pending({ done: true, value: undefined }) + this.#onFinish() + } + + finish(): void { + if (this.#done) { + this.#queue = [] + return + } + this.#done = true + this.#queue = [] + this.#onFinish() + for (const waiter of this.#waiters.splice(0)) waiter({ done: true, value: undefined }) + } + + #trimQueue(): void { + while (this.#queue.length > this.#maxQueuedEvents) { + const packetIndex = this.#queue.findIndex((event) => event.type === 'packet') + this.#queue.splice(packetIndex >= 0 ? packetIndex : 0, 1) + } + } +} + +function cloneCaptureDescriptor(descriptor: CaptureSessionDescriptor): CaptureSessionDescriptor { + return CaptureSessionDescriptorSchema.parse(structuredClone(descriptor)) +} + +function cloneCaptureSourceEvent(event: CaptureSourceEvent): CaptureSourceEvent { + if (event.type === 'descriptor') { + return { type: 'descriptor', descriptor: cloneCaptureDescriptor(event.descriptor) } + } + if (event.type === 'packet') { + return { + type: 'packet', + packet: CaptureStreamPacketSchema.parse(structuredClone(event.packet)), + } + } + return { type: 'closed' } +} diff --git a/packages/capture-protocol/tsconfig.json b/packages/capture-protocol/tsconfig.json new file mode 100644 index 0000000000..06b4bb5999 --- /dev/null +++ b/packages/capture-protocol/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "@pascal/typescript-config/react-library.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "noEmit": false, + "composite": true, + "incremental": true, + "types": ["bun"] + }, + "include": ["src"], + "exclude": ["node_modules", "dist", "**/*.test.ts"] +} diff --git a/packages/capture-viewer/README.md b/packages/capture-viewer/README.md new file mode 100644 index 0000000000..01c04f61d5 --- /dev/null +++ b/packages/capture-viewer/README.md @@ -0,0 +1,33 @@ +# `@pascal-app/capture-viewer` + +Reference capture layers for `@pascal-app/viewer`. + +Mount `CaptureRuntime` as a child of `Viewer` and provide a source resolver. The host owns access +control and transport; the runtime owns source lifecycle, scan-node placement, layer visibility, +and reference renderers for RoomPlan models, device trajectories, and PLY/live point clouds. + +```tsx + + reportCaptureError(error, context)} + resolveSource={(locator) => + createHttpCaptureSource(locator, { credentials: 'include' }) + } + retryKey={retryVersion} + /> + +``` + +Unknown streams remain in the descriptor and can be rendered by passing a custom renderer keyed by +stream role or kind. A live transport implements `CaptureSource.subscribe()`; no particular +WebSocket, WebRTC, or collaboration backend is required by this package. + +`CaptureRuntime` keeps telemetry host-neutral: pass `onError` to report source or per-stream +failures in the host, then increment `retryKey` to reload every affected session. Direct +`useCaptureSource()` consumers can call its `retry()` function instead. + +Hosts can pass `defaultLayerVisibility` to keep expensive optional layers disabled until a user +enables them. Persisted values in the scan node's `layers` map always override those host defaults; +without host defaults, every available layer remains visible for backwards compatibility. +Hidden sessions and layers are unmounted rather than only made visually transparent, so they stop +raycasting, artifact work, animation, and live packet subscriptions while disabled. diff --git a/packages/capture-viewer/package.json b/packages/capture-viewer/package.json new file mode 100644 index 0000000000..4829d0fa15 --- /dev/null +++ b/packages/capture-viewer/package.json @@ -0,0 +1,63 @@ +{ + "name": "@pascal-app/capture-viewer", + "version": "1.0.0-beta.4", + "description": "Open capture-session runtime and reference renderers for the Pascal viewer", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsc --build", + "dev": "tsgo --build --watch", + "test": "bun test src", + "prepublishOnly": "npm run build" + }, + "peerDependencies": { + "@pascal-app/capture-protocol": "^1.0.0-beta.4", + "@pascal-app/core": "^1.0.0-beta.4", + "@pascal-app/viewer": "^1.0.0-beta.4", + "@react-three/drei": "^10", + "@react-three/fiber": "^9", + "react": "^18 || ^19", + "three": "^0.185" + }, + "devDependencies": { + "@pascal-app/capture-protocol": "^1.0.0-beta.4", + "@pascal-app/core": "^1.0.0-beta.4", + "@pascal-app/viewer": "^1.0.0-beta.4", + "@pascal/typescript-config": "*", + "@react-three/drei": "^10.7.7", + "@react-three/fiber": "^9.5.0", + "@types/bun": "^1.3.0", + "@types/react": "^19.2.2", + "@types/three": "^0.184.0", + "react": "^19.2.4", + "three": "^0.185.0", + "typescript": "6.0.3" + }, + "keywords": [ + "3d", + "capture", + "point-cloud", + "react-three-fiber", + "viewer" + ], + "repository": { + "type": "git", + "url": "https://github.com/pascalorg/editor.git", + "directory": "packages/capture-viewer" + }, + "license": "MIT", + "homepage": "https://github.com/pascalorg/editor/tree/main/packages/capture-viewer#readme", + "bugs": "https://github.com/pascalorg/editor/issues" +} diff --git a/packages/capture-viewer/src/asset-url.ts b/packages/capture-viewer/src/asset-url.ts new file mode 100644 index 0000000000..c0eec374ea --- /dev/null +++ b/packages/capture-viewer/src/asset-url.ts @@ -0,0 +1,14 @@ +export function rewriteLoopbackAssetUrl(value: string): string { + try { + const url = new URL(value) + if ( + typeof window !== 'undefined' && + (url.hostname === '127.0.0.1' || url.hostname === 'localhost') + ) { + url.hostname = window.location.hostname + } + return url.toString() + } catch { + return value + } +} diff --git a/packages/capture-viewer/src/capture-runtime.tsx b/packages/capture-viewer/src/capture-runtime.tsx new file mode 100644 index 0000000000..93455ebb47 --- /dev/null +++ b/packages/capture-viewer/src/capture-runtime.tsx @@ -0,0 +1,374 @@ +'use client' + +import { + type CaptureArtifactReference, + CaptureArtifactReferenceSchema, + type CaptureArtifactResolution, + type CaptureSessionDescriptor, + type CaptureSessionLocator, + type CaptureSource, + type CaptureSourceResolver, + type CaptureStreamDescriptor, + type CaptureStreamPacket, + captureLayerKey, + DeviceMotionTrajectorySchema, +} from '@pascal-app/capture-protocol' +import { type ScanNode, sceneRegistry, useScene } from '@pascal-app/core' +import { ErrorBoundary, useNodeEvents, useViewer } from '@pascal-app/viewer' +import { createPortal, useFrame } from '@react-three/fiber' +import { + type ComponentType, + type ReactNode, + Suspense, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react' +import type { Object3D } from 'three' +import { resolveCaptureFrameMatrix } from './frame' +import { isCaptureSessionVisible, isCaptureStreamVisible } from './layer-visibility' +import { CaptureDeviceMotionLayer } from './layers/device-motion-layer' +import { CapturePointCloudLayer } from './layers/point-cloud-layer' +import { CaptureRoomModel } from './layers/room-model-layer' +import { CaptureSurfaceMeshLayer } from './layers/surface-mesh-layer' +import { useCaptureSource } from './source-state' +import { + captureModelFormat, + isCaptureModelArtifact, + isCapturePointCloudArtifact, + isCaptureStreamRenderable, +} from './stream-rendering' +import { parseDeviceTrajectoryPackets, parseDeviceTrajectoryPayload } from './trajectory' + +export type CaptureStreamRendererProps = { + artifactUrl: string | null + descriptor: CaptureSessionDescriptor + packets: readonly CaptureStreamPacket[] + scan: ScanNode + source: CaptureSource + stream: CaptureStreamDescriptor + streamEpoch: string +} + +export type CaptureStreamRenderer = ComponentType + +export type CaptureRuntimeErrorContext = + | { + phase: 'source' + scanId: ScanNode['id'] + sessionId: string + } + | { + layerKey: string + phase: 'stream' + scanId: ScanNode['id'] + sessionId: string + streamId: string + streamKind: string + } + +export type CaptureRuntimeProps = { + defaultLayerVisibility?: Readonly> + maxPacketsPerStream?: number + onError?: (error: Error, context: CaptureRuntimeErrorContext) => void + renderers?: Readonly> + resolveSource: CaptureSourceResolver + retryKey?: number | string +} + +const EMPTY_RENDERERS: Readonly> = {} +const EMPTY_LAYER_VISIBILITY: Readonly> = {} +type CaptureSessionScan = ScanNode & { captureSession: CaptureSessionLocator } + +export function CaptureRuntime({ + defaultLayerVisibility = EMPTY_LAYER_VISIBILITY, + maxPacketsPerStream = 32, + onError, + renderers = EMPTY_RENDERERS, + resolveSource, + retryKey = 0, +}: CaptureRuntimeProps) { + const nodes = useScene((state) => state.nodes) + const showScans = useViewer((state) => state.showScans) + const scans = useMemo( + () => + Object.values(nodes).filter( + (node): node is CaptureSessionScan => + node.type === 'scan' && + node.captureSession !== null && + isCaptureSessionVisible(showScans, node.visible), + ), + [nodes, showScans], + ) + + return ( + <> + {scans.map((scan) => ( + + ))} + + ) +} + +function CaptureSessionPortal({ + defaultLayerVisibility, + maxPacketsPerStream, + onError, + renderers, + resolveSource, + scan, +}: { + defaultLayerVisibility: Readonly> + maxPacketsPerStream: number + onError?: (error: Error, context: CaptureRuntimeErrorContext) => void + renderers: Readonly> + resolveSource: CaptureSourceResolver + scan: CaptureSessionScan +}) { + const [target, setTarget] = useState(null) + const onErrorRef = useRef(onError) + const handlers = useNodeEvents(scan, 'scan') + const customRendererKeys = useMemo(() => new Set(Object.keys(renderers)), [renderers]) + const streamFilter = useCallback( + (stream: CaptureStreamDescriptor) => + isCaptureStreamVisible(stream, scan.layers, defaultLayerVisibility) && + isCaptureStreamRenderable(stream, customRendererKeys), + [customRendererKeys, defaultLayerVisibility, scan.layers], + ) + const sourceState = useCaptureSource(scan.captureSession, resolveSource, { + maxPacketsPerStream, + streamFilter, + }) + + useEffect(() => { + onErrorRef.current = onError + }, [onError]) + + useEffect(() => { + if (!sourceState.error) return + onErrorRef.current?.(sourceState.error, { + phase: 'source', + scanId: scan.id, + sessionId: scan.captureSession.sessionId, + }) + }, [scan.captureSession.sessionId, scan.id, sourceState.error]) + + useFrame(() => { + const nextTarget = sceneRegistry.nodes.get(scan.id) ?? null + if (nextTarget !== target) setTarget(nextTarget) + }) + + const descriptor = sourceState.descriptor + const source = sourceState.source + if (!(target && descriptor && source)) return null + + const visibleStreams = descriptor.streams.filter(streamFilter) + + return createPortal( + + {visibleStreams.map((stream) => { + const layerKey = captureLayerKey(stream) + const renderKey = captureStreamRenderKey(stream) + const packets = sourceState.packets[stream.id] ?? [] + const streamEpoch = + sourceState.streamEpochs[stream.id] ?? + `descriptor:${descriptor.revisionId ?? ''}:${stream.id}:${stream.frameId ?? ''}` + const latestPacket = packets.at(-1) + const packetRevision = latestPacket + ? `${latestPacket.generation}:${latestPacket.sequence}:${latestPacket.frameId ?? ''}` + : 'static' + return ( + } + key={renderKey} + onError={(error) => + onErrorRef.current?.(error, { + layerKey, + phase: 'stream', + scanId: scan.id, + sessionId: scan.captureSession.sessionId, + streamId: stream.id, + streamKind: stream.kind, + }) + } + resetKey={`${renderKey}:${sourceState.descriptorVersion}:${streamEpoch}:${packetRevision}`} + scope={`capture:${layerKey}`} + > + + + + + ) + })} + , + target, + ) +} + +function captureStreamRenderKey(stream: CaptureStreamDescriptor): string { + const artifact = stream.artifact + return [ + stream.id, + stream.availability, + artifact?.id ?? '', + artifact?.sha256 ?? '', + artifact?.uri ?? '', + ].join(':') +} + +export function CaptureStreamLayer({ + descriptor, + packets, + renderers, + scan, + source, + stream, + streamEpoch, +}: Omit & { + renderers: Readonly> +}) { + const artifactUrl = useResolvedArtifact(source, stream.artifact) + const layerKey = captureLayerKey(stream) + const Renderer = renderers[layerKey] ?? renderers[stream.kind] + const frameId = packets.at(-1)?.frameId ?? stream.frameId ?? stream.artifact?.frameId + const frameMatrix = useMemo( + () => resolveCaptureFrameMatrix(descriptor, frameId), + [descriptor, frameId], + ) + const trajectory = useMemo(() => { + if (layerKey !== 'deviceMotion') return null + const inline = DeviceMotionTrajectorySchema.safeParse(stream.inline) + return inline.success + ? parseDeviceTrajectoryPayload(inline.data) + : parseDeviceTrajectoryPackets(packets.map((packet) => packet.payload)) + }, [layerKey, packets, stream.inline]) + const motionPlaybackKey = useMemo(() => { + if (layerKey !== 'deviceMotion') return '' + const inlineVersion = packets.length === 0 ? JSON.stringify(stream.inline ?? null) : '' + return [descriptor.revisionId ?? '', streamEpoch, inlineVersion].join(':') + }, [descriptor.revisionId, layerKey, packets.length, stream.inline, streamEpoch]) + if (frameId && !frameMatrix) { + throw new Error(`Capture stream ${stream.id} references an invalid frame: ${frameId}.`) + } + let content: ReactNode = null + if (Renderer) { + content = ( + + ) + } else if ( + layerKey === 'model' && + isCaptureModelArtifact(stream.artifact) && + stream.artifact && + artifactUrl + ) { + content = ( + + ) + } else if (layerKey === 'deviceMotion') { + content = trajectory ? ( + + ) : null + } else if (layerKey === 'pointCloud') { + content = ( + + ) + } else if (layerKey === 'surfaceMesh') { + content = + } + if (!(content && frameMatrix)) return content + return ( + + {content} + + ) +} + +function useResolvedArtifact( + source: CaptureSource, + artifact: CaptureArtifactReference | undefined, +): string | null { + const [error, setError] = useState(null) + const [url, setUrl] = useState(null) + const artifactKey = artifact ? JSON.stringify(artifact) : null + const artifactSnapshot = useMemo( + () => + artifactKey + ? CaptureArtifactReferenceSchema.parse(JSON.parse(artifactKey) as unknown) + : undefined, + [artifactKey], + ) + + useEffect(() => { + const abort = new AbortController() + let dispose: (() => void) | undefined + setError(null) + setUrl(null) + if (!artifactSnapshot) return () => abort.abort() + + const resolve: Promise = source.resolveArtifact + ? source.resolveArtifact(artifactSnapshot, abort.signal) + : artifactSnapshot.uri + ? Promise.resolve({ url: artifactSnapshot.uri }) + : Promise.reject(new Error(`Capture artifact ${artifactSnapshot.id} has no URI.`)) + void resolve + .then((result) => { + if (abort.signal.aborted) { + result.dispose?.() + return + } + dispose = result.dispose + setUrl(result.url) + }) + .catch((cause: unknown) => { + if (!abort.signal.aborted) { + setError( + cause instanceof Error ? cause : new Error('Could not resolve capture artifact.'), + ) + } + }) + return () => { + abort.abort() + dispose?.() + } + }, [artifactSnapshot, source]) + + if (error) throw error + return url +} diff --git a/packages/capture-viewer/src/frame.test.ts b/packages/capture-viewer/src/frame.test.ts new file mode 100644 index 0000000000..2dfe47e19a --- /dev/null +++ b/packages/capture-viewer/src/frame.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from 'bun:test' +import type { CaptureSessionDescriptor } from '@pascal-app/capture-protocol' +import { Vector3 } from 'three' +import { resolveCaptureFrameMatrix } from './frame' + +const descriptor: CaptureSessionDescriptor = { + schemaVersion: 2, + sessionId: 'capture_123', + state: 'ready', + clocks: [], + coordinateFrames: [ + { + id: 'world', + convention: 'right-handed-y-up', + transform: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 10, 0, 0, 1], + }, + { + id: 'sensor', + parentId: 'world', + convention: 'arkit-camera', + transform: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 2, 0, 1], + }, + ], + streams: [], +} + +describe('resolveCaptureFrameMatrix', () => { + test('composes local-to-parent transforms into session space', () => { + const position = new Vector3(0, 0, 0).applyMatrix4( + resolveCaptureFrameMatrix(descriptor, 'sensor')!, + ) + expect(position.toArray()).toEqual([10, 2, 0]) + }) + + test('returns null for missing or cyclic frame chains', () => { + expect(resolveCaptureFrameMatrix(descriptor, 'missing')).toBeNull() + expect( + resolveCaptureFrameMatrix( + { + ...descriptor, + coordinateFrames: [ + { id: 'a', parentId: 'b', convention: 'test' }, + { id: 'b', parentId: 'a', convention: 'test' }, + ], + }, + 'a', + ), + ).toBeNull() + }) +}) diff --git a/packages/capture-viewer/src/frame.ts b/packages/capture-viewer/src/frame.ts new file mode 100644 index 0000000000..992c40e136 --- /dev/null +++ b/packages/capture-viewer/src/frame.ts @@ -0,0 +1,24 @@ +import type { CaptureSessionDescriptor } from '@pascal-app/capture-protocol' +import { Matrix4 } from 'three' + +export function resolveCaptureFrameMatrix( + descriptor: CaptureSessionDescriptor, + frameId: string | undefined, +): Matrix4 | null { + if (!frameId) return null + const frames = new Map(descriptor.coordinateFrames.map((frame) => [frame.id, frame])) + const visited = new Set() + const matrix = new Matrix4() + let currentId: string | undefined = frameId + + while (currentId) { + if (visited.has(currentId)) return null + visited.add(currentId) + const frame = frames.get(currentId) + if (!frame) return null + if (frame.transform) matrix.premultiply(new Matrix4().fromArray(frame.transform)) + currentId = frame.parentId + } + + return matrix +} diff --git a/packages/capture-viewer/src/index.ts b/packages/capture-viewer/src/index.ts new file mode 100644 index 0000000000..93b16c1719 --- /dev/null +++ b/packages/capture-viewer/src/index.ts @@ -0,0 +1,52 @@ +export { rewriteLoopbackAssetUrl } from './asset-url' +export { + CaptureRuntime, + type CaptureRuntimeErrorContext, + type CaptureRuntimeProps, + CaptureStreamLayer, + type CaptureStreamRenderer, + type CaptureStreamRendererProps, +} from './capture-runtime' +export { resolveCaptureFrameMatrix } from './frame' +export { + isCaptureLayerVisible, + isCaptureSessionVisible, + isCaptureStreamVisible, +} from './layer-visibility' +export { + CaptureDeviceMotionLayer, + DEVICE_MOTION_PLAYBACK_SPEED, +} from './layers/device-motion-layer' +export { + buildPointCloudData, + CapturePointCloudLayer, + type PointCloudData, +} from './layers/point-cloud-layer' +export { CaptureRoomModel } from './layers/room-model-layer' +export { + buildSurfaceMeshData, + CaptureSurfaceMeshLayer, + type SurfaceMeshData, +} from './layers/surface-mesh-layer' +export { + appendCapturePacket, + type CaptureSourceState, + captureSubscriptionStreamIds, + type UseCaptureSourceOptions, + useCaptureSource, +} from './source-state' +export { + type CaptureModelFormat, + captureModelFormat, + isCaptureModelArtifact, + isCapturePointCloudArtifact, + isCaptureStreamRenderable, +} from './stream-rendering' +export { + type DeviceTrajectory, + type DeviceTrajectoryFrame, + type DeviceTrajectoryPose, + parseDeviceTrajectoryPackets, + parseDeviceTrajectoryPayload, + sampleDeviceTrajectory, +} from './trajectory' diff --git a/packages/capture-viewer/src/layer-visibility.test.ts b/packages/capture-viewer/src/layer-visibility.test.ts new file mode 100644 index 0000000000..d447afe057 --- /dev/null +++ b/packages/capture-viewer/src/layer-visibility.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from 'bun:test' +import { + isCaptureLayerVisible, + isCaptureSessionVisible, + isCaptureStreamVisible, +} from './layer-visibility' + +describe('isCaptureLayerVisible', () => { + test('keeps capture layers visible when the host has no default', () => { + expect(isCaptureLayerVisible({}, 'pointCloud')).toBe(true) + }) + + test('uses the host default for an unset layer', () => { + expect(isCaptureLayerVisible({}, 'pointCloud', { pointCloud: false })).toBe(false) + }) + + test('lets persisted scene visibility override the host default', () => { + expect(isCaptureLayerVisible({ pointCloud: true }, 'pointCloud', { pointCloud: false })).toBe( + true, + ) + expect(isCaptureLayerVisible({ model: false }, 'model', { model: true })).toBe(false) + }) +}) + +describe('isCaptureStreamVisible', () => { + test('resolves a stream through its capture layer', () => { + const stream = { + id: 'points', + kind: 'point-cloud', + role: 'pointCloud', + availability: 'ready', + } as const + + expect(isCaptureStreamVisible(stream, {}, { pointCloud: false })).toBe(false) + expect(isCaptureStreamVisible(stream, { pointCloud: true }, { pointCloud: false })).toBe(true) + }) +}) + +describe('isCaptureSessionVisible', () => { + test('requires both the global scan display and node visibility', () => { + expect(isCaptureSessionVisible(true, true)).toBe(true) + expect(isCaptureSessionVisible(false, true)).toBe(false) + expect(isCaptureSessionVisible(true, false)).toBe(false) + }) +}) diff --git a/packages/capture-viewer/src/layer-visibility.ts b/packages/capture-viewer/src/layer-visibility.ts new file mode 100644 index 0000000000..5524e7fd86 --- /dev/null +++ b/packages/capture-viewer/src/layer-visibility.ts @@ -0,0 +1,24 @@ +import type { CaptureStreamDescriptor } from '@pascal-app/capture-protocol' +import { captureLayerKey } from '@pascal-app/capture-protocol' + +const EMPTY_LAYER_VISIBILITY: Readonly> = {} + +export function isCaptureLayerVisible( + layers: Readonly>, + layerKey: string, + defaultLayerVisibility: Readonly> = EMPTY_LAYER_VISIBILITY, +): boolean { + return layers[layerKey] ?? defaultLayerVisibility[layerKey] ?? true +} + +export function isCaptureStreamVisible( + stream: CaptureStreamDescriptor, + layers: Readonly>, + defaultLayerVisibility: Readonly> = EMPTY_LAYER_VISIBILITY, +): boolean { + return isCaptureLayerVisible(layers, captureLayerKey(stream), defaultLayerVisibility) +} + +export function isCaptureSessionVisible(showScans: boolean, scanVisible: boolean): boolean { + return showScans && scanVisible +} diff --git a/packages/capture-viewer/src/layers/device-motion-layer.tsx b/packages/capture-viewer/src/layers/device-motion-layer.tsx new file mode 100644 index 0000000000..723456b1d9 --- /dev/null +++ b/packages/capture-viewer/src/layers/device-motion-layer.tsx @@ -0,0 +1,141 @@ +'use client' + +import { useFrame } from '@react-three/fiber' +import { useEffect, useMemo, useRef } from 'react' +import { + BufferGeometry, + type Group, + LineBasicMaterial, + LineSegments, + Quaternion, + Line as ThreeLine, + Vector3, +} from 'three' +import { type DeviceTrajectory, sampleDeviceTrajectory } from '../trajectory' + +export const DEVICE_MOTION_PLAYBACK_SPEED = 3 + +export function CaptureDeviceMotionLayer({ + lineWidth = 2.5, + trajectory, +}: { + lineWidth?: number + trajectory: DeviceTrajectory +}) { + const deviceRef = useRef(null) + const elapsedRef = useRef(0) + const position = useMemo(() => new Vector3(), []) + const toPosition = useMemo(() => new Vector3(), []) + const fromQuaternion = useMemo(() => new Quaternion(), []) + const toQuaternion = useMemo(() => new Quaternion(), []) + const trajectorySegments = useMemo(() => { + const segments = new Map() + for (const pose of trajectory.poses) { + const points = segments.get(pose.segment) ?? [] + points.push(pose.position) + segments.set(pose.segment, points) + } + return [...segments.entries()] + .filter(([, points]) => points.length > 1) + .map(([segment, points]) => ({ points, segment })) + }, [trajectory]) + + useFrame((_, delta) => { + if (!deviceRef.current) return + elapsedRef.current += delta * DEVICE_MOTION_PLAYBACK_SPEED + + const frame = sampleDeviceTrajectory(trajectory, elapsedRef.current) + position + .fromArray(frame.from.position) + .lerp(toPosition.fromArray(frame.to.position), frame.alpha) + fromQuaternion.fromArray(frame.from.quaternion) + toQuaternion.fromArray(frame.to.quaternion) + deviceRef.current.position.copy(position) + deviceRef.current.quaternion.slerpQuaternions(fromQuaternion, toQuaternion, frame.alpha) + }) + + return ( + + {trajectorySegments.map(({ points, segment }) => ( + + ))} + + + + + ) +} + +function CameraFrustum({ lineWidth }: { lineWidth: number }) { + const apex: [number, number, number] = [0, 0, 0] + const topRight: [number, number, number] = [0.14, 0.1, -0.28] + const topLeft: [number, number, number] = [-0.14, 0.1, -0.28] + const bottomLeft: [number, number, number] = [-0.14, -0.1, -0.28] + const bottomRight: [number, number, number] = [0.14, -0.1, -0.28] + const corners: [number, number, number][] = [topRight, topLeft, bottomLeft, bottomRight] + const points = [ + ...corners.map((corner) => [apex, corner] as const), + [topRight, topLeft] as const, + [topLeft, bottomLeft] as const, + [bottomLeft, bottomRight] as const, + [bottomRight, topRight] as const, + ].flat() + + return ( + + + + + + + + ) +} + +function CaptureLine({ + color, + lineWidth, + opacity = 1, + points, + segments = false, +}: { + color: string + lineWidth: number + opacity?: number + points: readonly [number, number, number][] + segments?: boolean +}) { + const line = useMemo(() => { + const geometry = new BufferGeometry().setFromPoints( + points.map(([x, y, z]) => new Vector3(x, y, z)), + ) + const material = new LineBasicMaterial({ + color, + depthWrite: opacity >= 1, + linewidth: lineWidth, + opacity, + transparent: opacity < 1, + }) + const object = segments + ? new LineSegments(geometry, material) + : new ThreeLine(geometry, material) + object.frustumCulled = false + return object + }, [color, lineWidth, opacity, points, segments]) + + useEffect( + () => () => { + line.geometry.dispose() + line.material.dispose() + }, + [line], + ) + + return +} diff --git a/packages/capture-viewer/src/layers/point-cloud-layer.tsx b/packages/capture-viewer/src/layers/point-cloud-layer.tsx new file mode 100644 index 0000000000..34274604c4 --- /dev/null +++ b/packages/capture-viewer/src/layers/point-cloud-layer.tsx @@ -0,0 +1,155 @@ +'use client' + +import type { CaptureStreamPacket } from '@pascal-app/capture-protocol' +import { useLoader } from '@react-three/fiber' +import { useEffect, useMemo } from 'react' +import { BufferGeometry, Float32BufferAttribute } from 'three' +import { PLYLoader } from 'three/addons/loaders/PLYLoader.js' +import { rewriteLoopbackAssetUrl } from '../asset-url' + +export type PointCloudData = { + colors: Float32Array | null + positions: Float32Array +} + +export function CapturePointCloudLayer({ + artifactUrl, + inline, + maxPoints = 250_000, + packets = [], + pointSize = 0.012, +}: { + artifactUrl?: string + inline?: unknown + maxPoints?: number + packets?: readonly CaptureStreamPacket[] + pointSize?: number +}) { + const liveData = useMemo(() => buildPointCloudData(packets, maxPoints), [maxPoints, packets]) + const inlineData = useMemo( + () => buildPointCloudPayloadData(inline, maxPoints), + [inline, maxPoints], + ) + if (liveData.positions.length > 0) { + return + } + if (inlineData.positions.length > 0) { + return + } + return artifactUrl ? : null +} + +function PlyPointCloud({ pointSize, url }: { pointSize: number; url: string }) { + const source = useLoader(PLYLoader, rewriteLoopbackAssetUrl(url)) + const geometry = useMemo(() => source.clone(), [source]) + useEffect(() => () => geometry.dispose(), [geometry]) + + return ( + + + + ) +} + +function PointCloudDataLayer({ data, pointSize }: { data: PointCloudData; pointSize: number }) { + const geometry = useMemo(() => { + const next = new BufferGeometry() + next.setAttribute('position', new Float32BufferAttribute(data.positions, 3)) + if (data.colors) next.setAttribute('color', new Float32BufferAttribute(data.colors, 3)) + next.computeBoundingSphere() + return next + }, [data]) + useEffect(() => () => geometry.dispose(), [geometry]) + + return ( + + + + ) +} + +export function buildPointCloudData( + packets: readonly CaptureStreamPacket[], + maxPoints: number, +): PointCloudData { + const chunks: Array<{ colors: number[] | null; positions: number[] }> = [] + let pointCount = 0 + let allHaveColors = true + + for (let index = packets.length - 1; index >= 0 && pointCount < maxPoints; index -= 1) { + const parsed = parsePointPayload(packets[index]?.payload) + if (!parsed) continue + const availablePoints = Math.floor(parsed.positions.length / 3) + const keepPoints = Math.min(availablePoints, maxPoints - pointCount) + if (keepPoints <= 0) continue + const start = (availablePoints - keepPoints) * 3 + chunks.unshift({ + colors: parsed.colors?.slice(start) ?? null, + positions: parsed.positions.slice(start), + }) + pointCount += keepPoints + allHaveColors &&= parsed.colors !== null + } + + const positions = new Float32Array(pointCount * 3) + const colors = allHaveColors && pointCount > 0 ? new Float32Array(pointCount * 3) : null + let offset = 0 + for (const chunk of chunks) { + positions.set(chunk.positions, offset) + if (colors && chunk.colors) colors.set(normalizeColors(chunk.colors), offset) + offset += chunk.positions.length + } + return { colors, positions } +} + +export function buildPointCloudPayloadData(value: unknown, maxPoints: number): PointCloudData { + const parsed = parsePointPayload(value) + if (!parsed) return { colors: null, positions: new Float32Array() } + + const availablePoints = Math.floor(parsed.positions.length / 3) + const keepPoints = Math.min(availablePoints, maxPoints) + const start = (availablePoints - keepPoints) * 3 + const positions = new Float32Array(parsed.positions.slice(start)) + const colors = parsed.colors + ? new Float32Array(normalizeColors(parsed.colors.slice(start))) + : null + return { colors, positions } +} + +function parsePointPayload( + value: unknown, +): { colors: number[] | null; positions: number[] } | null { + if (!(value && typeof value === 'object')) return null + const payload = value as { colors?: unknown; positions?: unknown } + const positions = numericArray(payload.positions) + if (!(positions && positions.length >= 3 && positions.length % 3 === 0)) return null + const colors = numericArray(payload.colors) + return { + colors: colors && colors.length === positions.length ? colors : null, + positions, + } +} + +function numericArray(value: unknown): number[] | null { + if (Array.isArray(value) && value.every((entry) => Number.isFinite(entry))) return value + if (ArrayBuffer.isView(value)) { + const entries = Array.from(value as unknown as ArrayLike) + return entries.every(Number.isFinite) ? entries : null + } + return null +} + +function normalizeColors(colors: number[]): number[] { + const divisor = colors.some((value) => value > 1) ? 255 : 1 + return colors.map((value) => Math.min(1, Math.max(0, value / divisor))) +} diff --git a/packages/capture-viewer/src/layers/room-model-layer.tsx b/packages/capture-viewer/src/layers/room-model-layer.tsx new file mode 100644 index 0000000000..a91dee9a87 --- /dev/null +++ b/packages/capture-viewer/src/layers/room-model-layer.tsx @@ -0,0 +1,86 @@ +'use client' + +import { useGLTFKTX2 } from '@pascal-app/viewer' +import { useLoader } from '@react-three/fiber' +import { useEffect, useMemo } from 'react' +import type { Material, Mesh, Object3D } from 'three' +import { USDLoader } from 'three/addons/loaders/USDLoader.js' +import { rewriteLoopbackAssetUrl } from '../asset-url' +import type { CaptureModelFormat } from '../stream-rendering' + +export function CaptureRoomModel({ + format, + mediaType, + opacity = 100, + url, +}: { + format?: CaptureModelFormat + mediaType: string + opacity?: number + url: string +}) { + if ( + format === 'usdz' || + mediaType === 'model/vnd.usdz+zip' || + url.toLowerCase().endsWith('.usdz') + ) { + return + } + return +} + +function UsdzRoomModel({ opacity, url }: { opacity: number; url: string }) { + const source = useLoader(USDLoader, rewriteLoopbackAssetUrl(url)) + const model = useClonedModel(source, opacity) + return +} + +function GlbRoomModel({ opacity, url }: { opacity: number; url: string }) { + const gltf = useGLTFKTX2(rewriteLoopbackAssetUrl(url)) as { scene: Object3D } + const model = useClonedModel(gltf.scene, opacity) + return +} + +function useClonedModel(source: Object3D, opacity: number): Object3D { + const model = useMemo(() => { + const clone = source.clone(true) + clone.traverse((child) => { + const mesh = child as Mesh + if (!mesh.isMesh) return + mesh.material = Array.isArray(mesh.material) + ? mesh.material.map((material) => material.clone()) + : mesh.material.clone() + }) + return clone + }, [source]) + + useEffect(() => { + const normalizedOpacity = opacity / 100 + const transparent = normalizedOpacity < 1 + model.traverse((child) => { + const mesh = child as Mesh + if (!mesh.isMesh) return + const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material] + for (const material of materials) { + material.transparent = transparent + material.opacity = normalizedOpacity + material.depthWrite = !transparent + material.needsUpdate = true + } + }) + }, [model, opacity]) + + useEffect( + () => () => { + model.traverse((child) => { + const mesh = child as Mesh + if (!mesh.isMesh) return + const materials: Material[] = Array.isArray(mesh.material) ? mesh.material : [mesh.material] + for (const material of materials) material.dispose() + }) + }, + [model], + ) + + return model +} diff --git a/packages/capture-viewer/src/layers/surface-mesh-layer.tsx b/packages/capture-viewer/src/layers/surface-mesh-layer.tsx new file mode 100644 index 0000000000..f8cf463608 --- /dev/null +++ b/packages/capture-viewer/src/layers/surface-mesh-layer.tsx @@ -0,0 +1,96 @@ +'use client' + +import { SurfaceMeshPayloadSchema } from '@pascal-app/capture-protocol' +import { useEffect, useMemo } from 'react' +import { BufferGeometry, DoubleSide, Float32BufferAttribute, Uint16BufferAttribute } from 'three' + +export type SurfaceMeshData = { + colors: Float32Array + indices: Uint16Array + positions: Float32Array +} + +export function CaptureSurfaceMeshLayer({ inline }: { inline: unknown }) { + const data = useMemo(() => buildSurfaceMeshData(inline), [inline]) + const geometry = useMemo(() => { + if (!data) return null + const next = new BufferGeometry() + next.setAttribute('position', new Float32BufferAttribute(data.positions, 3)) + next.setAttribute('color', new Float32BufferAttribute(data.colors, 3)) + next.setIndex(new Uint16BufferAttribute(data.indices, 1)) + next.computeVertexNormals() + next.computeBoundingSphere() + return next + }, [data]) + useEffect(() => () => geometry?.dispose(), [geometry]) + if (!geometry) return null + + return ( + + + + ) +} + +export function buildSurfaceMeshData(value: unknown): SurfaceMeshData | null { + const parsed = SurfaceMeshPayloadSchema.safeParse(value) + if (!parsed.success) return null + const payload = parsed.data + const positionBytes = decodeBase64(payload.positions) + const colorBytes = decodeBase64(payload.colors) + const indexBytes = decodeBase64(payload.indices) + if ( + positionBytes.byteLength !== payload.vertexCount * 3 * 2 || + colorBytes.byteLength !== payload.vertexCount * 3 || + indexBytes.byteLength !== payload.faceCount * 3 * 2 + ) { + return null + } + + const positions = new Float32Array(payload.vertexCount * 3) + const colors = new Float32Array(payload.vertexCount * 3) + const indices = new Uint16Array(payload.faceCount * 3) + const positionView = new DataView( + positionBytes.buffer, + positionBytes.byteOffset, + positionBytes.byteLength, + ) + const indexView = new DataView(indexBytes.buffer, indexBytes.byteOffset, indexBytes.byteLength) + for (let index = 0; index < payload.vertexCount; index += 1) { + for (let axis = 0; axis < 3; axis += 1) { + const offset = index * 3 + axis + const minimum = payload.boundsMin[axis] ?? 0 + const maximum = payload.boundsMax[axis] ?? minimum + const quantized = positionView.getUint16(offset * 2, true) + positions[offset] = minimum + (quantized / 65_535) * (maximum - minimum) + colors[offset] = (colorBytes[offset] ?? 0) / 255 + } + } + for (let index = 0; index < indices.length; index += 1) { + const vertexIndex = indexView.getUint16(index * 2, true) + if (vertexIndex >= payload.vertexCount) return null + indices[index] = vertexIndex + } + return { colors, indices, positions } +} + +function decodeBase64(value: string): Uint8Array { + const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + const clean = value.replace(/\s/g, '') + const padding = clean.endsWith('==') ? 2 : clean.endsWith('=') ? 1 : 0 + const outputLength = Math.floor((clean.length * 3) / 4) - padding + const output = new Uint8Array(Math.max(0, outputLength)) + let outputIndex = 0 + for (let index = 0; index < clean.length; index += 4) { + const a = alphabet.indexOf(clean[index] ?? '') + const b = alphabet.indexOf(clean[index + 1] ?? '') + const c = clean[index + 2] === '=' ? 0 : alphabet.indexOf(clean[index + 2] ?? '') + const d = clean[index + 3] === '=' ? 0 : alphabet.indexOf(clean[index + 3] ?? '') + if (a < 0 || b < 0 || c < 0 || d < 0) return new Uint8Array() + const bits = (a << 18) | (b << 12) | (c << 6) | d + if (outputIndex < output.length) output[outputIndex++] = (bits >> 16) & 0xff + if (outputIndex < output.length) output[outputIndex++] = (bits >> 8) & 0xff + if (outputIndex < output.length) output[outputIndex++] = bits & 0xff + } + return output +} diff --git a/packages/capture-viewer/src/point-cloud-layer.test.ts b/packages/capture-viewer/src/point-cloud-layer.test.ts new file mode 100644 index 0000000000..bb47013797 --- /dev/null +++ b/packages/capture-viewer/src/point-cloud-layer.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from 'bun:test' +import type { CaptureStreamPacket } from '@pascal-app/capture-protocol' +import { buildPointCloudData, buildPointCloudPayloadData } from './layers/point-cloud-layer' + +function packet(sequence: number, positions: number[], colors?: number[]): CaptureStreamPacket { + return { + protocolVersion: 1, + sessionId: 'capture_123', + streamId: 'points', + generation: 0, + sequence, + timestamp: sequence, + payload: { colors, positions }, + } +} + +describe('buildPointCloudData', () => { + test('keeps the newest bounded points and normalizes byte colors', () => { + const data = buildPointCloudData( + [packet(0, [0, 0, 0], [255, 0, 0]), packet(1, [1, 0, 0, 2, 0, 0], [0, 255, 0, 0, 0, 255])], + 2, + ) + + expect([...data.positions]).toEqual([1, 0, 0, 2, 0, 0]) + expect(data.colors ? [...data.colors] : null).toEqual([0, 1, 0, 0, 0, 1]) + }) + + test('renders bounded inline capture points', () => { + const data = buildPointCloudPayloadData( + { + coordinateSystem: 'arkit-world', + positions: [0, 0, 0, 1, 0, 0, 2, 0, 0], + }, + 2, + ) + + expect([...data.positions]).toEqual([1, 0, 0, 2, 0, 0]) + expect(data.colors).toBeNull() + }) + + test('drops non-finite live packet geometry', () => { + expect(buildPointCloudData([packet(2, [0, 0, Number.NaN])], 100).positions).toHaveLength(0) + }) +}) diff --git a/packages/capture-viewer/src/source-state.test.ts b/packages/capture-viewer/src/source-state.test.ts new file mode 100644 index 0000000000..f2ba53acf1 --- /dev/null +++ b/packages/capture-viewer/src/source-state.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, test } from 'bun:test' +import type { CaptureStreamPacket } from '@pascal-app/capture-protocol' +import { + appendCapturePacket, + captureSubscriptionStreamIds, + nextCaptureStreamEpoch, + retainLiveCapturePackets, +} from './source-state' + +function packet(generation: number, sequence: number): CaptureStreamPacket { + return { + protocolVersion: 1, + sessionId: 'capture_123', + streamId: 'points', + generation, + sequence, + timestamp: sequence, + payload: {}, + } +} + +describe('appendCapturePacket', () => { + test('deduplicates, orders, bounds, and resets on a new generation', () => { + let state: Readonly> = {} + state = appendCapturePacket(state, packet(0, 2), 2) + state = appendCapturePacket(state, packet(0, 1), 2) + state = appendCapturePacket(state, packet(0, 2), 2) + expect(state.points?.map((value) => value.sequence)).toEqual([1, 2]) + + state = appendCapturePacket(state, packet(1, 0), 2) + expect(state.points).toEqual([packet(1, 0)]) + expect(appendCapturePacket(state, packet(0, 3), 2)).toBe(state) + }) + + test('resets a stream on keyframes and coordinate-frame changes', () => { + let state: Readonly> = {} + state = appendCapturePacket(state, { ...packet(0, 0), frameId: 'world' }, 4) + state = appendCapturePacket(state, { ...packet(0, 1), frameId: 'world' }, 4) + state = appendCapturePacket(state, { ...packet(0, 2), frameId: 'world', keyframe: true }, 4) + expect(state.points?.map((value) => value.sequence)).toEqual([2]) + + state = appendCapturePacket(state, { ...packet(0, 3), frameId: 'sensor' }, 4) + expect(state.points?.map((value) => value.sequence)).toEqual([3]) + }) + + test('does not let a stale keyframe replace newer live packets', () => { + let state: Readonly> = {} + state = appendCapturePacket(state, { ...packet(0, 5), frameId: 'world' }, 4) + const unchanged = appendCapturePacket( + state, + { ...packet(0, 2), frameId: 'world', keyframe: true }, + 4, + ) + + expect(unchanged).toBe(state) + expect(unchanged.points?.map((value) => value.sequence)).toEqual([5]) + }) + + test('does not reinsert ordinary packets older than an accepted keyframe', () => { + let state: Readonly> = {} + state = appendCapturePacket(state, { ...packet(0, 5), frameId: 'world', keyframe: true }, 4) + const unchanged = appendCapturePacket(state, { ...packet(0, 4), frameId: 'world' }, 4) + + expect(unchanged).toBe(state) + expect(unchanged.points?.map((value) => value.sequence)).toEqual([5]) + }) + + test('keeps a stable playback epoch when a bounded live window advances', () => { + let state: Readonly> = {} + let epoch: string | undefined + const append = (nextPacket: CaptureStreamPacket) => { + const previous = state.points ?? [] + const next = appendCapturePacket(state, nextPacket, 2) + if (next !== state) epoch = nextCaptureStreamEpoch(epoch, previous, nextPacket) + state = next + } + append({ ...packet(0, 0), frameId: 'world' }) + append({ ...packet(0, 1), frameId: 'world' }) + const initialEpoch = epoch + + append({ ...packet(0, 2), frameId: 'world' }) + expect(state.points?.map((value) => value.sequence)).toEqual([1, 2]) + expect(epoch).toBe(initialEpoch) + + append({ ...packet(0, 3), frameId: 'world', keyframe: true }) + expect(epoch).not.toBe(initialEpoch) + const keyframeEpoch = epoch + + append({ ...packet(0, 4), frameId: 'world' }) + expect(epoch).toBe(keyframeEpoch) + + append({ ...packet(1, 0), frameId: 'world' }) + expect(epoch).not.toBe(keyframeEpoch) + }) +}) + +describe('retainLiveCapturePackets', () => { + test('drops preview packets when a stream finalizes', () => { + const packets = { points: [packet(0, 1)] } + expect( + retainLiveCapturePackets(packets, { + schemaVersion: 2, + sessionId: 'capture_123', + state: 'ready', + clocks: [], + coordinateFrames: [], + streams: [{ id: 'points', kind: 'point-cloud', availability: 'ready' }], + }), + ).toEqual({}) + }) +}) + +describe('captureSubscriptionStreamIds', () => { + const descriptor = { + schemaVersion: 2, + sessionId: 'capture_123', + state: 'live', + clocks: [], + coordinateFrames: [], + streams: [ + { id: 'model', kind: 'room-model', role: 'model', availability: 'ready' }, + { id: 'points', kind: 'point-cloud', role: 'pointCloud', availability: 'live' }, + ], + } as const + + test('leaves subscriptions unrestricted without a filter', () => { + expect(captureSubscriptionStreamIds(descriptor, undefined)).toBeUndefined() + }) + + test('subscribes only to streams accepted by the host', () => { + expect( + captureSubscriptionStreamIds(descriptor, (stream) => stream.role !== 'pointCloud'), + ).toEqual(['model']) + }) +}) diff --git a/packages/capture-viewer/src/source-state.ts b/packages/capture-viewer/src/source-state.ts new file mode 100644 index 0000000000..434040771e --- /dev/null +++ b/packages/capture-viewer/src/source-state.ts @@ -0,0 +1,273 @@ +import type { + CaptureSessionDescriptor, + CaptureSessionLocator, + CaptureSource, + CaptureSourceResolver, + CaptureStreamDescriptor, + CaptureStreamPacket, +} from '@pascal-app/capture-protocol' +import { useCallback, useEffect, useState } from 'react' + +export type CaptureSourceState = { + descriptor: CaptureSessionDescriptor | null + descriptorVersion: number + error: Error | null + loading: boolean + packets: Readonly> + retry: () => void + source: CaptureSource | null + streamEpochs: Readonly> +} + +type CaptureSourceSnapshot = Omit + +export type UseCaptureSourceOptions = { + maxPacketsPerStream?: number + streamFilter?: (stream: CaptureStreamDescriptor) => boolean + subscribe?: boolean +} + +const EMPTY_PACKETS: Readonly> = {} +const EMPTY_STREAM_EPOCHS: Readonly> = {} + +export function useCaptureSource( + locator: CaptureSessionLocator | null, + resolveSource: CaptureSourceResolver, + options: UseCaptureSourceOptions = {}, +): CaptureSourceState { + const { maxPacketsPerStream = 32, streamFilter, subscribe = true } = options + const [retryVersion, setRetryVersion] = useState(0) + const [state, setState] = useState({ + descriptor: null, + descriptorVersion: 0, + error: null, + loading: Boolean(locator), + packets: EMPTY_PACKETS, + source: null, + streamEpochs: EMPTY_STREAM_EPOCHS, + }) + + useEffect(() => { + const abort = new AbortController() + if (!locator) { + setState({ + descriptor: null, + descriptorVersion: 0, + error: null, + loading: false, + packets: EMPTY_PACKETS, + source: null, + streamEpochs: EMPTY_STREAM_EPOCHS, + }) + return () => abort.abort() + } + + setState({ + descriptor: null, + descriptorVersion: retryVersion, + error: null, + loading: true, + packets: EMPTY_PACKETS, + source: null, + streamEpochs: EMPTY_STREAM_EPOCHS, + }) + void consumeCaptureSource( + locator, + resolveSource, + abort.signal, + retryVersion, + maxPacketsPerStream, + streamFilter, + subscribe, + setState, + ) + return () => abort.abort() + }, [locator, maxPacketsPerStream, resolveSource, retryVersion, streamFilter, subscribe]) + + const retry = useCallback(() => setRetryVersion((current) => current + 1), []) + return { ...state, retry } +} + +async function consumeCaptureSource( + locator: CaptureSessionLocator, + resolveSource: CaptureSourceResolver, + signal: AbortSignal, + descriptorVersion: number, + maxPacketsPerStream: number, + streamFilter: ((stream: CaptureStreamDescriptor) => boolean) | undefined, + subscribe: boolean, + setState: ( + update: CaptureSourceSnapshot | ((current: CaptureSourceSnapshot) => CaptureSourceSnapshot), + ) => void, +): Promise { + try { + const source = await resolveSource(locator) + const descriptor = await source.describe(signal) + if (signal.aborted) return + setState({ + descriptor, + descriptorVersion, + error: null, + loading: false, + packets: EMPTY_PACKETS, + source, + streamEpochs: EMPTY_STREAM_EPOCHS, + }) + + if (!(subscribe && source.subscribe)) return + const streamIds = captureSubscriptionStreamIds(descriptor, streamFilter) + const iterator = source.subscribe({ signal, streamIds })[Symbol.asyncIterator]() + const closeIterator = () => void iterator.return?.() + signal.addEventListener('abort', closeIterator, { once: true }) + try { + for (;;) { + const next = await iterator.next() + if (next.done || signal.aborted || next.value.type === 'closed') return + const event = next.value + if (event.type === 'descriptor') { + setState((current) => { + const revisionChanged = + current.descriptor?.revisionId !== event.descriptor.revisionId && + (current.descriptor?.revisionId !== undefined || + event.descriptor.revisionId !== undefined) + return { + ...current, + descriptor: event.descriptor, + descriptorVersion: current.descriptorVersion + 1, + packets: revisionChanged + ? EMPTY_PACKETS + : retainLiveCapturePackets(current.packets, event.descriptor), + streamEpochs: revisionChanged + ? EMPTY_STREAM_EPOCHS + : retainLiveCaptureStreamValues(current.streamEpochs, event.descriptor), + } + }) + } else { + setState((current) => { + const previousPackets = current.packets[event.packet.streamId] ?? [] + const packets = appendCapturePacket(current.packets, event.packet, maxPacketsPerStream) + if (packets === current.packets) return current + return { + ...current, + packets, + streamEpochs: { + ...current.streamEpochs, + [event.packet.streamId]: nextCaptureStreamEpoch( + current.streamEpochs[event.packet.streamId], + previousPackets, + event.packet, + ), + }, + } + }) + } + } + } finally { + signal.removeEventListener('abort', closeIterator) + await iterator.return?.() + } + } catch (cause) { + if (signal.aborted) return + setState({ + descriptor: null, + descriptorVersion, + error: cause instanceof Error ? cause : new Error('Could not load capture session.'), + loading: false, + packets: EMPTY_PACKETS, + source: null, + streamEpochs: EMPTY_STREAM_EPOCHS, + }) + } +} + +export function captureSubscriptionStreamIds( + descriptor: CaptureSessionDescriptor, + streamFilter: ((stream: CaptureStreamDescriptor) => boolean) | undefined, +): readonly string[] | undefined { + return streamFilter + ? descriptor.streams.filter(streamFilter).map((stream) => stream.id) + : undefined +} + +export function retainLiveCapturePackets( + packetsByStream: Readonly>, + descriptor: CaptureSessionDescriptor, +): Readonly> { + const liveStreamIds = new Set( + descriptor.streams + .filter((stream) => stream.availability === 'live') + .map((stream) => stream.id), + ) + return Object.fromEntries( + Object.entries(packetsByStream).filter(([streamId]) => liveStreamIds.has(streamId)), + ) +} + +function retainLiveCaptureStreamValues( + valuesByStream: Readonly>, + descriptor: CaptureSessionDescriptor, +): Readonly> { + const liveStreamIds = new Set( + descriptor.streams + .filter((stream) => stream.availability === 'live') + .map((stream) => stream.id), + ) + return Object.fromEntries( + Object.entries(valuesByStream).filter(([streamId]) => liveStreamIds.has(streamId)), + ) +} + +export function appendCapturePacket( + packetsByStream: Readonly>, + packet: CaptureStreamPacket, + maxPacketsPerStream: number, +): Readonly> { + const previous = packetsByStream[packet.streamId] ?? [] + const latest = previous.at(-1) + const currentGeneration = latest?.generation + if (currentGeneration !== undefined && packet.generation < currentGeneration) + return packetsByStream + const resetSequence = previous[0]?.keyframe ? previous[0].sequence : null + if ( + currentGeneration === packet.generation && + resetSequence !== null && + packet.sequence <= resetSequence + ) { + return packetsByStream + } + const resetsStream = Boolean(packet.keyframe) || latest?.frameId !== packet.frameId + if ( + latest && + currentGeneration === packet.generation && + resetsStream && + packet.sequence <= latest.sequence + ) { + return packetsByStream + } + const sameGeneration = currentGeneration === packet.generation && !resetsStream ? previous : [] + if (sameGeneration.some((candidate) => candidate.sequence === packet.sequence)) + return packetsByStream + const limit = Math.max(1, maxPacketsPerStream) + const next = [...sameGeneration, packet] + .sort((left, right) => left.sequence - right.sequence) + .slice(-limit) + return { ...packetsByStream, [packet.streamId]: next } +} + +export function nextCaptureStreamEpoch( + currentEpoch: string | undefined, + previousPackets: readonly CaptureStreamPacket[], + packet: CaptureStreamPacket, +): string { + const latest = previousPackets.at(-1) + const resetsStream = + previousPackets.length === 0 || + latest?.generation !== packet.generation || + latest.frameId !== packet.frameId || + Boolean(packet.keyframe) + if (resetsStream) return `${packet.generation}:${packet.frameId ?? ''}:${packet.sequence}` + return ( + currentEpoch ?? + `${latest.generation}:${latest.frameId ?? ''}:${previousPackets[0]?.sequence ?? latest.sequence}` + ) +} diff --git a/packages/capture-viewer/src/stream-rendering.test.ts b/packages/capture-viewer/src/stream-rendering.test.ts new file mode 100644 index 0000000000..48d00c02bd --- /dev/null +++ b/packages/capture-viewer/src/stream-rendering.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from 'bun:test' +import { isCaptureStreamRenderable } from './stream-rendering' + +describe('isCaptureStreamRenderable', () => { + test('only advertises point-cloud formats handled by the reference renderer', () => { + expect( + isCaptureStreamRenderable({ + id: 'points', + kind: 'point-cloud', + role: 'pointCloud', + availability: 'ready', + artifact: { id: 'points', mediaType: 'application/vnd.las', uri: '/points.las' }, + }), + ).toBe(false) + expect( + isCaptureStreamRenderable({ + id: 'points', + kind: 'point-cloud', + role: 'pointCloud', + availability: 'ready', + artifact: { id: 'points', mediaType: 'application/ply', uri: '/points.ply' }, + }), + ).toBe(true) + expect( + isCaptureStreamRenderable({ + id: 'inline-points', + kind: 'point-cloud', + role: 'pointCloud', + availability: 'ready', + inline: { + coordinateSystem: 'arkit-world', + positions: [0, 0, 0, 1, 1, 1], + }, + }), + ).toBe(true) + }) + + test('allows a host renderer to claim an otherwise unknown stream', () => { + expect( + isCaptureStreamRenderable( + { id: 'splat', kind: 'gaussian-splat', availability: 'ready' }, + new Set(['gaussian-splat']), + ), + ).toBe(true) + }) + + test('renders a valid inline color surface mesh', () => { + expect( + isCaptureStreamRenderable({ + id: 'surface-mesh', + kind: 'surface-mesh', + role: 'surfaceMesh', + availability: 'ready', + inline: { + version: 1, + coordinateSystem: 'arkit-world', + representation: 'quantized-indexed-triangle-mesh', + appearance: 'camera-vertex-color', + vertexCount: 3, + faceCount: 1, + boundsMin: [0, 0, 0], + boundsMax: [1, 1, 0], + positionEncoding: 'uint16x3-base64-little-endian', + colorEncoding: 'uint8x3-base64-srgb', + indexEncoding: 'uint16x3-base64-little-endian', + positions: 'AAAAAAAAAAAAAAAAAAAAAAAA', + colors: '////////////', + indices: 'AAABAAIA', + }, + }), + ).toBe(true) + }) +}) diff --git a/packages/capture-viewer/src/stream-rendering.ts b/packages/capture-viewer/src/stream-rendering.ts new file mode 100644 index 0000000000..e5e773d8cb --- /dev/null +++ b/packages/capture-viewer/src/stream-rendering.ts @@ -0,0 +1,69 @@ +import { + type CaptureArtifactReference, + type CaptureStreamDescriptor, + captureLayerKey, + DeviceMotionTrajectorySchema, + PointCloudPayloadSchema, + SurfaceMeshPayloadSchema, +} from '@pascal-app/capture-protocol' + +const GLB_MEDIA_TYPES = new Set(['model/gltf-binary', 'model/gltf+json']) +const USDZ_MEDIA_TYPES = new Set(['model/vnd.usdz+zip']) +const PLY_MEDIA_TYPES = new Set(['application/ply', 'application/vnd.ply', 'model/ply']) + +export type CaptureModelFormat = 'gltf' | 'usdz' + +export function isCaptureStreamRenderable( + stream: CaptureStreamDescriptor, + customRendererKeys: ReadonlySet = new Set(), +): boolean { + if (stream.availability === 'failed' || stream.availability === 'pending') return false + const layerKey = captureLayerKey(stream) + if (customRendererKeys.has(layerKey) || customRendererKeys.has(stream.kind)) return true + if (layerKey === 'model') return isCaptureModelArtifact(stream.artifact) + if (layerKey === 'deviceMotion') { + return ( + stream.availability === 'live' || + DeviceMotionTrajectorySchema.safeParse(stream.inline).success + ) + } + if (layerKey === 'pointCloud') { + return ( + stream.availability === 'live' || + isCapturePointCloudArtifact(stream.artifact) || + PointCloudPayloadSchema.safeParse(stream.inline).success + ) + } + if (layerKey === 'surfaceMesh') return SurfaceMeshPayloadSchema.safeParse(stream.inline).success + return false +} + +export function isCaptureModelArtifact(artifact: CaptureArtifactReference | undefined): boolean { + return captureModelFormat(artifact) !== null +} + +export function captureModelFormat( + artifact: CaptureArtifactReference | undefined, +): CaptureModelFormat | null { + if (!artifact) return null + if (USDZ_MEDIA_TYPES.has(artifact.mediaType) || hasExtension(artifact.uri, ['.usdz'])) { + return 'usdz' + } + if (GLB_MEDIA_TYPES.has(artifact.mediaType) || hasExtension(artifact.uri, ['.glb', '.gltf'])) { + return 'gltf' + } + return null +} + +export function isCapturePointCloudArtifact( + artifact: CaptureArtifactReference | undefined, +): boolean { + if (!artifact) return false + return PLY_MEDIA_TYPES.has(artifact.mediaType) || hasExtension(artifact.uri, ['.ply']) +} + +function hasExtension(uri: string | undefined, extensions: readonly string[]): boolean { + if (!uri) return false + const path = uri.split(/[?#]/, 1)[0]?.toLowerCase() ?? '' + return extensions.some((extension) => path.endsWith(extension)) +} diff --git a/packages/capture-viewer/src/surface-mesh-layer.test.ts b/packages/capture-viewer/src/surface-mesh-layer.test.ts new file mode 100644 index 0000000000..44a408c9f1 --- /dev/null +++ b/packages/capture-viewer/src/surface-mesh-layer.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from 'bun:test' +import { buildSurfaceMeshData } from './layers/surface-mesh-layer' + +describe('surface mesh layer', () => { + test('decodes quantized positions, vertex colors, and triangle indices', () => { + const positions = new Uint8Array([0, 0, 0, 0, 0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 255, 255, 0, 0]) + const colors = new Uint8Array([255, 0, 0, 0, 255, 0, 0, 0, 255]) + const indices = new Uint8Array([0, 0, 1, 0, 2, 0]) + const data = buildSurfaceMeshData({ + version: 1, + coordinateSystem: 'arkit-world', + representation: 'quantized-indexed-triangle-mesh', + appearance: 'camera-vertex-color', + vertexCount: 3, + faceCount: 1, + boundsMin: [0, 0, 0], + boundsMax: [2, 2, 0], + positionEncoding: 'uint16x3-base64-little-endian', + colorEncoding: 'uint8x3-base64-srgb', + indexEncoding: 'uint16x3-base64-little-endian', + positions: Buffer.from(positions).toString('base64'), + colors: Buffer.from(colors).toString('base64'), + indices: Buffer.from(indices).toString('base64'), + }) + + expect(Array.from(data?.positions ?? [])).toEqual([0, 0, 0, 2, 0, 0, 0, 2, 0]) + expect(Array.from(data?.indices ?? [])).toEqual([0, 1, 2]) + expect(Array.from(data?.colors ?? [])).toEqual([1, 0, 0, 0, 1, 0, 0, 0, 1]) + }) + + test('rejects malformed buffers and out-of-range indices', () => { + expect( + buildSurfaceMeshData({ + version: 1, + coordinateSystem: 'arkit-world', + representation: 'quantized-indexed-triangle-mesh', + appearance: 'camera-vertex-color', + vertexCount: 1, + faceCount: 1, + boundsMin: [0, 0, 0], + boundsMax: [1, 1, 1], + positionEncoding: 'uint16x3-base64-little-endian', + colorEncoding: 'uint8x3-base64-srgb', + indexEncoding: 'uint16x3-base64-little-endian', + positions: 'AA==', + colors: 'AAAA', + indices: 'AAABAAIA', + }), + ).toBeNull() + }) +}) diff --git a/packages/capture-viewer/src/trajectory.test.ts b/packages/capture-viewer/src/trajectory.test.ts new file mode 100644 index 0000000000..158334bb4a --- /dev/null +++ b/packages/capture-viewer/src/trajectory.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from 'bun:test' +import { parseDeviceTrajectoryPackets } from './trajectory' + +const identity = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] + +describe('parseDeviceTrajectoryPackets', () => { + test('applies individual samples after the latest full trajectory snapshot', () => { + const trajectory = parseDeviceTrajectoryPackets([ + { + coordinateSystem: 'arkit-world', + samples: [ + { segment: 0, timestamp: 0, transform: identity }, + { segment: 0, timestamp: 1, transform: identity }, + ], + }, + { segment: 0, timestamp: 2, transform: identity }, + ]) + + expect(trajectory?.poses.map((pose) => pose.timestamp)).toEqual([0, 1, 2]) + }) +}) diff --git a/packages/capture-viewer/src/trajectory.ts b/packages/capture-viewer/src/trajectory.ts new file mode 100644 index 0000000000..688f2e6828 --- /dev/null +++ b/packages/capture-viewer/src/trajectory.ts @@ -0,0 +1,106 @@ +import { + DeviceMotionSampleSchema, + type DeviceMotionTrajectoryPayload, + DeviceMotionTrajectorySchema, +} from '@pascal-app/capture-protocol' +import { Matrix4, Quaternion, Vector3 } from 'three' + +export type DeviceTrajectoryPose = { + position: [number, number, number] + quaternion: [number, number, number, number] + segment: number + timestamp: number +} + +export type DeviceTrajectory = { + duration: number + poses: DeviceTrajectoryPose[] +} + +export type DeviceTrajectoryFrame = { + alpha: number + from: DeviceTrajectoryPose + to: DeviceTrajectoryPose +} + +export function parseDeviceTrajectoryPayload( + trajectory: DeviceMotionTrajectoryPayload | null | undefined, +): DeviceTrajectory | null { + if (!trajectory) return null + const parsed = DeviceMotionTrajectorySchema.safeParse(trajectory) + if (!parsed.success) return null + + const poses = parsed.data.samples + .map(parsePose) + .sort((left, right) => left.timestamp - right.timestamp) + if (poses.length < 2) return null + + const firstTimestamp = poses[0]?.timestamp ?? 0 + for (const pose of poses) pose.timestamp -= firstTimestamp + const duration = poses.at(-1)?.timestamp ?? 0 + if (!(duration > 0)) return null + return { duration, poses } +} + +export function parseDeviceTrajectoryPackets( + payloads: readonly unknown[], +): DeviceTrajectory | null { + let coordinateSystem = 'source' + let samples: DeviceMotionTrajectoryPayload['samples'] = [] + for (const payload of payloads) { + const trajectory = DeviceMotionTrajectorySchema.safeParse(payload) + if (trajectory.success) { + coordinateSystem = trajectory.data.coordinateSystem + samples = [...trajectory.data.samples] + continue + } + const sample = DeviceMotionSampleSchema.safeParse(payload) + if (sample.success) samples.push(sample.data) + } + if (samples.length < 2) return null + return parseDeviceTrajectoryPayload({ coordinateSystem, samples }) +} + +export function sampleDeviceTrajectory( + trajectory: DeviceTrajectory, + elapsed: number, +): DeviceTrajectoryFrame { + const time = positiveModulo(elapsed, trajectory.duration) + const poses = trajectory.poses + const first = poses[0] + const last = poses.at(-1) + if (!(first && last)) throw new Error('Device trajectory requires at least two poses.') + let upperIndex = poses.findIndex((pose) => pose.timestamp > time) + + if (upperIndex < 0) upperIndex = poses.length - 1 + if (upperIndex === 0) return { alpha: 0, from: first, to: first } + + const from = poses[upperIndex - 1] ?? first + const to = poses[upperIndex] ?? last + if (from.segment !== to.segment) return { alpha: 0, from, to: from } + + const interval = to.timestamp - from.timestamp + return { + alpha: interval > 0 ? Math.min(1, Math.max(0, (time - from.timestamp) / interval)) : 0, + from, + to, + } +} + +function parsePose(value: DeviceMotionTrajectoryPayload['samples'][number]): DeviceTrajectoryPose { + const matrix = new Matrix4().fromArray(value.transform) + const position = new Vector3() + const quaternion = new Quaternion() + matrix.decompose(position, quaternion, new Vector3()) + + return { + position: [position.x, position.y, position.z], + quaternion: [quaternion.x, quaternion.y, quaternion.z, quaternion.w], + segment: value.segment, + timestamp: value.timestamp, + } +} + +function positiveModulo(value: number, divisor: number): number { + return ((value % divisor) + divisor) % divisor +} diff --git a/packages/capture-viewer/tsconfig.json b/packages/capture-viewer/tsconfig.json new file mode 100644 index 0000000000..3b503afab8 --- /dev/null +++ b/packages/capture-viewer/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "@pascal/typescript-config/react-library.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "noEmit": false, + "composite": true, + "incremental": true, + "types": ["bun"] + }, + "include": ["src"], + "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.test.tsx"], + "references": [{ "path": "../capture-protocol" }, { "path": "../core" }, { "path": "../viewer" }] +} diff --git a/packages/core/package.json b/packages/core/package.json index 476bb26230..adce140d36 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -75,6 +75,7 @@ "three": "^0.185" }, "dependencies": { + "@pascal-app/capture-protocol": "^1.0.0-beta.4", "dedent": "^1.7.1", "idb-keyval": "^6.2.2", "mitt": "^3.0.1", diff --git a/packages/core/src/schema/index.ts b/packages/core/src/schema/index.ts index 8ab8725948..bf08161e06 100644 --- a/packages/core/src/schema/index.ts +++ b/packages/core/src/schema/index.ts @@ -249,7 +249,11 @@ export { roofFacePointToSegment, segmentPointToRoofWallFace, } from './nodes/roof-segment-walls' -export { ScanNode } from './nodes/scan' +export { + CaptureSessionReference, + type CaptureSessionReferenceInput, + ScanNode, +} from './nodes/scan' export { ShelfNode } from './nodes/shelf' export { SiteNode } from './nodes/site' export { diff --git a/packages/core/src/schema/nodes/scan.test.ts b/packages/core/src/schema/nodes/scan.test.ts new file mode 100644 index 0000000000..b223dcbad5 --- /dev/null +++ b/packages/core/src/schema/nodes/scan.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, test } from 'bun:test' +import { ScanNode } from './scan' + +describe('ScanNode', () => { + test('keeps legacy GLB-backed scans loadable', () => { + const scan = ScanNode.parse({ + id: 'scan_legacy', + type: 'scan', + url: 'https://cdn.pascal.app/scans/room.glb', + }) + + expect(scan.url).toBe('https://cdn.pascal.app/scans/room.glb') + expect(scan.captureSession).toBeNull() + expect(scan.layers).toEqual({ model: true, deviceMotion: true }) + }) + + test('accepts a capture session without a renderable mesh', () => { + const scan = ScanNode.parse({ + id: 'scan_session', + type: 'scan', + captureSession: { + sessionId: 'session_123', + manifestUrl: '/api/projects/project_1/captures/capture_1/artifacts/manifest.json', + schemaVersion: 1, + }, + }) + + expect(scan.url).toBeNull() + expect(scan.captureSession).toEqual({ + sessionId: 'session_123', + manifestUrl: '/api/projects/project_1/captures/capture_1/artifacts/manifest.json', + schemaVersion: 1, + }) + expect(scan.layers).toEqual({ model: true, deviceMotion: true }) + }) + + test('persists independent model and device-motion visibility', () => { + const scan = ScanNode.parse({ + id: 'scan_session', + type: 'scan', + layers: { + model: false, + deviceMotion: true, + }, + }) + + expect(scan.layers).toEqual({ model: false, deviceMotion: true }) + }) + + test('preserves known layer defaults when a legacy scene stores a partial map', () => { + const scan = ScanNode.parse({ + id: 'scan_session', + type: 'scan', + layers: { model: false }, + }) + + expect(scan.layers).toEqual({ deviceMotion: true, model: false }) + }) + + test('supports host-resolved sessions and future layer keys', () => { + const scan = ScanNode.parse({ + id: 'scan_session', + type: 'scan', + captureSession: { + sessionId: 'session_123', + revisionId: 'revision_2', + }, + layers: { + model: true, + pointCloud: false, + wifiRanging: true, + }, + }) + + expect(scan.captureSession).toEqual({ + sessionId: 'session_123', + revisionId: 'revision_2', + }) + expect(scan.layers).toEqual({ + deviceMotion: true, + model: true, + pointCloud: false, + wifiRanging: true, + }) + }) + + test('rejects unsafe manifest URLs', () => { + const result = ScanNode.safeParse({ + id: 'scan_session', + type: 'scan', + captureSession: { + sessionId: 'session_123', + manifestUrl: 'javascript:alert(1)', + }, + }) + + expect(result.success).toBe(false) + }) +}) diff --git a/packages/core/src/schema/nodes/scan.ts b/packages/core/src/schema/nodes/scan.ts index 9f50a1b5e4..36e16a323a 100644 --- a/packages/core/src/schema/nodes/scan.ts +++ b/packages/core/src/schema/nodes/scan.ts @@ -1,15 +1,30 @@ +import { CaptureSessionLocatorSchema } from '@pascal-app/capture-protocol' import { z } from 'zod' import { AssetUrl } from '../asset-url' import { BaseNode, nodeType, objectId } from '../base' +export const CaptureSessionReference = CaptureSessionLocatorSchema.extend({ + manifestUrl: AssetUrl.optional(), +}) + +export const ScanLayerVisibility = z + .record(z.string().min(1), z.boolean()) + .default({ deviceMotion: true, model: true }) + .transform((layers): Record => ({ deviceMotion: true, model: true, ...layers })) + export const ScanNode = BaseNode.extend({ id: objectId('scan'), type: nodeType('scan'), - url: AssetUrl, + url: AssetUrl.nullable().default(null), + captureSession: CaptureSessionReference.nullable().default(null), + layers: ScanLayerVisibility, position: z.tuple([z.number(), z.number(), z.number()]).default([0, 0, 0]), rotation: z.tuple([z.number(), z.number(), z.number()]).default([0, 0, 0]), scale: z.number().default(1), opacity: z.number().min(0).max(100).default(100), }) +export type CaptureSessionReference = z.infer +export type CaptureSessionReferenceInput = z.input +export type ScanLayerVisibility = z.infer export type ScanNode = z.infer diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index bf55849b88..957e2ce9cb 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -9,5 +9,6 @@ "types": ["bun"] }, "include": ["src"], - "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.test.tsx"] + "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.test.tsx"], + "references": [{ "path": "../capture-protocol" }] } diff --git a/packages/editor/src/components/ui/panels/reference-panel.tsx b/packages/editor/src/components/ui/panels/reference-panel.tsx index 069a51288a..86da5ccd89 100644 --- a/packages/editor/src/components/ui/panels/reference-panel.tsx +++ b/packages/editor/src/components/ui/panels/reference-panel.tsx @@ -13,6 +13,7 @@ import { EyeOff, LocateFixed, Lock, + Move, RotateCcw, Ruler, Trash2, @@ -144,6 +145,12 @@ export function ReferencePanel() { guideEmitter.emit('guide:cancel-reference-scale') }, []) + const handleMoveScan = useCallback(() => { + if (node?.type !== 'scan') return + useEditor.getState().setMovingNode(node as never) + setSelectedReferenceId(null) + }, [node, setSelectedReferenceId]) + useEffect(() => { if (node?.type !== 'guide' || !node.url.startsWith('asset://')) { setIsAssetMissing(false) @@ -172,7 +179,7 @@ export function ReferencePanel() { return ( {!isScan && ( @@ -329,6 +336,29 @@ export function ReferencePanel() { )} + {isScan && ( + + + } + label="Move" + onClick={handleMoveScan} + /> + + ) : ( + + ) + } + label={node.visible === false ? 'Show' : 'Hide'} + onClick={() => handleUpdate({ visible: node.visible === false })} + /> + + + )} + (null) + const copyResetTimeoutRef = useRef(null) const nodes = useScene((state) => state.nodes) const rootNodeIds = useScene((state) => state.rootNodeIds) const installedPlugins = useScene((state) => state.installedPlugins) @@ -198,6 +201,9 @@ export function SettingsPanel({ const [isGeneratingThumbnail, setIsGeneratingThumbnail] = useState(false) const [exportOnlyVisible, setExportOnlyVisible] = useState(true) const [pendingImport, setPendingImport] = useState(null) + const [projectIdCopyState, setProjectIdCopyState] = useState<'idle' | 'copied' | 'error'>( + 'idle', + ) const sceneGraphValue = useMemo( () => buildSceneGraphValue(nodes as Record, rootNodeIds), [nodes, rootNodeIds], @@ -213,6 +219,15 @@ export function SettingsPanel({ } }, []) + useEffect( + () => () => { + if (copyResetTimeoutRef.current !== null) { + window.clearTimeout(copyResetTimeoutRef.current) + } + }, + [], + ) + const isLocalProject = false // Props-based; only show cloud sections when projectId provided const handleSaveBuild = () => { @@ -312,6 +327,25 @@ export function SettingsPanel({ setTimeout(() => setIsGeneratingThumbnail(false), 3000) } + const handleCopyProjectId = async () => { + if (!projectId) return + if (copyResetTimeoutRef.current !== null) { + window.clearTimeout(copyResetTimeoutRef.current) + } + + try { + await navigator.clipboard.writeText(projectId) + setProjectIdCopyState('copied') + } catch { + setProjectIdCopyState('error') + } + + copyResetTimeoutRef.current = window.setTimeout(() => { + setProjectIdCopyState('idle') + copyResetTimeoutRef.current = null + }, 2000) + } + const handleVisibilityChange = async ( field: 'isPrivate' | 'showScansPublic' | 'showGuidesPublic', value: boolean, @@ -321,6 +355,40 @@ export function SettingsPanel({ return (
+ {projectId && ( +
+ +
Project ID
+
+ + +
+
+ )} + {/* Visibility Section (only for cloud projects) */} {projectId && !isLocalProject && (
diff --git a/packages/editor/src/components/ui/sidebar/panels/site-panel/index.tsx b/packages/editor/src/components/ui/sidebar/panels/site-panel/index.tsx index e397930aab..3f4cae036a 100644 --- a/packages/editor/src/components/ui/sidebar/panels/site-panel/index.tsx +++ b/packages/editor/src/components/ui/sidebar/panels/site-panel/index.tsx @@ -15,7 +15,10 @@ import { useViewer } from '@pascal-app/viewer' import { Camera, ChevronDown, + ChevronRight, Copy, + Eye, + EyeOff, Loader2, MoreHorizontal, Pencil, @@ -25,7 +28,7 @@ import { X, } from 'lucide-react' import { AnimatePresence, LayoutGroup, motion } from 'motion/react' -import { memo, useEffect, useRef, useState } from 'react' +import { memo, useEffect, useRef, useState, useSyncExternalStore } from 'react' import { useShallow } from 'zustand/react/shallow' import { ColorDot } from './../../../../../components/ui/primitives/color-dot' import { @@ -48,6 +51,7 @@ import { squareMetersToAreaUnit, } from './../../../../../lib/measurements' import { createLocalGuideImage } from './../../../../../lib/local-guide-image' +import { editorHostTreeChildrenRegistry } from './../../../../../lib/host-tree-children' import { cn } from './../../../../../lib/utils' import useEditor from './../../../../../store/use-editor' import { useUploadStore } from '../../../../../store/use-upload' @@ -350,8 +354,25 @@ const ReferenceItem = memo(function ReferenceItem({ handleDelete: (id: string, e: React.MouseEvent) => void }) { const [isEditing, setIsEditing] = useState(false) + const [isExpanded, setIsExpanded] = useState(true) + const updateNode = useScene((state) => state.updateNode) + const selectedReferenceId = useEditor((state) => state.selectedReferenceId) + const isCapture = refNode.type === 'scan' + const isVisible = refNode.visible !== false + useSyncExternalStore( + editorHostTreeChildrenRegistry.subscribe, + editorHostTreeChildrenRegistry.getSnapshot, + editorHostTreeChildrenRegistry.getSnapshot, + ) + const hostChildren = isCapture + ? editorHostTreeChildrenRegistry.childrenForKind(refNode.type) + : undefined + const hasHostChildren = Boolean(hostChildren?.hasChildren(refNode)) + const HostChildren = hasHostChildren ? hostChildren?.component : undefined + const handleSelect = () => { setSelectedReferenceId(refNode.id) + useViewer.getState().setSelection({ selectedIds: [], zoneId: null }) } const handleDoubleClick = () => { @@ -359,53 +380,98 @@ const ReferenceItem = memo(function ReferenceItem({ } return ( -
+
-
+ onClick={handleSelect} + onDoubleClick={handleDoubleClick} + > +
+
-
- {refNode.type === 'scan' ? ( - Scan - ) : ( - Guide { + event.stopPropagation() + if (hasHostChildren) setIsExpanded((expanded) => !expanded) + }} + type="button" + > + {hasHostChildren ? ( + + ) : null} + + ) : null} + +
+ {isCapture ? ( + Capture + ) : ( + Guide + )} + setIsEditing(true)} + onStopEditing={() => setIsEditing(false)} /> - )} - setIsEditing(true)} - onStopEditing={() => setIsEditing(false)} - /> -
+
- + {isCapture ? ( + + ) : null} + +
+ {isExpanded && HostChildren ? ( + + ) : null}
) }) @@ -1332,7 +1398,10 @@ const ContentSection = memo(function ContentSection() { if (!selectedLevelId) return [] const lvl = s.nodes[selectedLevelId] as LevelNode | undefined if (!lvl) return [] - return lvl.children.filter((childId) => s.nodes[childId]?.type !== 'zone') + return lvl.children.filter((childId) => { + const type = s.nodes[childId]?.type + return type !== 'zone' && type !== 'scan' + }) }), ) diff --git a/packages/editor/src/components/ui/sidebar/panels/site-panel/registry-tree-node.tsx b/packages/editor/src/components/ui/sidebar/panels/site-panel/registry-tree-node.tsx index 5e31e4ff4e..d17fa1bf5a 100644 --- a/packages/editor/src/components/ui/sidebar/panels/site-panel/registry-tree-node.tsx +++ b/packages/editor/src/components/ui/sidebar/panels/site-panel/registry-tree-node.tsx @@ -2,8 +2,9 @@ import { Icon as IconifyIcon } from '@iconify/react' import { type AnyNodeId, nodeRegistry, useScene } from '@pascal-app/core' import { useViewer } from '@pascal-app/viewer' import Image from 'next/image' -import { memo, useCallback, useEffect, useState } from 'react' +import { memo, useCallback, useEffect, useState, useSyncExternalStore } from 'react' import { useShallow } from 'zustand/react/shallow' +import { editorHostTreeChildrenRegistry } from '../../../../../lib/host-tree-children' import { resolveNodeSnapTarget, SnapTargetIcon } from '../../../snap-target-badge' import { InlineRenameInput } from './inline-rename-input' import { @@ -43,9 +44,19 @@ export const RegistryTreeNode = memo(function RegistryTreeNode({ const isHovered = useViewer((state) => state.hoveredId === nodeId) const setSelection = useViewer((state) => state.setSelection) const setHoveredId = useViewer((state) => state.setHoveredId) + useSyncExternalStore( + editorHostTreeChildrenRegistry.subscribe, + editorHostTreeChildrenRegistry.getSnapshot, + editorHostTreeChildrenRegistry.getSnapshot, + ) const presentation = node ? nodeRegistry.get(node.type)?.presentation : undefined const tree = node ? nodeRegistry.get(node.type)?.tree : undefined + const hostChildren = node + ? editorHostTreeChildrenRegistry.childrenForKind(node.type) + : undefined + const hasHostChildren = Boolean(node && hostChildren?.hasChildren(node)) + const HostChildren = hasHostChildren ? hostChildren?.component : undefined const icon = presentation?.icon const iconSrc = icon?.kind === 'url' ? icon.src : '/icons/roof.webp' const iconElement = @@ -63,7 +74,7 @@ export const RegistryTreeNode = memo(function RegistryTreeNode({ const snapTarget = resolveNodeSnapTarget(node) const defaultName = node ? tree?.label?.(node, useScene.getState().nodes) || node.name || presentation?.label || 'Node' : 'Node' - const hasChildren = children.length > 0 + const hasChildren = children.length > 0 || hasHostChildren useEffect(() => { return useViewer.subscribe((state) => { @@ -126,15 +137,21 @@ export const RegistryTreeNode = memo(function RegistryTreeNode({ onMouseLeave={() => setHoveredId(null)} onToggle={() => setExpanded((prev) => !prev)} > - {hasChildren && - children.map((childId, index) => ( - - ))} + {hasChildren && ( + <> + {children.map((childId, index) => ( + + ))} + {HostChildren ? ( + + ) : null} + + )} ) }) diff --git a/packages/editor/src/components/ui/sidebar/panels/site-panel/tree-node.tsx b/packages/editor/src/components/ui/sidebar/panels/site-panel/tree-node.tsx index 9140660b15..dd1d9578d4 100644 --- a/packages/editor/src/components/ui/sidebar/panels/site-panel/tree-node.tsx +++ b/packages/editor/src/components/ui/sidebar/panels/site-panel/tree-node.tsx @@ -173,6 +173,7 @@ const treeNodeByType: Record = { 'eyebrow-vent': RegistryTreeNode, skylight: RegistryTreeNode, roof: RoofTreeNode, + scan: RegistryTreeNode, stair: StairTreeNode, door: DoorTreeNode, window: WindowTreeNode, diff --git a/packages/editor/src/index.tsx b/packages/editor/src/index.tsx index 84ddb9f193..834adec5f7 100644 --- a/packages/editor/src/index.tsx +++ b/packages/editor/src/index.tsx @@ -429,6 +429,12 @@ export { runUndo, subscribeHistoryCommandState, } from './lib/history' +export { + type EditorHostTreeChildren, + type EditorHostTreeChildrenProps, + editorHostTreeChildrenRegistry, + registerEditorHostTreeChildren, +} from './lib/host-tree-children' export { boundaryReshapeScope, curveReshapeScope, diff --git a/packages/editor/src/lib/host-tree-children.test.ts b/packages/editor/src/lib/host-tree-children.test.ts new file mode 100644 index 0000000000..d46f2b8bc3 --- /dev/null +++ b/packages/editor/src/lib/host-tree-children.test.ts @@ -0,0 +1,27 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { + editorHostTreeChildrenRegistry, + registerEditorHostTreeChildren, +} from './host-tree-children' + +describe('editorHostTreeChildrenRegistry', () => { + afterEach(() => editorHostTreeChildrenRegistry.reset()) + + test('exposes host children by scene node kind and notifies mounted trees', () => { + let notifications = 0 + const unsubscribe = editorHostTreeChildrenRegistry.subscribe(() => { + notifications += 1 + }) + + registerEditorHostTreeChildren({ + kind: 'scan', + component: () => null, + hasChildren: (node) => node.type === 'scan', + }) + + expect(editorHostTreeChildrenRegistry.childrenForKind('scan')).toBeDefined() + expect(editorHostTreeChildrenRegistry.childrenForKind('wall')).toBeUndefined() + expect(notifications).toBe(1) + unsubscribe() + }) +}) diff --git a/packages/editor/src/lib/host-tree-children.ts b/packages/editor/src/lib/host-tree-children.ts new file mode 100644 index 0000000000..1efbfed584 --- /dev/null +++ b/packages/editor/src/lib/host-tree-children.ts @@ -0,0 +1,79 @@ +import type { AnyNode, AnyNodeId } from '@pascal-app/core' +import type { ComponentType } from 'react' + +export type EditorHostTreeChildrenProps = { + nodeId: AnyNodeId + depth: number + parentVisible: boolean +} + +export type EditorHostTreeChildren = { + kind: string + component: ComponentType + hasChildren: (node: AnyNode) => boolean +} + +function isDevMode(): boolean { + try { + const meta = import.meta as { env?: { DEV?: boolean } } + if (typeof meta?.env?.DEV === 'boolean') return meta.env.DEV + } catch { + // import.meta unavailable in some CJS contexts — fall through. + } + if (typeof process !== 'undefined' && process.env?.NODE_ENV) { + return process.env.NODE_ENV !== 'production' + } + return false +} + +class EditorHostTreeChildrenRegistryImpl { + private readonly entries = new Map() + private readonly listeners = new Set<() => void>() + private revision = 0 + + subscribe = (onChange: () => void): (() => void) => { + this.listeners.add(onChange) + return () => { + this.listeners.delete(onChange) + } + } + + getSnapshot = (): number => this.revision + + childrenForKind = (kind: string): EditorHostTreeChildren | undefined => this.entries.get(kind) + + reset(): void { + this.entries.clear() + this.emit() + } + + register(entry: EditorHostTreeChildren): void { + if (typeof entry.kind !== 'string' || entry.kind.length === 0) { + throw new Error('[editor:host-tree-children] kind must be a non-empty string') + } + if (this.entries.has(entry.kind)) { + if (isDevMode()) { + console.warn( + `[editor:host-tree-children] re-registering children for "${entry.kind}" (HMR)`, + ) + } else { + throw new Error( + `[editor:host-tree-children] duplicate kind: "${entry.kind}" already registered`, + ) + } + } + this.entries.set(entry.kind, entry) + this.emit() + } + + private emit(): void { + this.revision += 1 + for (const listener of this.listeners) listener() + } +} + +export const editorHostTreeChildrenRegistry = new EditorHostTreeChildrenRegistryImpl() + +export function registerEditorHostTreeChildren(entry: EditorHostTreeChildren): void { + editorHostTreeChildrenRegistry.register(entry) +} diff --git a/packages/nodes/src/scan/definition.ts b/packages/nodes/src/scan/definition.ts index bc6cc2f704..ba13caf293 100644 --- a/packages/nodes/src/scan/definition.ts +++ b/packages/nodes/src/scan/definition.ts @@ -3,16 +3,15 @@ import { scanParametrics } from './parametrics' import { ScanNode } from './schema' /** - * Scan — Stage A. Mesh imported from the capture pipeline (LiDAR / - * photogrammetry). `ScanSystem` handles mesh loading + per-frame - * positioning; renderer mounts the imported geometry. + * Scan — Stage A. Capture-session reference with an optional renderable + * mesh. Raw sensor streams stay in the external session manifest. */ export const scanDefinition: NodeDefinition = { kind: 'scan', // Heavy LiDAR asset: stripped from the bake, re-added live from scene_graph // in the viewer (see plans → Part D; glb-reference-nodes.tsx). bake: 'strip', - schemaVersion: 1, + schemaVersion: 4, schema: ScanNode, category: 'site', @@ -24,6 +23,9 @@ export const scanDefinition: NodeDefinition = { capabilities: { selectable: { hitVolume: 'bbox' }, + movable: { axes: ['x', 'y', 'z'], gridSnap: true }, + rotatable: { axes: ['y'], snapAngles: [Math.PI / 4] }, + scalable: { axes: ['x', 'y', 'z'], min: 0.01, max: 10 }, duplicable: false, deletable: true, // Scans carry user-uploaded imagery — cataloging them as @@ -43,14 +45,14 @@ export const scanDefinition: NodeDefinition = { }, presentation: { - label: 'Scan', - description: 'A captured mesh (LiDAR / photogrammetry) imported as a scene reference.', + label: 'Capture', + description: 'A captured session with optional mesh, motion, media, and sensor data.', icon: { kind: 'url', src: '/icons/mesh.webp' }, paletteSection: 'site', paletteOrder: 40, }, mcp: { - description: 'A captured mesh import.', + description: 'A captured session reference with an optional renderable mesh.', }, } diff --git a/packages/nodes/src/scan/parametrics.ts b/packages/nodes/src/scan/parametrics.ts index 0852530d8f..6e2f389aa2 100644 --- a/packages/nodes/src/scan/parametrics.ts +++ b/packages/nodes/src/scan/parametrics.ts @@ -1,5 +1,17 @@ import type { ParametricDescriptor, ScanNode } from '@pascal-app/core' export const scanParametrics: ParametricDescriptor = { - groups: [], + groups: [ + { + label: 'Transform', + fields: [ + { key: 'position', kind: 'vec3' }, + { key: 'scale', kind: 'number', min: 0.01, max: 10, step: 0.1 }, + ], + }, + { + label: 'Appearance', + fields: [{ key: 'opacity', kind: 'number', unit: '%', min: 0, max: 100, step: 1 }], + }, + ], } diff --git a/packages/nodes/src/scan/renderer.tsx b/packages/nodes/src/scan/renderer.tsx index e05698740c..98376a65db 100644 --- a/packages/nodes/src/scan/renderer.tsx +++ b/packages/nodes/src/scan/renderer.tsx @@ -7,28 +7,37 @@ import type { Group, Material, Mesh } from 'three' export const ScanRenderer = ({ node }: { node: ScanNode }) => { const showScans = useViewer((s) => s.showScans) + const visible = showScans && node.visible const ref = useRef(null!) useRegistry(node.id, 'scan', ref) - const resolvedUrl = useAssetUrl(node.url) - return ( - {resolvedUrl && ( - - - + {visible && (node.layers?.model ?? true) && node.url && ( + )} ) } +const ScanAsset = ({ url, opacity }: { url: string; opacity: number }) => { + const resolvedUrl = useAssetUrl(url) + + if (!resolvedUrl) return null + + return ( + + + + ) +} + const ScanModel = ({ url, opacity }: { url: string; opacity: number }) => { const gltf = useGLTFKTX2(url) as any const scene = gltf.scene diff --git a/packages/viewer/src/systems/scan/scan-system.tsx b/packages/viewer/src/systems/scan/scan-system.tsx index 5704c690ad..9b50959f5c 100644 --- a/packages/viewer/src/systems/scan/scan-system.tsx +++ b/packages/viewer/src/systems/scan/scan-system.tsx @@ -1,19 +1,19 @@ -import { sceneRegistry } from '@pascal-app/core' +import { type ScanNode, sceneRegistry, useScene } from '@pascal-app/core' import { useEffect } from 'react' import useViewer from '../../store/use-viewer' export const ScanSystem = () => { const showScans = useViewer((state) => state.showScans) + const nodes = useScene((state) => state.nodes) useEffect(() => { const scans = sceneRegistry.byType.scan || new Set() scans.forEach((scanId) => { const node = sceneRegistry.nodes.get(scanId) - if (node) { - node.visible = showScans - } + const scan = nodes[scanId as ScanNode['id']] + if (node && scan?.type === 'scan') node.visible = showScans && scan.visible }) - }, [showScans]) + }, [nodes, showScans]) return null } diff --git a/wiki/architecture/README.md b/wiki/architecture/README.md index 51dbf359e6..0ef72a8255 100644 --- a/wiki/architecture/README.md +++ b/wiki/architecture/README.md @@ -17,6 +17,7 @@ Canonical rules for code that touches `packages/core`, `packages/viewer`, `packa | [measurements](measurements.md) | Persistent measurement data, 2D/3D draft ownership, snapping, units, and visibility | | [interaction-scope](interaction-scope.md) | The authoritative interaction state machine ("the spine"): `InteractionScope` union, the begin/update/end/endIf contract, the raycast hot-set, and the overlay scope matrix | | [viewer-isolation](viewer-isolation.md) | Keeping `@pascal-app/viewer` editor-agnostic | +| [capture-runtime](capture-runtime.md) | Open capture protocol, host source boundary, static/live viewer layers, and stream extension | | [selection-managers](selection-managers.md) | Two-layer selection (viewer + editor), events, outliner | | [selection-groups](selection-groups.md) | Session multi-select groups (Ctrl/Cmd+G), expand-on-click, how they differ from collections | | [scene-registry](scene-registry.md) | Global node ID → Object3D map and `useRegistry` | diff --git a/wiki/architecture/capture-runtime.md b/wiki/architecture/capture-runtime.md new file mode 100644 index 0000000000..8df4a6b66b --- /dev/null +++ b/wiki/architecture/capture-runtime.md @@ -0,0 +1,51 @@ +# Capture runtime + +Capture data is an optional viewer extension, not a private Community renderer and not a second +scene graph. + +## Package boundaries + +- `@pascal-app/capture-protocol` owns versioned manifests, normalized stream descriptors, stable + session locators, incremental packet headers, and the `CaptureSource` interface. It has no React, + Three.js, authentication, database, or prescribed transport. +- `@pascal-app/capture-viewer` mounts inside `Viewer` through its existing children slot. It resolves + `scan.captureSession`, portals layers into that scan node's registered group, honors per-layer + visibility, composes declared local-to-parent coordinate frames into session space, and supplies + reference model, device-motion, point-cloud, and compact color-surface renderers. +- `@pascal-app/core` stores only the scene anchor: session locator, optional current mesh URL, + placement, opacity, and an extensible visibility map. Raw samples and artifact inventories never + enter scene JSON. +- A host owns source resolution, access control, signed URLs, persistence, retention, collaboration, + and transport selection. Community's resolver uses its authenticated capture manifest route. + +## Static and live use the same source + +Every source implements `describe()`. Static HTTP sources stop there. Live sources additionally +implement `subscribe()` and yield descriptor changes or bounded stream packets. The runtime applies +generation and sequence ordering before renderers consume packets. + +The protocol intentionally does not choose WebSocket, WebRTC, Supabase Realtime, or another +transport. An embedded viewer can use a public HTTP manifest; a local tool can use files or an +in-memory producer; Community can layer its collaboration and authorization model on the same +interface. + +Community deliberately does not mount capture artifacts in its public project viewer yet. Its +current manifest route requires edit access; a future public surface needs an explicit view-scoped +artifact and privacy policy before it can use the same runtime safely. + +## Stream extension + +Manifest v2 streams use stable IDs plus open `kind` and `role` strings. Known roles currently map to +`model`, `deviceMotion`, `pointCloud`, and `surfaceMesh`. The reference surface renderer accepts the +bounded quantized inline preview emitted by Capture; a future UV-textured or server-reconstructed +mesh can be another artifact-backed stream without changing `ScanNode`. Unknown streams remain +available to hosts, which can add a renderer keyed by role or kind without changing the scene +schema. A splat adapter should remain a separate composited renderer while still consuming the same +source and visibility contract. + +## Compatibility + +The protocol normalizes Community's v1 RoomPlan/device-motion manifest, so existing captures remain +viewable. `ScanNode` keeps legacy GLB-backed scans loadable, makes `manifestUrl` optional for +host-resolved sessions, and uses an extensible visibility record so adding a data modality does not +require another node-schema release.