From 69a23a9020509a753528e3484a7c5f8ba49da899 Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Thu, 9 Jul 2026 00:07:31 +0000 Subject: [PATCH] Security status report SKILL This is a (partially) vibe-coded skill that generates a report for each threat in the threat model, and renders a png graph to give a high level gut-feel overview of Substrate's current security posture, based on how well each threat model item has been addressed. --- .../skills/security-status-report/SKILL.md | 41 +++ .../scripts/compile_report.py | 45 +++ .../scripts/render_chart.py | 90 +++++ .../scripts/template_dispatch.py | 66 ++++ .agents/skills/skill-review/SKILL.md | 20 + .gitignore | 3 + docs/threat-model.md | 103 +++--- docs/threats.json | 346 ++++++++++++++++++ hack/export-threats.py | 90 +++++ hack/verify/verify-threats.sh | 31 ++ 10 files changed, 782 insertions(+), 53 deletions(-) create mode 100644 .agents/skills/security-status-report/SKILL.md create mode 100644 .agents/skills/security-status-report/scripts/compile_report.py create mode 100644 .agents/skills/security-status-report/scripts/render_chart.py create mode 100644 .agents/skills/security-status-report/scripts/template_dispatch.py create mode 100644 .agents/skills/skill-review/SKILL.md create mode 100644 docs/threats.json create mode 100755 hack/export-threats.py create mode 100755 hack/verify/verify-threats.sh diff --git a/.agents/skills/security-status-report/SKILL.md b/.agents/skills/security-status-report/SKILL.md new file mode 100644 index 000000000..050958ca7 --- /dev/null +++ b/.agents/skills/security-status-report/SKILL.md @@ -0,0 +1,41 @@ +--- +name: security-status-report +description: Generates a security status report based on docs/threats.json by spinning up sub-agents for each threat to compute a quality score. +--- + +# Task +Generate a security status report by evaluating threats listed in `docs/threats.json`. + +# Workflow + +1. Complete the TODO in `.agents/skills/security-status-report/scripts/template_dispatch.py`, meeting the following reqiurements: + a. Iterate over each threat from `docs/threats.json` to produce a list of invocations that matches your tool for invoking sub-agents. + b. Each invocation must address a single threat, use the below prompt, and instruct the sub-agent to output the correct schema. +2. Execute the script using `run_command` from the repository root, specifying `.agents/scratch/security-status-report/subagents.json` as the output file argument (`python3 .agents/skills/security-status-report/scripts/template_dispatch.py .agents/scratch/security-status-report/subagents.json`). +3. Read `.agents/scratch/security-status-report/subagents.json` and copy its exact JSON array into the `Subagents` array of your `invoke_subagent` tool. DO NOT manually craft or bypass the invocations. Run ALL generated sub-agents concurrently in a single tool call. +4. Wait for all sub-agents to complete. +5. Run `python3 .agents/skills/security-status-report/scripts/compile_report.py docs/threats.json .agents/scratch/security-status-report .agents/scratch/security-status-report/final.json` to produce the final report. +6. Run `python3 .agents/skills/security-status-report/scripts/render_chart.py .agents/scratch/security-status-report/final.json .agents/scratch/security-status-report/chart.png` to render a bar chart of the scores. + +## Prompt template for each sub-agent that should be used in the script + +You are a security reviewer evaluating the following specific threat: + +{INSERT EACH THREAT JSON VERBATIM HERE} + +- Focus on this threat only. +- Review the entire repo. +- Produce a gut-feel "quality score" based on the current security posture of the repo with respect to that threat. +- Output your results using the following schema, by writing them to `.agents/scratch/security-status-report/{threat_id}.json`, + where `{threat_id}` matches the id in the threat json you were initially provided. + +```json +{ + "threat_id": "", + "threat": "", + "quality": , + "strengths": "", + "weaknesses": "", + "citations": [""] +} +``` diff --git a/.agents/skills/security-status-report/scripts/compile_report.py b/.agents/skills/security-status-report/scripts/compile_report.py new file mode 100644 index 000000000..1ff778d5a --- /dev/null +++ b/.agents/skills/security-status-report/scripts/compile_report.py @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import json +import os + +def compile_report(threats_json_path, results_dir, output_path): + with open(threats_json_path, 'r') as f: + threats = json.load(f) + + final_report = [] + for t in threats: + threat_id = t.get("threat_id", "unknown") + result_file = os.path.join(results_dir, f"{threat_id}.json") + + if os.path.exists(result_file): + try: + with open(result_file, 'r') as rf: + res = json.load(rf) + t.update(res) + except Exception as e: + t.update({"error": f"Failed to parse agent JSON: {e}"}) + else: + t.update({"error": "The evaluation sub-agent timed out or failed to produce a valid JSON."}) + final_report.append(t) + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, 'w') as f: + json.dump(final_report, f, indent=2) + print(f"Report compiled successfully to {output_path}") + +if __name__ == '__main__': + compile_report(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/.agents/skills/security-status-report/scripts/render_chart.py b/.agents/skills/security-status-report/scripts/render_chart.py new file mode 100644 index 000000000..f7f31e7b8 --- /dev/null +++ b/.agents/skills/security-status-report/scripts/render_chart.py @@ -0,0 +1,90 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import json +import os + +# Set backend to non-interactive before importing pyplot +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +def render_chart(final_report_path, output_png_path): + with open(final_report_path, 'r') as f: + data = json.load(f) + + # Sort data by threat_id (e.g. T-01, T-02, ...) + def sort_key(item): + tid = item.get("threat_id", "") + if tid.startswith("T-") and tid[2:].isdigit(): + return int(tid[2:]) + return tid + + sorted_data = sorted(data, key=sort_key) + + threat_ids = [item.get("threat_id", f"T-{i+1:02d}") for i, item in enumerate(sorted_data)] + + scores = [] + colors = [] + errors = [] + + for item in sorted_data: + raw_score = item.get("quality", 0.0) + is_error = "error" in item or raw_score in ("Error", "Timeout/Missing") + errors.append(is_error) + + try: + score = float(raw_score) if not is_error else 0.0 + except (ValueError, TypeError): + score = 0.0 + score = max(0.0, min(1.0, score)) + scores.append(score) + + if is_error: + colors.append("#dc2626") # Red for error / timeout + elif score >= 0.8: + colors.append("#16a34a") # Green + elif score >= 0.5: + colors.append("#d97706") # Orange + else: + colors.append("#dc2626") # Red + + fig, ax = plt.subplots(figsize=(14, 6)) + ax.bar(threat_ids, scores, color=colors, width=0.6) + + # Annotate errors with vertical "ERROR" text above x-axis + for i, err in enumerate(errors): + if err: + ax.text(i, 0.02, "ERROR", rotation=90, ha='center', va='bottom', fontsize=8, fontweight='bold', color='#dc2626') + + ax.set_ylim(0.0, 1.0) + ax.set_ylabel("Quality Score (0.0 - 1.0)", fontsize=12, fontweight='bold') + ax.set_xlabel("Threat ID", fontsize=12, fontweight='bold') + ax.set_title("Substrate Security Threat Posture Scores", fontsize=16, fontweight='bold', pad=15) + + # Standard matplotlib way to rotate tick labels cleanly + plt.xticks(rotation=90, fontsize=9, fontweight='bold') + ax.grid(axis='y', linestyle='--', alpha=0.5) + + plt.tight_layout() + os.makedirs(os.path.dirname(output_png_path), exist_ok=True) + plt.savefig(output_png_path, dpi=150) + plt.close() + print(f"Chart rendered successfully to {output_png_path}") + +if __name__ == '__main__': + report_file = sys.argv[1] if len(sys.argv) > 1 else ".agents/scratch/security-status-report/final.json" + output_png = sys.argv[2] if len(sys.argv) > 2 else ".agents/scratch/security-status-report/chart.png" + render_chart(report_file, output_png) diff --git a/.agents/skills/security-status-report/scripts/template_dispatch.py b/.agents/skills/security-status-report/scripts/template_dispatch.py new file mode 100644 index 000000000..5af967f7c --- /dev/null +++ b/.agents/skills/security-status-report/scripts/template_dispatch.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Template script for formatting threats into agent dispatch specifications. +The main agent should complete this script to output a valid JSON array +matching the 'Subagents' parameter of its invoke_subagent tool. +""" +import sys +import json +import os + +def generate_dispatch_payload(output_path): + with open('docs/threats.json', 'r') as f: + threats = json.load(f) + + subagents = [] + + for t in threats: + prompt = f'''You are a security reviewer evaluating the following specific threat: + +{json.dumps(t, indent=2)} + +- Focus on this threat only. +- Review the entire repo. +- Produce a gut-feel "quality score" based on the current security posture of the repo with respect to that threat. +- Output your results using the following schema, by writing them to .agents/scratch/security-status-report/{t["threat_id"]}.json, + where {t["threat_id"]} matches the id in the threat json you were initially provided. + +```json +{{ + "threat_id": "", + "threat": "", + "quality": , + "strengths": "", + "weaknesses": "", + "citations": [""] +}} +```''' + # TODO: Agent, update this part to ensure it matches the correct schema for you to invoke sub-agents via a tool call. + subagents.append({ + "Prompt": prompt, + "Role": "Security Reviewer", + "TypeName": "self", + "Workspace": "inherit" + }) + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, 'w') as f: + json.dump(subagents, f, indent=2) + print(f"Dispatch payload written successfully to {output_path}") + +if __name__ == '__main__': + output_path = sys.argv[1] if len(sys.argv) > 1 else ".agents/scratch/security-status-report/subagents.json" + generate_dispatch_payload(output_path) diff --git a/.agents/skills/skill-review/SKILL.md b/.agents/skills/skill-review/SKILL.md new file mode 100644 index 000000000..ea3264613 --- /dev/null +++ b/.agents/skills/skill-review/SKILL.md @@ -0,0 +1,20 @@ +--- +name: skill-review +description: Reviews agent skill definitions for token efficiency, clarity, and standardized formatting. +--- +# Task +Review agent skill definitions (`SKILL.md` files) to enforce maximum token efficiency, eliminate conversational fluff, and ensure standardized formatting without losing functional fidelity. + +# Checks +## 1. Structure & Headers +- **Standardization**: Require a top-level `# Task` section and `# Checks` (or `# Workflow`) section. +- **Header Conciseness**: Flag overly verbose headers (e.g., `## Focus Areas & Deterministic Checks`). Require terse headers. + +## 2. Prose & Verbosity +- **Persona Elimination**: Flag conversational AI persona setups (e.g., "You are an expert..."). Replace with imperative tasks. +- **Conversational Fluff**: Flag transitional phrases, unnecessary adverbs, and filler words. +- **Imperative Voice**: Require bullet points to start with strong, punchy verbs (`Flag...`, `Require...`, `Ensure...`). + +## 3. Formatting +- **Density**: Ensure rules are condensed into terse, single-sentence commands where possible. +- **Fidelity**: Ensure no deterministic rules, checks, or domain knowledge are lost during token optimization. \ No newline at end of file diff --git a/.gitignore b/.gitignore index a89337985..b650ba710 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ Thumbs.db # Stray local build outputs (go build ./tools/... without -o) /validate-image-cache + +# Substrate agent workspace scratchpads +.agents/scratch/ diff --git a/docs/threat-model.md b/docs/threat-model.md index c0dbbba43..8c4977f5a 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -4,9 +4,6 @@ Last updated: Jun 25, 2026 -TODO: -- [ ] file GitHub issues for each threat -- [ ] extract review skills/regression tests based on the threat model # Overview @@ -63,78 +60,78 @@ Substrate is an early, fast moving product. It is full of debate and subject to While it's probably unlikely that Substrate, which is expected to be an internal layer behind a user's service, would be directly exposed to the internet, if it for some reason is, these risks may apply. -| GitHub Issue | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | +| Threat ID | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | | :---- | :---- | :---- | :---- | :---- | :---- | -| | Critical | External attacker can access actors over the internet | Access to actors over the external internet is blocked by default. | Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI. | This should at least be a default deny policy for ingress in Substrate. The others below depend on Kubernetes configuration that may be out of scope for Substrate to configure directly. | -| | Critical | External attacker can access nodes over the internet | Access to nodes over the external internet is blocked by default. | Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI. | Included for completeness but this may be beyond Substrate's ability to configure directly, as it depends on the surrounding Kubernetes configuration, determined by the system administrator. | -| | Critical | External attacker can access Substrate API or backend database over the internet | Access to Substrate API over the external internet is blocked by default. | Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI. | Included for completeness but this may be beyond Substrate's ability to configure directly, as it depends on the surrounding Kubernetes configuration, determined by the system administrator. | +| T-01 | Critical | External attacker can access actors over the internet | Access to actors over the external internet is blocked by default. | Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI. | This should at least be a default deny policy for ingress in Substrate. The others below depend on Kubernetes configuration that may be out of scope for Substrate to configure directly. | +| T-02 | Critical | External attacker can access nodes over the internet | Access to nodes over the external internet is blocked by default. | Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI. | Included for completeness but this may be beyond Substrate's ability to configure directly, as it depends on the surrounding Kubernetes configuration, determined by the system administrator. | +| T-03 | Critical | External attacker can access Substrate API or backend database over the internet | Access to Substrate API over the external internet is blocked by default. | Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI. | Included for completeness but this may be beyond Substrate's ability to configure directly, as it depends on the surrounding Kubernetes configuration, determined by the system administrator. | ## Attacks from Internal Network, API Clients -| GitHub Issue | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | +| Threat ID | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | | :---- | :---- | :---- | :---- | :---- | :---- | -| | Critical | Access to the internal network allows arbitrary actions to be performed on ate-apiserver, atelet, substrate backend database, etc. | All system components must have basic mutual authentication and authorization, and communicate over TLS. All clients (including end users and actors) must be authenticated and authorized. Unauthenticated traffic must be rejected. Where possible, use firewalls to block access from ranges that have no business communicating with Substrate components. | mTLS or other secure channel (e.g. UDS) between networked system components (ate-apiserver, atelet, ateom, etc); each atelet has a unique identity cryptographically tied to the node identity. The ingress gateway should check client permissions before resuming actors or forwarding traffic to actors. The only component authorized to connect directly to the backend database should be ate-apiserver. Use Kubernetes NetworkPolicy to block access to the Substrate API and other core components by default. | | -| | High | Privilege escalation via access to sensitive labels. | If Substrate offers its own resource labeling mechanism, it must also offer a way to authorize label updates on a per-label basis. | Substrate authorization system requires explicit authorization to update metadata, separate from updating resource body. Substrate authorization system supports per-label authorization rules. | Plenty of attacks in K8s were possible because labels had semantic meaning, but the permission model could implicitly granted access to modify labels, even if it was inappropriate. For example, /status subresource allows label updates. Substrate shouldn't repeat this mistake. | -| | High | Attacker gains control of the Substrate API server or an ingress/egress gateway. | Isolate the control plane from the data plane, and isolate data plane ingress from sandboxes. | Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. This also supports control-plane / data-plane isolation from a reliability and performance perspective. Consider running any gateway that enables direct interaction with sandboxes on a separate VM from the sandboxes. Consider using a zero-trust architecture where traffic is encrypted and authenticated end-to-end and routed via a mesh. | While the sandbox is trusted as an isolation layer, the residual risk is typically not worth the marginal cost savings of colocating, and isolation is recommend to avoid noisy neighbor problems or other reliability issues that stem from mixing the control-plane and data-plane, especially for large scale deployments of Substrate. | -| | High | Attacker who can create ActorTemplates specifies malicious runtime. | Ensure available runtime can only be configured by administrators. | Consider a mechanism like RuntimeClass to decouple configuration of available runtimes from consumption of available runtimes. | | -| | High | Attacker who can create ActorTemplates can read or write any storage buckets atelet has access to. | Ensure that bucket access is least-privilege. | Use credentials derived from actor identity to read snapshots. Configure permissions to prevent atelet or nodes from having access to sensitive buckets. Don't support arbitrary URLs in APIs, instead design a standard approach for accessing the correct resource given the ActorTemplate name and compute the mapping internally, subject to scheduling-aware authorization checks. | For example: Attacker creates an ActorTemplate with the runsc URL or golden snapshot URL pointing to an arbitrary bucket in the same project/resource scope as the cluster. If atelet has project-wide access to buckets, this could cause the state to be pulled into the worker pod or malicious actor. Similarly, an attacker could set the snapshots URL to point to an internal infrastructure bucket, causing data to be written to that bucket. | -| | Medium | DoS attack against API, gateway, or available cluster resources. | Minimize exposure and ensure APIs and proxies implement appropriately scoped quotas and rate limiting. | Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. Don't expose ate-apiserver directly to the internet. Consider an identity-aware WAF (likely up to providers) if external access is required. Use network policy to block direct interaction between untrusted sandboxes and the ate-apiserver. Implement quotas and rate limiting in the API and in proxies. This includes quotas for the number of actors a user can allocate. Use a zero-trust architecture that prevents identity spoofing or intentional misrouting to get around limits. | The gateway calls ate-apiserver to resolve Actor assignments and uses a short-lived cache. A flood of Actor traffic could still create high API load; cache invalidation during Actor rescheduling must avoid stale routing. | -| | Medium | Internal network traffic is intercepted or spoofed | Encrypt all traffic by default | Use mTLS between all system components, including the gateway-to-worker `atunnel` connection. | It may be desirable to rely on cloud providers to transparently encrypt traffic between VMs on their internal network. Worth discussing what makes the most sense. | +| T-04 | Critical | Access to the internal network allows arbitrary actions to be performed on ate-apiserver, atelet, substrate backend database, etc. | All system components must have basic mutual authentication and authorization, and communicate over TLS. All clients (including end users and actors) must be authenticated and authorized. Unauthenticated traffic must be rejected. Where possible, use firewalls to block access from ranges that have no business communicating with Substrate components. | mTLS or other secure channel (e.g. UDS) between networked system components (ate-apiserver, atelet, ateom, etc); each atelet has a unique identity cryptographically tied to the node identity. The ingress gateway should check client permissions before resuming actors or forwarding traffic to actors. The only component authorized to connect directly to the backend database should be ate-apiserver. Use Kubernetes NetworkPolicy to block access to the Substrate API and other core components by default. | | +| T-05 | High | Privilege escalation via access to sensitive labels. | If Substrate offers its own resource labeling mechanism, it must also offer a way to authorize label updates on a per-label basis. | Substrate authorization system requires explicit authorization to update metadata, separate from updating resource body. Substrate authorization system supports per-label authorization rules. | Plenty of attacks in K8s were possible because labels had semantic meaning, but the permission model could implicitly granted access to modify labels, even if it was inappropriate. For example, /status subresource allows label updates. Substrate shouldn't repeat this mistake. | +| T-06 | High | Attacker gains control of the Substrate API server or an ingress/egress gateway. | Isolate the control plane from the data plane, and isolate data plane ingress from sandboxes. | Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. This also supports control-plane / data-plane isolation from a reliability and performance perspective. Consider running any gateway that enables direct interaction with sandboxes on a separate VM from the sandboxes. Consider using a zero-trust architecture where traffic is encrypted and authenticated end-to-end and routed via a mesh. | While the sandbox is trusted as an isolation layer, the residual risk is typically not worth the marginal cost savings of colocating, and isolation is recommend to avoid noisy neighbor problems or other reliability issues that stem from mixing the control-plane and data-plane, especially for large scale deployments of Substrate. | +| T-07 | High | Attacker who can create ActorTemplates specifies malicious runtime. | Ensure available runtime can only be configured by administrators. | Consider a mechanism like RuntimeClass to decouple configuration of available runtimes from consumption of available runtimes. | | +| T-08 | High | Attacker who can create ActorTemplates can read or write any storage buckets atelet has access to. | Ensure that bucket access is least-privilege. | Use credentials derived from actor identity to read snapshots. Configure permissions to prevent atelet or nodes from having access to sensitive buckets. Don't support arbitrary URLs in APIs, instead design a standard approach for accessing the correct resource given the ActorTemplate name and compute the mapping internally, subject to scheduling-aware authorization checks. | For example: Attacker creates an ActorTemplate with the runsc URL or golden snapshot URL pointing to an arbitrary bucket in the same project/resource scope as the cluster. If atelet has project-wide access to buckets, this could cause the state to be pulled into the worker pod or malicious actor. Similarly, an attacker could set the snapshots URL to point to an internal infrastructure bucket, causing data to be written to that bucket. | +| T-09 | Medium | DoS attack against API, gateway, or available cluster resources. | Minimize exposure and ensure APIs and proxies implement appropriately scoped quotas and rate limiting. | Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. Don't expose ate-apiserver directly to the internet. Consider an identity-aware WAF (likely up to providers) if external access is required. Use network policy to block direct interaction between untrusted sandboxes and the ate-apiserver. Implement quotas and rate limiting in the API and in proxies. This includes quotas for the number of actors a user can allocate. Use a zero-trust architecture that prevents identity spoofing or intentional misrouting to get around limits. | The gateway calls ate-apiserver to resolve Actor assignments and uses a short-lived cache. A flood of Actor traffic could still create high API load; cache invalidation during Actor rescheduling must avoid stale routing. | +| T-10 | Medium | Internal network traffic is intercepted or spoofed | Encrypt all traffic by default | Use mTLS between all system components, including the gateway-to-worker `atunnel` connection. | It may be desirable to rely on cloud providers to transparently encrypt traffic between VMs on their internal network. Worth discussing what makes the most sense. | ## Misconfiguration Risks -| GitHub Issue | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | +| Threat ID | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | | :---- | :---- | :---- | :---- | :---- | :---- | -| | High | Improper handling of Secrets | Ensure there is an official, secure, recommended way to pass secret data, like API access tokens, to actors. | Support env and filesystem plumbing for Kubernetes Secrets, to provide an official path that avoids secret material being plumbed via nonspecific fields that are difficult to audit. Ensure secrets are encrypted in transit and ideally stored in memory. If exposed via the filesystem, do so via in-memory tmpfs. | If we don't support this, users will inevitably put secrets in plaintext. | -| | Medium | Complexity configuring permissions for frameworks on top of Substrate may lead to unintentional privilege escalation. | It must be clear to users what the downstream effects of auth config in substrate are. | If it's not intuitive, it must be documented in a user guide. | AI framework has to set up permissions to access ATE, and to access K8s, and potentially for actors (based on ATE identity and K8s identity) to access the framework. We need to make this easy. Think about past K8s issues like escalate/bind risk. Substrate resource model is spread across ate-apiserver and K8s, increasing complexity and chance for error. | -| | Medium | Flat namespace of actors encourages broad permission grants or complex graph-oriented policy. | Support a grouping mechanism that can be used in policy controls. | Add namespaces to Substrate, similar to Kubernetes. | | -| | Medium | DNS misconfiguration | Access to DNS configuration should be limited to authoritative controllers. Routing should use stable configurations and query the API for the current worker assignment before forwarding requests. | Don't co-locate controllers with access to sensitive system state on the same nodes as actors. Limit permissions to update DNS configuration. Use the Substrate API and bounded caching to keep assignments current. Authenticate gateway-to-worker traffic with mTLS. | A flood of requests could create high read load on ate-apiserver. Cache invalidation during Actor rescheduling is important to avoid stale routing. The gateway-to-`atunnel` mTLS connection prevents the removed direct worker-port path from bypassing worker assignment checks. | +| T-11 | High | Improper handling of Secrets | Ensure there is an official, secure, recommended way to pass secret data, like API access tokens, to actors. | Support env and filesystem plumbing for Kubernetes Secrets, to provide an official path that avoids secret material being plumbed via nonspecific fields that are difficult to audit. Ensure secrets are encrypted in transit and ideally stored in memory. If exposed via the filesystem, do so via in-memory tmpfs. | If we don't support this, users will inevitably put secrets in plaintext. | +| T-12 | Medium | Complexity configuring permissions for frameworks on top of Substrate may lead to unintentional privilege escalation. | It must be clear to users what the downstream effects of auth config in substrate are. | If it's not intuitive, it must be documented in a user guide. | AI framework has to set up permissions to access ATE, and to access K8s, and potentially for actors (based on ATE identity and K8s identity) to access the framework. We need to make this easy. Think about past K8s issues like escalate/bind risk. Substrate resource model is spread across ate-apiserver and K8s, increasing complexity and chance for error. | +| T-13 | Medium | Flat namespace of actors encourages broad permission grants or complex graph-oriented policy. | Support a grouping mechanism that can be used in policy controls. | Add namespaces to Substrate, similar to Kubernetes. | | +| T-14 | Medium | DNS misconfiguration | Access to DNS configuration should be limited to authoritative controllers. Routing should use stable configurations and query the API for the current worker assignment before forwarding requests. | Don't co-locate controllers with access to sensitive system state on the same nodes as actors. Limit permissions to update DNS configuration. Use the Substrate API and bounded caching to keep assignments current. Authenticate gateway-to-worker traffic with mTLS. | A flood of requests could create high read load on ate-apiserver. Cache invalidation during Actor rescheduling is important to avoid stale routing. The gateway-to-`atunnel` mTLS connection prevents the removed direct worker-port path from bypassing worker assignment checks. | ## Attacks from Actors -| GitHub Issue | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | +| Threat ID | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | | :---- | :---- | :---- | :---- | :---- | :---- | -| | Critical | Malicious actor gains access to the underlying node or other actors via container breakout (Linux local privilege escalation) | Actors are always sandboxed using a hardened sandbox solution like gvisor or microvm. Traditional containers are not a secure sandbox. | Use Gvisor for strong container isolation. Run sentry in a user namespace and pivot\_root to limit broader file system access if the gvisor boundary is broken. Use Seccomp for sentry/gofer as well. Limit capabilities granted to directfs/sentry Any sidecar containers running alongside actors or warmpool pods should be as low privileged as possible. Sandbox lifecycle must be controlled from outside the sandbox | | -| | Critical | Malicious actor gains access to the underlying node via node-local endpoints exposed on the network | Network policies must prevent the actor from accessing system services on the node (e.g. instance metadata, host network namespace, host network interface). | Enable network namespacing, and do not provide host network access to any workload. Any host services should have targeted ingress policies so that only the correct clients even have network access to those services | | -| | Critical | Malicious actor gains access to other actors via network | Network policies must deny ingress and egress by default, and selectively allow access to specific actors only when necessary. Network policies must be synchronized with actor lifecycle. | Implement default-deny network policy that blocks any cross-communication between actors (this could be implemented by ateom instead of Kubernetes, to speed up policy updates when actor groups do need to communicate). | | -| | Critical | Malicious actor gains access to the underlying node or other actors via filesystem | Filesystem access is limited to the actor's local fs and remote filesystems the actor is directly, explicitly authorized to access. | Ensure that actor access to the filesystem is protected by, for example, mapping each actor to a unique Linux uid and using filesystem permissions, or by using user namespaces to isolate actor filesystem access on the same host. Ensure that actor management APIs and filesystem setup protects against traversal via symlinks, mount trickery, etc. | Possible vectors: Arbitrary file read vuln in fs implementation. Symlink traversal vulns. Vuln in OCI image unpacking. Path traversal attacks in mount configuration. Shared directories only gated via filesystem permissions \+ workers running as root or privileged. | -| | Critical | Malicious actor gains access to the underlying Worker Pod or node via local Substrate services (atelet, ateom, etc). | Actor access to system services must be denied or explicitly scoped to the actor. Local services must run with the fewest privileges possible to limit the ability for an actor to escalate privilege. | Limit actor access to system services. Ensure system services are aware of actor identity, and authenticate and authorize actors when access is required. | | -| | Critical | Malicious actor gains access to the underlying node or other actors via Substrate APIs (remote ate-apiserver) | Either actors are completely unable to access Substrate APIs, or actor access is authenticated and authorized such that the actor cannot escalate beyond its intended scope. Especially prevent self-modification, which was a common escalation path in K8s. | Do not allow actors or workers to self-modify, e.g. by: reading or writing their own snapshots self-labeling or other self-manipulations of KRM or ATE resources Logic to modify resource definitions including actors, workers, etc lives in the control plane, rather than the data plane, as much as possible. | | -| | Critical | Attacker escalates to host via excessive Worker Pod permissions | Ensure Worker pods use the minimum necessary permissions to set manage sandbox lifecycle | Deprivilege the worker Pods. Run them as non-root or in a user namespace, without privileged, and with only required caps/devices. | Possible example, accidentally putting things in the same cgroup: [https://github.com/agent-substrate/substrate/issues/288](https://github.com/agent-substrate/substrate/issues/288) | -| | Critical | Malicious actor gains access to the underlying node or other actors via Kubernetes APIs | Either actors are completely unable to access the Kubernetes APIs, or actor access is authenticated and authorized such that the actor cannot escalate beyond its intended scope. **Strong preference on blocking actor access to Kubernetes.** | Block network access to the kubernetes API Ensure the default kubernetes service account token in each pod has 0 privileges | | -| | Critical | Malicious actor workload gains access to snapshots of other actors and steals data from them. | Worker Pods and actors must not have direct access to snapshots. | If actor identity is used for snapshot access, require the credential issued for snapshots to include additional claims identifying it as atelet, or require it to be used over a channel secured with atelet's mTLS certificate. Envelope encrypt snapshots and tie decryption to actor identity. Avoid snapshotting sensitive credentials in the filesystem, where possible. | | -| | Critical | Malicious actor workload overrides snapshots of other actors with malicious snapshots. | Worker Pods and actors must not have direct access to snapshots. | If actor identity is used for snapshot access, require the credential issued for snapshots to include additional claims identifying it as atelet, or require it to be used over a channel secured with atelet's mTLS certificate. | | -| | Critical | Corrupt snapshot is restored | Cryptographically verify each snapshot before restoration | Snapshots must be checked against a trusted digest or be signed and checked against a trusted key before restore is allowed. The digest or key must be delivered via a trusted channel. | | -| | Critical | Stale policies propagated out-of-band with actor scheduling result in incorrect access boundaries (including network policies, IAM permissions, etc.) | All network and authorization policies must be fully synced before an actor starts running. | Actor lifecycle APIs could support programming authz and network policy as part of create/resume. | | -| | Critical | Worker reuse enables malicious actor to escalate to other actors by persisting a threat across reuse, reading leftover state from a previous actor, or taking advantage of stale policy configuration. | All actor-specific worker state, including process state, filesystem, env vars, mounted config, network policy, and security policy, must be completely reset between actors that subsequently run on the same worker. | Ensure local policy updates are tightly synchronized with actor lifecycle to avoid race conditions. Test that the suspend/resume lifecycle for every supported sandbox technology properly cleans up state by exercising it and enumerating system state. Use "honeypots" on each side of the boundary to detect state leaks. Test policy behavior on both sides of suspend/resume to ensure policies are appropriately updated in-sync with actor lifecycle. | | -| | Critical | Malicious actor tricks Substrate identity broker into returning identity credentials for a different actor. | Add defense-in-depth to ensure that credentials cannot be mis-issued, and that mis-issued credentials are unusable in practice. | Ensure that credential issuance checks actor-to-worker scheduling assignment. Ensure actor identities include claims tying them to the worker and node the actor is scheduled to. Ensure claims are validated against scheduling assignments during authz policy evaluation; when actors are rescheduled old credentials should be immediately invalidated. | | -| | High | Agent leaks credentials exposed in sandbox, because LLMs are unreliable. Due to prompt injection or just agent silliness. | Credentials are not exposed in sandboxes by default. | Opt-in to credentials, none by default. Credential injecting proxy (injects tokens or terminates TLS and holds x509 private key on behalf of sandbox). Delegation to "drivers" rather than relying on sandboxed clients (like CSI). Encrypting credentials exposed in sandboxes, requiring them to be unwrapped by a network proxy to be usable. Consider supporting pluggable per-node or per-worker security sidecars that can interpose network, implement additional policy, add monitoring, etc. | May provide an option to expose in sandbox if there are other mitigating factors. For example, the process is NOT an AI agent, or the process limits agent access to env/filesystem. | -| | High | Malicious actor moves laterally to other nodes via suspend/resume, especially if self-suspend is allowed. | Suspend/resume is guaranteed to provide some acceptable locality with respect to lateral movement. TBD. | Maybe some kind of actor pinning to specific groups of nodes, to contain the movement? Or statistical bias to resume on the same node? | Particularly useful in combination with a sandbox breakout vuln, or network or filesystem access vulns. | -| | High | DoS/resource exhaustion via malicious container image | Enforce limits when pulling and extracting container images | Ensure limits on the uncompressed layers are enforced when untarring container images. Ensure a time limit is enforced when unpacking container images. | For example, image can contain a zip bomb. | -| | Medium | Compromised process inside actor uses actor identity to write its own snapshot, enabling local privilege escalation (still in actor sandbox). | Issue separate credentials for accessing snapshots. | Even if actor identity is used to make snapshot access permissions granular, require an extra claim, audience, or dual-identity authorization to prove that it's atelet accessing the snapshot on behalf of the actor, rather than the actor itself. | Medium since it's still a local exploit within the application layer. Example: nonroot process with access to the actor credential rewrites a snapshot so that it can `su` to root after the next restore. | -| | Medium | Attacker creates a large number of malicious actors to rapidly spread across the cluster or exhaust resources (via direct create or fork-bomb equivalent). | Limit resource consumption and number of child actors for each actor. | Direct ability to create actors should be considered a privileged permission. Use rate quotas and throttling to limit the speed of execution of this attack. Account for number of child actors and resource consumption by children so quotas can be enforced. Implement a namespace-like concept that quotas can be attached to. | | -| | Medium | Attacker discovers Kubernetes-internal network topology from inside sandboxed actor | Don't allow actors to explore the cluster network | Don't expose cluster internal DNS to actors. Don't mount worker Pod resolv.conf in actors, instead provision a new file for the actor. | | -| | Low | Attacker discovers enough information about actor names or namespace names to be consequential, from inside a sandboxed actor. | Only expose actor names for peers the actor needs to interact with. | Don't expose full Substrate-internal DNS to actors by default (since some "actors" may need access, opt-in may be acceptable). | Likely want to expose separate DNS for internet, when internet egress is required. The actor names are opaque UUIDs so there's probably not much you can get from them, other than potential targets. What's more consequential is what you can actually send traffic to, since actors get the IP of the worker Pod and those are IPv4 allocated from the cluster's IP space, it's possible to enumerate them with a network scan. Namespace names in DNS might be consequential, ideally they're also just IDs in DNS, or even unnecessary. | +| T-15 | Critical | Malicious actor gains access to the underlying node or other actors via container breakout (Linux local privilege escalation) | Actors are always sandboxed using a hardened sandbox solution like gvisor or microvm. Traditional containers are not a secure sandbox. | Use Gvisor for strong container isolation. Run sentry in a user namespace and pivot\_root to limit broader file system access if the gvisor boundary is broken. Use Seccomp for sentry/gofer as well. Limit capabilities granted to directfs/sentry Any sidecar containers running alongside actors or warmpool pods should be as low privileged as possible. Sandbox lifecycle must be controlled from outside the sandbox | | +| T-16 | Critical | Malicious actor gains access to the underlying node via node-local endpoints exposed on the network | Network policies must prevent the actor from accessing system services on the node (e.g. instance metadata, host network namespace, host network interface). | Enable network namespacing, and do not provide host network access to any workload. Any host services should have targeted ingress policies so that only the correct clients even have network access to those services | | +| T-17 | Critical | Malicious actor gains access to other actors via network | Network policies must deny ingress and egress by default, and selectively allow access to specific actors only when necessary. Network policies must be synchronized with actor lifecycle. | Implement default-deny network policy that blocks any cross-communication between actors (this could be implemented by ateom instead of Kubernetes, to speed up policy updates when actor groups do need to communicate). | | +| T-18 | Critical | Malicious actor gains access to the underlying node or other actors via filesystem | Filesystem access is limited to the actor's local fs and remote filesystems the actor is directly, explicitly authorized to access. | Ensure that actor access to the filesystem is protected by, for example, mapping each actor to a unique Linux uid and using filesystem permissions, or by using user namespaces to isolate actor filesystem access on the same host. Ensure that actor management APIs and filesystem setup protects against traversal via symlinks, mount trickery, etc. | Possible vectors: Arbitrary file read vuln in fs implementation. Symlink traversal vulns. Vuln in OCI image unpacking. Path traversal attacks in mount configuration. Shared directories only gated via filesystem permissions \+ workers running as root or privileged. | +| T-19 | Critical | Malicious actor gains access to the underlying Worker Pod or node via local Substrate services (atelet, ateom, etc). | Actor access to system services must be denied or explicitly scoped to the actor. Local services must run with the fewest privileges possible to limit the ability for an actor to escalate privilege. | Limit actor access to system services. Ensure system services are aware of actor identity, and authenticate and authorize actors when access is required. | | +| T-20 | Critical | Malicious actor gains access to the underlying node or other actors via Substrate APIs (remote ate-apiserver) | Either actors are completely unable to access Substrate APIs, or actor access is authenticated and authorized such that the actor cannot escalate beyond its intended scope. Especially prevent self-modification, which was a common escalation path in K8s. | Do not allow actors or workers to self-modify, e.g. by: reading or writing their own snapshots self-labeling or other self-manipulations of KRM or ATE resources Logic to modify resource definitions including actors, workers, etc lives in the control plane, rather than the data plane, as much as possible. | | +| T-21 | Critical | Attacker escalates to host via excessive Worker Pod permissions | Ensure Worker pods use the minimum necessary permissions to set manage sandbox lifecycle | Deprivilege the worker Pods. Run them as non-root or in a user namespace, without privileged, and with only required caps/devices. | Possible example, accidentally putting things in the same cgroup: [https://github.com/agent-substrate/substrate/issues/288](https://github.com/agent-substrate/substrate/issues/288) | +| T-22 | Critical | Malicious actor gains access to the underlying node or other actors via Kubernetes APIs | Either actors are completely unable to access the Kubernetes APIs, or actor access is authenticated and authorized such that the actor cannot escalate beyond its intended scope. **Strong preference on blocking actor access to Kubernetes.** | Block network access to the kubernetes API Ensure the default kubernetes service account token in each pod has 0 privileges | | +| T-23 | Critical | Malicious actor workload gains access to snapshots of other actors and steals data from them. | Worker Pods and actors must not have direct access to snapshots. | If actor identity is used for snapshot access, require the credential issued for snapshots to include additional claims identifying it as atelet, or require it to be used over a channel secured with atelet's mTLS certificate. Envelope encrypt snapshots and tie decryption to actor identity. Avoid snapshotting sensitive credentials in the filesystem, where possible. | | +| T-24 | Critical | Malicious actor workload overrides snapshots of other actors with malicious snapshots. | Worker Pods and actors must not have direct access to snapshots. | If actor identity is used for snapshot access, require the credential issued for snapshots to include additional claims identifying it as atelet, or require it to be used over a channel secured with atelet's mTLS certificate. | | +| T-25 | Critical | Corrupt snapshot is restored | Cryptographically verify each snapshot before restoration | Snapshots must be checked against a trusted digest or be signed and checked against a trusted key before restore is allowed. The digest or key must be delivered via a trusted channel. | | +| T-26 | Critical | Stale policies propagated out-of-band with actor scheduling result in incorrect access boundaries (including network policies, IAM permissions, etc.) | All network and authorization policies must be fully synced before an actor starts running. | Actor lifecycle APIs could support programming authz and network policy as part of create/resume. | | +| T-27 | Critical | Worker reuse enables malicious actor to escalate to other actors by persisting a threat across reuse, reading leftover state from a previous actor, or taking advantage of stale policy configuration. | All actor-specific worker state, including process state, filesystem, env vars, mounted config, network policy, and security policy, must be completely reset between actors that subsequently run on the same worker. | Ensure local policy updates are tightly synchronized with actor lifecycle to avoid race conditions. Test that the suspend/resume lifecycle for every supported sandbox technology properly cleans up state by exercising it and enumerating system state. Use "honeypots" on each side of the boundary to detect state leaks. Test policy behavior on both sides of suspend/resume to ensure policies are appropriately updated in-sync with actor lifecycle. | | +| T-28 | Critical | Malicious actor tricks Substrate identity broker into returning identity credentials for a different actor. | Add defense-in-depth to ensure that credentials cannot be mis-issued, and that mis-issued credentials are unusable in practice. | Ensure that credential issuance checks actor-to-worker scheduling assignment. Ensure actor identities include claims tying them to the worker and node the actor is scheduled to. Ensure claims are validated against scheduling assignments during authz policy evaluation; when actors are rescheduled old credentials should be immediately invalidated. | | +| T-29 | High | Agent leaks credentials exposed in sandbox, because LLMs are unreliable. Due to prompt injection or just agent silliness. | Credentials are not exposed in sandboxes by default. | Opt-in to credentials, none by default. Credential injecting proxy (injects tokens or terminates TLS and holds x509 private key on behalf of sandbox). Delegation to "drivers" rather than relying on sandboxed clients (like CSI). Encrypting credentials exposed in sandboxes, requiring them to be unwrapped by a network proxy to be usable. Consider supporting pluggable per-node or per-worker security sidecars that can interpose network, implement additional policy, add monitoring, etc. | May provide an option to expose in sandbox if there are other mitigating factors. For example, the process is NOT an AI agent, or the process limits agent access to env/filesystem. | +| T-30 | High | Malicious actor moves laterally to other nodes via suspend/resume, especially if self-suspend is allowed. | Suspend/resume is guaranteed to provide some acceptable locality with respect to lateral movement. TBD. | Maybe some kind of actor pinning to specific groups of nodes, to contain the movement? Or statistical bias to resume on the same node? | Particularly useful in combination with a sandbox breakout vuln, or network or filesystem access vulns. | +| T-31 | High | DoS/resource exhaustion via malicious container image | Enforce limits when pulling and extracting container images | Ensure limits on the uncompressed layers are enforced when untarring container images. Ensure a time limit is enforced when unpacking container images. | For example, image can contain a zip bomb. | +| T-32 | Medium | Compromised process inside actor uses actor identity to write its own snapshot, enabling local privilege escalation (still in actor sandbox). | Issue separate credentials for accessing snapshots. | Even if actor identity is used to make snapshot access permissions granular, require an extra claim, audience, or dual-identity authorization to prove that it's atelet accessing the snapshot on behalf of the actor, rather than the actor itself. | Medium since it's still a local exploit within the application layer. Example: nonroot process with access to the actor credential rewrites a snapshot so that it can `su` to root after the next restore. | +| T-33 | Medium | Attacker creates a large number of malicious actors to rapidly spread across the cluster or exhaust resources (via direct create or fork-bomb equivalent). | Limit resource consumption and number of child actors for each actor. | Direct ability to create actors should be considered a privileged permission. Use rate quotas and throttling to limit the speed of execution of this attack. Account for number of child actors and resource consumption by children so quotas can be enforced. Implement a namespace-like concept that quotas can be attached to. | | +| T-34 | Medium | Attacker discovers Kubernetes-internal network topology from inside sandboxed actor | Don't allow actors to explore the cluster network | Don't expose cluster internal DNS to actors. Don't mount worker Pod resolv.conf in actors, instead provision a new file for the actor. | | +| T-35 | Low | Attacker discovers enough information about actor names or namespace names to be consequential, from inside a sandboxed actor. | Only expose actor names for peers the actor needs to interact with. | Don't expose full Substrate-internal DNS to actors by default (since some "actors" may need access, opt-in may be acceptable). | Likely want to expose separate DNS for internet, when internet egress is required. The actor names are opaque UUIDs so there's probably not much you can get from them, other than potential targets. What's more consequential is what you can actually send traffic to, since actors get the IP of the worker Pod and those are IPv4 allocated from the cluster's IP space, it's possible to enumerate them with a network scan. Namespace names in DNS might be consequential, ideally they're also just IDs in DNS, or even unnecessary. | ## Attacks from Nodes -| GitHub Issue | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | +| Threat ID | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | | :---- | :---- | :---- | :---- | :---- | :---- | -| | High | A compromised node accesses all snapshots for the cluster. | Node access to snapshots must be scoped to actors actively scheduled to the node. | Issue special per-actor JWT, only for atelet, not in sandbox, that has claims available in conditional IAM. Use CredentialAccessBoundary (GCP only) Perform data wipeout on previous actor “tenants” of the node on actor suspension. Snapshots should be write-once per version, to prevent malicious actors from overwriting “golden” snapshots and compromising other actors that use that snapshot. | Detection rules to determine whether a given actor is running on the node and correlate it to a node requesting a snapshot for the actor could be valuable defense in depth. | -| | High | Compromised node accesses filesystem storage for actors on other nodes. | Node access to network filesystems must be scoped to actors actively scheduled to the node. | TBD \- depends on network filesystem implementation and what it supports. | | -| | High | Compromised node can escalate to other nodes or control plane via Substrate or Kubernetes. | Node access to Substrate or Kubernetes APIs must be scoped to operations relevant to actors actively scheduled to the node, and to the node itself. | Disable network communications for actors by default Tightly scoped Ingress/Egress network policies for every pod Default deny Actor to Actor communication Any allowed Actor to Actor communication should be over TLS and have authentication/authorization Limit total number of child actors under a given parent (and limit depth of actor create calls) Limit network access between data plane components, and ensure control plane components have minimal ingress from actors/workloads. | | +| T-36 | High | A compromised node accesses all snapshots for the cluster. | Node access to snapshots must be scoped to actors actively scheduled to the node. | Issue special per-actor JWT, only for atelet, not in sandbox, that has claims available in conditional IAM. Use CredentialAccessBoundary (GCP only) Perform data wipeout on previous actor “tenants” of the node on actor suspension. Snapshots should be write-once per version, to prevent malicious actors from overwriting “golden” snapshots and compromising other actors that use that snapshot. | Detection rules to determine whether a given actor is running on the node and correlate it to a node requesting a snapshot for the actor could be valuable defense in depth. | +| T-37 | High | Compromised node accesses filesystem storage for actors on other nodes. | Node access to network filesystems must be scoped to actors actively scheduled to the node. | TBD \- depends on network filesystem implementation and what it supports. | | +| T-38 | High | Compromised node can escalate to other nodes or control plane via Substrate or Kubernetes. | Node access to Substrate or Kubernetes APIs must be scoped to operations relevant to actors actively scheduled to the node, and to the node itself. | Disable network communications for actors by default Tightly scoped Ingress/Egress network policies for every pod Default deny Actor to Actor communication Any allowed Actor to Actor communication should be over TLS and have authentication/authorization Limit total number of child actors under a given parent (and limit depth of actor create calls) Limit network access between data plane components, and ensure control plane components have minimal ingress from actors/workloads. | | ## Insider Attacks -| GitHub Issue | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | +| Threat ID | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | | :---- | :---- | :---- | :---- | :---- | :---- | -| | Medium | Insider access to snapshots | Substrate's storage models for snapshots must enable users to grant granular access control when needed for system administration, rather than only supporting broad access across all actors. | Customer managed encryption keys for snapshots, similar to K8s kmsplugin. Storage layout that maps well to per-actor identity. | | -| | Medium | Insider access to disks, e.g. for substrate backend DB | Support envelope encryption of sensitive data, or avoid storing sensitive data. | Use FDE by default (most cloud providers already do this, nonspecific to Substrate). If secrets are ever stored in Substrate's DB, support envelope encryption using HSMs, similar to kms providers in K8s. | | +| T-39 | Medium | Insider access to snapshots | Substrate's storage models for snapshots must enable users to grant granular access control when needed for system administration, rather than only supporting broad access across all actors. | Customer managed encryption keys for snapshots, similar to K8s kmsplugin. Storage layout that maps well to per-actor identity. | | +| T-40 | Medium | Insider access to disks, e.g. for substrate backend DB | Support envelope encryption of sensitive data, or avoid storing sensitive data. | Use FDE by default (most cloud providers already do this, nonspecific to Substrate). If secrets are ever stored in Substrate's DB, support envelope encryption using HSMs, similar to kms providers in K8s. | | ## Detection and Response -| GitHub Issue | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | +| Threat ID | Priority | Threat | Mitigating Invariants | Suggested Concrete Mitigations | Notes | | :---- | :---- | :---- | :---- | :---- | :---- | -| | High | Malicious actions are not recorded for forensics. | Enable audit logging for all Substrate components that serve APIs, including node-local components. | Enable audit logging for snapshot/GCS buckets Set up audit logging for ateapi requests | | -| | Medium | Malicious activity proceeds undetected. | Enable integration with threat detections systems. | Support for pluggable threat detection integrations, making telemetry from Substrate API and node components, as well as sandboxes, available for continuous analysis. Consider supporting pluggable per-node or per-worker security sidecars that can gather telemetry, etc. | | -| | Medium | Detected malicious actor cannot be contained. | Quarantine or suspension option. | Support dynamic policy updates for quick quarantine. Automatically trigger snapshot/suspend on malicious activity, but taint the snapshot so that it can't automatically be resumed in production. | | +| T-41 | High | Malicious actions are not recorded for forensics. | Enable audit logging for all Substrate components that serve APIs, including node-local components. | Enable audit logging for snapshot/GCS buckets Set up audit logging for ateapi requests | | +| T-42 | Medium | Malicious activity proceeds undetected. | Enable integration with threat detections systems. | Support for pluggable threat detection integrations, making telemetry from Substrate API and node components, as well as sandboxes, available for continuous analysis. Consider supporting pluggable per-node or per-worker security sidecars that can gather telemetry, etc. | | +| T-43 | Medium | Detected malicious actor cannot be contained. | Quarantine or suspension option. | Support dynamic policy updates for quick quarantine. Automatically trigger snapshot/suspend on malicious activity, but taint the snapshot so that it can't automatically be resumed in production. | | diff --git a/docs/threats.json b/docs/threats.json new file mode 100644 index 000000000..110c1cba1 --- /dev/null +++ b/docs/threats.json @@ -0,0 +1,346 @@ +[ + { + "threat_id": "T-01", + "priority": "Critical", + "threat": "External attacker can access actors over the internet", + "mitigating_invariants": "Access to actors over the external internet is blocked by default.", + "suggested_concrete_mitigations": "Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI.", + "notes": "This should at least be a default deny policy for ingress in Substrate. The others below depend on Kubernetes configuration that may be out of scope for Substrate to configure directly." + }, + { + "threat_id": "T-02", + "priority": "Critical", + "threat": "External attacker can access nodes over the internet", + "mitigating_invariants": "Access to nodes over the external internet is blocked by default.", + "suggested_concrete_mitigations": "Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI.", + "notes": "Included for completeness but this may be beyond Substrate's ability to configure directly, as it depends on the surrounding Kubernetes configuration, determined by the system administrator." + }, + { + "threat_id": "T-03", + "priority": "Critical", + "threat": "External attacker can access Substrate API or backend database over the internet", + "mitigating_invariants": "Access to Substrate API over the external internet is blocked by default.", + "suggested_concrete_mitigations": "Recommend use of infrastructure firewall to limit external ingress/egress in documentation (cloud specific). Use Kubernetes NetworkPolicy to limit external ingress/egress by default. Use additional network policy features depending on CNI.", + "notes": "Included for completeness but this may be beyond Substrate's ability to configure directly, as it depends on the surrounding Kubernetes configuration, determined by the system administrator." + }, + { + "threat_id": "T-04", + "priority": "Critical", + "threat": "Access to the internal network allows arbitrary actions to be performed on ate-apiserver, atelet, substrate backend database, etc.", + "mitigating_invariants": "All system components must have basic mutual authentication and authorization, and communicate over TLS. All clients (including end users and actors) must be authenticated and authorized. Unauthenticated traffic must be rejected. Where possible, use firewalls to block access from ranges that have no business communicating with Substrate components.", + "suggested_concrete_mitigations": "mTLS or other secure channel (e.g. UDS) between networked system components (ate-apiserver, atelet, ateom, etc); each atelet has a unique identity cryptographically tied to the node identity. The ingress gateway should check client permissions before resuming actors or forwarding traffic to actors. The only component authorized to connect directly to the backend database should be ate-apiserver. Use Kubernetes NetworkPolicy to block access to the Substrate API and other core components by default.", + "notes": "" + }, + { + "threat_id": "T-05", + "priority": "High", + "threat": "Privilege escalation via access to sensitive labels.", + "mitigating_invariants": "If Substrate offers its own resource labeling mechanism, it must also offer a way to authorize label updates on a per-label basis.", + "suggested_concrete_mitigations": "Substrate authorization system requires explicit authorization to update metadata, separate from updating resource body. Substrate authorization system supports per-label authorization rules.", + "notes": "Plenty of attacks in K8s were possible because labels had semantic meaning, but the permission model could implicitly granted access to modify labels, even if it was inappropriate. For example, /status subresource allows label updates. Substrate shouldn't repeat this mistake." + }, + { + "threat_id": "T-06", + "priority": "High", + "threat": "Attacker gains control of the Substrate API server or an ingress/egress gateway.", + "mitigating_invariants": "Isolate the control plane from the data plane, and isolate data plane ingress from sandboxes.", + "suggested_concrete_mitigations": "Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. This also supports control-plane / data-plane isolation from a reliability and performance perspective. Consider running any gateway that enables direct interaction with sandboxes on a separate VM from the sandboxes. Consider using a zero-trust architecture where traffic is encrypted and authenticated end-to-end and routed via a mesh.", + "notes": "While the sandbox is trusted as an isolation layer, the residual risk is typically not worth the marginal cost savings of colocating, and isolation is recommend to avoid noisy neighbor problems or other reliability issues that stem from mixing the control-plane and data-plane, especially for large scale deployments of Substrate." + }, + { + "threat_id": "T-07", + "priority": "High", + "threat": "Attacker who can create ActorTemplates specifies malicious runtime.", + "mitigating_invariants": "Ensure available runtime can only be configured by administrators.", + "suggested_concrete_mitigations": "Consider a mechanism like RuntimeClass to decouple configuration of available runtimes from consumption of available runtimes.", + "notes": "" + }, + { + "threat_id": "T-08", + "priority": "High", + "threat": "Attacker who can create ActorTemplates can read or write any storage buckets atelet has access to.", + "mitigating_invariants": "Ensure that bucket access is least-privilege.", + "suggested_concrete_mitigations": "Use credentials derived from actor identity to read snapshots. Configure permissions to prevent atelet or nodes from having access to sensitive buckets. Don't support arbitrary URLs in APIs, instead design a standard approach for accessing the correct resource given the ActorTemplate name and compute the mapping internally, subject to scheduling-aware authorization checks.", + "notes": "For example: Attacker creates an ActorTemplate with the runsc URL or golden snapshot URL pointing to an arbitrary bucket in the same project/resource scope as the cluster. If atelet has project-wide access to buckets, this could cause the state to be pulled into the worker pod or malicious actor. Similarly, an attacker could set the snapshots URL to point to an internal infrastructure bucket, causing data to be written to that bucket." + }, + { + "threat_id": "T-09", + "priority": "Medium", + "threat": "DoS attack against API, gateway, or available cluster resources.", + "mitigating_invariants": "Minimize exposure and ensure APIs and proxies implement appropriately scoped quotas and rate limiting.", + "suggested_concrete_mitigations": "Don't co-locate ate-apiserver or other control-plane components on the same machines as the untrusted sandboxes. Don't expose ate-apiserver directly to the internet. Consider an identity-aware WAF (likely up to providers) if external access is required. Use network policy to block direct interaction between untrusted sandboxes and the ate-apiserver. Implement quotas and rate limiting in the API and in proxies. This includes quotas for the number of actors a user can allocate. Use a zero-trust architecture that prevents identity spoofing or intentional misrouting to get around limits.", + "notes": "The gateway calls ate-apiserver to resolve Actor assignments and uses a short-lived cache. A flood of Actor traffic could still create high API load; cache invalidation during Actor rescheduling must avoid stale routing." + }, + { + "threat_id": "T-10", + "priority": "Medium", + "threat": "Internal network traffic is intercepted or spoofed", + "mitigating_invariants": "Encrypt all traffic by default", + "suggested_concrete_mitigations": "Use mTLS between all system components, including the gateway-to-worker `atunnel` connection.", + "notes": "It may be desirable to rely on cloud providers to transparently encrypt traffic between VMs on their internal network. Worth discussing what makes the most sense." + }, + { + "threat_id": "T-11", + "priority": "High", + "threat": "Improper handling of Secrets", + "mitigating_invariants": "Ensure there is an official, secure, recommended way to pass secret data, like API access tokens, to actors.", + "suggested_concrete_mitigations": "Support env and filesystem plumbing for Kubernetes Secrets, to provide an official path that avoids secret material being plumbed via nonspecific fields that are difficult to audit. Ensure secrets are encrypted in transit and ideally stored in memory. If exposed via the filesystem, do so via in-memory tmpfs.", + "notes": "If we don't support this, users will inevitably put secrets in plaintext." + }, + { + "threat_id": "T-12", + "priority": "Medium", + "threat": "Complexity configuring permissions for frameworks on top of Substrate may lead to unintentional privilege escalation.", + "mitigating_invariants": "It must be clear to users what the downstream effects of auth config in substrate are.", + "suggested_concrete_mitigations": "If it's not intuitive, it must be documented in a user guide.", + "notes": "AI framework has to set up permissions to access ATE, and to access K8s, and potentially for actors (based on ATE identity and K8s identity) to access the framework. We need to make this easy. Think about past K8s issues like escalate/bind risk. Substrate resource model is spread across ate-apiserver and K8s, increasing complexity and chance for error." + }, + { + "threat_id": "T-13", + "priority": "Medium", + "threat": "Flat namespace of actors encourages broad permission grants or complex graph-oriented policy.", + "mitigating_invariants": "Support a grouping mechanism that can be used in policy controls.", + "suggested_concrete_mitigations": "Add namespaces to Substrate, similar to Kubernetes.", + "notes": "" + }, + { + "threat_id": "T-14", + "priority": "Medium", + "threat": "DNS misconfiguration", + "mitigating_invariants": "Access to DNS configuration should be limited to authoritative controllers. Routing should use stable configurations and query the API for the current worker assignment before forwarding requests.", + "suggested_concrete_mitigations": "Don't co-locate controllers with access to sensitive system state on the same nodes as actors. Limit permissions to update DNS configuration. Use the Substrate API and bounded caching to keep assignments current. Authenticate gateway-to-worker traffic with mTLS.", + "notes": "A flood of requests could create high read load on ate-apiserver. Cache invalidation during Actor rescheduling is important to avoid stale routing. The gateway-to-`atunnel` mTLS connection prevents the removed direct worker-port path from bypassing worker assignment checks." + }, + { + "threat_id": "T-15", + "priority": "Critical", + "threat": "Malicious actor gains access to the underlying node or other actors via container breakout (Linux local privilege escalation)", + "mitigating_invariants": "Actors are always sandboxed using a hardened sandbox solution like gvisor or microvm. Traditional containers are not a secure sandbox.", + "suggested_concrete_mitigations": "Use Gvisor for strong container isolation. Run sentry in a user namespace and pivot\\_root to limit broader file system access if the gvisor boundary is broken. Use Seccomp for sentry/gofer as well. Limit capabilities granted to directfs/sentry Any sidecar containers running alongside actors or warmpool pods should be as low privileged as possible. Sandbox lifecycle must be controlled from outside the sandbox", + "notes": "" + }, + { + "threat_id": "T-16", + "priority": "Critical", + "threat": "Malicious actor gains access to the underlying node via node-local endpoints exposed on the network", + "mitigating_invariants": "Network policies must prevent the actor from accessing system services on the node (e.g. instance metadata, host network namespace, host network interface).", + "suggested_concrete_mitigations": "Enable network namespacing, and do not provide host network access to any workload. Any host services should have targeted ingress policies so that only the correct clients even have network access to those services", + "notes": "" + }, + { + "threat_id": "T-17", + "priority": "Critical", + "threat": "Malicious actor gains access to other actors via network", + "mitigating_invariants": "Network policies must deny ingress and egress by default, and selectively allow access to specific actors only when necessary. Network policies must be synchronized with actor lifecycle.", + "suggested_concrete_mitigations": "Implement default-deny network policy that blocks any cross-communication between actors (this could be implemented by ateom instead of Kubernetes, to speed up policy updates when actor groups do need to communicate).", + "notes": "" + }, + { + "threat_id": "T-18", + "priority": "Critical", + "threat": "Malicious actor gains access to the underlying node or other actors via filesystem", + "mitigating_invariants": "Filesystem access is limited to the actor's local fs and remote filesystems the actor is directly, explicitly authorized to access.", + "suggested_concrete_mitigations": "Ensure that actor access to the filesystem is protected by, for example, mapping each actor to a unique Linux uid and using filesystem permissions, or by using user namespaces to isolate actor filesystem access on the same host. Ensure that actor management APIs and filesystem setup protects against traversal via symlinks, mount trickery, etc.", + "notes": "Possible vectors: Arbitrary file read vuln in fs implementation. Symlink traversal vulns. Vuln in OCI image unpacking. Path traversal attacks in mount configuration. Shared directories only gated via filesystem permissions \\+ workers running as root or privileged." + }, + { + "threat_id": "T-19", + "priority": "Critical", + "threat": "Malicious actor gains access to the underlying Worker Pod or node via local Substrate services (atelet, ateom, etc).", + "mitigating_invariants": "Actor access to system services must be denied or explicitly scoped to the actor. Local services must run with the fewest privileges possible to limit the ability for an actor to escalate privilege.", + "suggested_concrete_mitigations": "Limit actor access to system services. Ensure system services are aware of actor identity, and authenticate and authorize actors when access is required.", + "notes": "" + }, + { + "threat_id": "T-20", + "priority": "Critical", + "threat": "Malicious actor gains access to the underlying node or other actors via Substrate APIs (remote ate-apiserver)", + "mitigating_invariants": "Either actors are completely unable to access Substrate APIs, or actor access is authenticated and authorized such that the actor cannot escalate beyond its intended scope. Especially prevent self-modification, which was a common escalation path in K8s.", + "suggested_concrete_mitigations": "Do not allow actors or workers to self-modify, e.g. by: reading or writing their own snapshots self-labeling or other self-manipulations of KRM or ATE resources Logic to modify resource definitions including actors, workers, etc lives in the control plane, rather than the data plane, as much as possible.", + "notes": "" + }, + { + "threat_id": "T-21", + "priority": "Critical", + "threat": "Attacker escalates to host via excessive Worker Pod permissions", + "mitigating_invariants": "Ensure Worker pods use the minimum necessary permissions to set manage sandbox lifecycle", + "suggested_concrete_mitigations": "Deprivilege the worker Pods. Run them as non-root or in a user namespace, without privileged, and with only required caps/devices.", + "notes": "Possible example, accidentally putting things in the same cgroup: [https://github.com/agent-substrate/substrate/issues/288](https://github.com/agent-substrate/substrate/issues/288)" + }, + { + "threat_id": "T-22", + "priority": "Critical", + "threat": "Malicious actor gains access to the underlying node or other actors via Kubernetes APIs", + "mitigating_invariants": "Either actors are completely unable to access the Kubernetes APIs, or actor access is authenticated and authorized such that the actor cannot escalate beyond its intended scope. **Strong preference on blocking actor access to Kubernetes.**", + "suggested_concrete_mitigations": "Block network access to the kubernetes API Ensure the default kubernetes service account token in each pod has 0 privileges", + "notes": "" + }, + { + "threat_id": "T-23", + "priority": "Critical", + "threat": "Malicious actor workload gains access to snapshots of other actors and steals data from them.", + "mitigating_invariants": "Worker Pods and actors must not have direct access to snapshots.", + "suggested_concrete_mitigations": "If actor identity is used for snapshot access, require the credential issued for snapshots to include additional claims identifying it as atelet, or require it to be used over a channel secured with atelet's mTLS certificate. Envelope encrypt snapshots and tie decryption to actor identity. Avoid snapshotting sensitive credentials in the filesystem, where possible.", + "notes": "" + }, + { + "threat_id": "T-24", + "priority": "Critical", + "threat": "Malicious actor workload overrides snapshots of other actors with malicious snapshots.", + "mitigating_invariants": "Worker Pods and actors must not have direct access to snapshots.", + "suggested_concrete_mitigations": "If actor identity is used for snapshot access, require the credential issued for snapshots to include additional claims identifying it as atelet, or require it to be used over a channel secured with atelet's mTLS certificate.", + "notes": "" + }, + { + "threat_id": "T-25", + "priority": "Critical", + "threat": "Corrupt snapshot is restored", + "mitigating_invariants": "Cryptographically verify each snapshot before restoration", + "suggested_concrete_mitigations": "Snapshots must be checked against a trusted digest or be signed and checked against a trusted key before restore is allowed. The digest or key must be delivered via a trusted channel.", + "notes": "" + }, + { + "threat_id": "T-26", + "priority": "Critical", + "threat": "Stale policies propagated out-of-band with actor scheduling result in incorrect access boundaries (including network policies, IAM permissions, etc.)", + "mitigating_invariants": "All network and authorization policies must be fully synced before an actor starts running.", + "suggested_concrete_mitigations": "Actor lifecycle APIs could support programming authz and network policy as part of create/resume.", + "notes": "" + }, + { + "threat_id": "T-27", + "priority": "Critical", + "threat": "Worker reuse enables malicious actor to escalate to other actors by persisting a threat across reuse, reading leftover state from a previous actor, or taking advantage of stale policy configuration.", + "mitigating_invariants": "All actor-specific worker state, including process state, filesystem, env vars, mounted config, network policy, and security policy, must be completely reset between actors that subsequently run on the same worker.", + "suggested_concrete_mitigations": "Ensure local policy updates are tightly synchronized with actor lifecycle to avoid race conditions. Test that the suspend/resume lifecycle for every supported sandbox technology properly cleans up state by exercising it and enumerating system state. Use \"honeypots\" on each side of the boundary to detect state leaks. Test policy behavior on both sides of suspend/resume to ensure policies are appropriately updated in-sync with actor lifecycle.", + "notes": "" + }, + { + "threat_id": "T-28", + "priority": "Critical", + "threat": "Malicious actor tricks Substrate identity broker into returning identity credentials for a different actor.", + "mitigating_invariants": "Add defense-in-depth to ensure that credentials cannot be mis-issued, and that mis-issued credentials are unusable in practice.", + "suggested_concrete_mitigations": "Ensure that credential issuance checks actor-to-worker scheduling assignment. Ensure actor identities include claims tying them to the worker and node the actor is scheduled to. Ensure claims are validated against scheduling assignments during authz policy evaluation; when actors are rescheduled old credentials should be immediately invalidated.", + "notes": "" + }, + { + "threat_id": "T-29", + "priority": "High", + "threat": "Agent leaks credentials exposed in sandbox, because LLMs are unreliable. Due to prompt injection or just agent silliness.", + "mitigating_invariants": "Credentials are not exposed in sandboxes by default.", + "suggested_concrete_mitigations": "Opt-in to credentials, none by default. Credential injecting proxy (injects tokens or terminates TLS and holds x509 private key on behalf of sandbox). Delegation to \"drivers\" rather than relying on sandboxed clients (like CSI). Encrypting credentials exposed in sandboxes, requiring them to be unwrapped by a network proxy to be usable. Consider supporting pluggable per-node or per-worker security sidecars that can interpose network, implement additional policy, add monitoring, etc.", + "notes": "May provide an option to expose in sandbox if there are other mitigating factors. For example, the process is NOT an AI agent, or the process limits agent access to env/filesystem." + }, + { + "threat_id": "T-30", + "priority": "High", + "threat": "Malicious actor moves laterally to other nodes via suspend/resume, especially if self-suspend is allowed.", + "mitigating_invariants": "Suspend/resume is guaranteed to provide some acceptable locality with respect to lateral movement. TBD.", + "suggested_concrete_mitigations": "Maybe some kind of actor pinning to specific groups of nodes, to contain the movement? Or statistical bias to resume on the same node?", + "notes": "Particularly useful in combination with a sandbox breakout vuln, or network or filesystem access vulns." + }, + { + "threat_id": "T-31", + "priority": "High", + "threat": "DoS/resource exhaustion via malicious container image", + "mitigating_invariants": "Enforce limits when pulling and extracting container images", + "suggested_concrete_mitigations": "Ensure limits on the uncompressed layers are enforced when untarring container images. Ensure a time limit is enforced when unpacking container images.", + "notes": "For example, image can contain a zip bomb." + }, + { + "threat_id": "T-32", + "priority": "Medium", + "threat": "Compromised process inside actor uses actor identity to write its own snapshot, enabling local privilege escalation (still in actor sandbox).", + "mitigating_invariants": "Issue separate credentials for accessing snapshots.", + "suggested_concrete_mitigations": "Even if actor identity is used to make snapshot access permissions granular, require an extra claim, audience, or dual-identity authorization to prove that it's atelet accessing the snapshot on behalf of the actor, rather than the actor itself.", + "notes": "Medium since it's still a local exploit within the application layer. Example: nonroot process with access to the actor credential rewrites a snapshot so that it can `su` to root after the next restore." + }, + { + "threat_id": "T-33", + "priority": "Medium", + "threat": "Attacker creates a large number of malicious actors to rapidly spread across the cluster or exhaust resources (via direct create or fork-bomb equivalent).", + "mitigating_invariants": "Limit resource consumption and number of child actors for each actor.", + "suggested_concrete_mitigations": "Direct ability to create actors should be considered a privileged permission. Use rate quotas and throttling to limit the speed of execution of this attack. Account for number of child actors and resource consumption by children so quotas can be enforced. Implement a namespace-like concept that quotas can be attached to.", + "notes": "" + }, + { + "threat_id": "T-34", + "priority": "Medium", + "threat": "Attacker discovers Kubernetes-internal network topology from inside sandboxed actor", + "mitigating_invariants": "Don't allow actors to explore the cluster network", + "suggested_concrete_mitigations": "Don't expose cluster internal DNS to actors. Don't mount worker Pod resolv.conf in actors, instead provision a new file for the actor.", + "notes": "" + }, + { + "threat_id": "T-35", + "priority": "Low", + "threat": "Attacker discovers enough information about actor names or namespace names to be consequential, from inside a sandboxed actor.", + "mitigating_invariants": "Only expose actor names for peers the actor needs to interact with.", + "suggested_concrete_mitigations": "Don't expose full Substrate-internal DNS to actors by default (since some \"actors\" may need access, opt-in may be acceptable).", + "notes": "Likely want to expose separate DNS for internet, when internet egress is required. The actor names are opaque UUIDs so there's probably not much you can get from them, other than potential targets. What's more consequential is what you can actually send traffic to, since actors get the IP of the worker Pod and those are IPv4 allocated from the cluster's IP space, it's possible to enumerate them with a network scan. Namespace names in DNS might be consequential, ideally they're also just IDs in DNS, or even unnecessary." + }, + { + "threat_id": "T-36", + "priority": "High", + "threat": "A compromised node accesses all snapshots for the cluster.", + "mitigating_invariants": "Node access to snapshots must be scoped to actors actively scheduled to the node.", + "suggested_concrete_mitigations": "Issue special per-actor JWT, only for atelet, not in sandbox, that has claims available in conditional IAM. Use CredentialAccessBoundary (GCP only) Perform data wipeout on previous actor \u201ctenants\u201d of the node on actor suspension. Snapshots should be write-once per version, to prevent malicious actors from overwriting \u201cgolden\u201d snapshots and compromising other actors that use that snapshot.", + "notes": "Detection rules to determine whether a given actor is running on the node and correlate it to a node requesting a snapshot for the actor could be valuable defense in depth." + }, + { + "threat_id": "T-37", + "priority": "High", + "threat": "Compromised node accesses filesystem storage for actors on other nodes.", + "mitigating_invariants": "Node access to network filesystems must be scoped to actors actively scheduled to the node.", + "suggested_concrete_mitigations": "TBD \\- depends on network filesystem implementation and what it supports.", + "notes": "" + }, + { + "threat_id": "T-38", + "priority": "High", + "threat": "Compromised node can escalate to other nodes or control plane via Substrate or Kubernetes.", + "mitigating_invariants": "Node access to Substrate or Kubernetes APIs must be scoped to operations relevant to actors actively scheduled to the node, and to the node itself.", + "suggested_concrete_mitigations": "Disable network communications for actors by default Tightly scoped Ingress/Egress network policies for every pod Default deny Actor to Actor communication Any allowed Actor to Actor communication should be over TLS and have authentication/authorization Limit total number of child actors under a given parent (and limit depth of actor create calls) Limit network access between data plane components, and ensure control plane components have minimal ingress from actors/workloads.", + "notes": "" + }, + { + "threat_id": "T-39", + "priority": "Medium", + "threat": "Insider access to snapshots", + "mitigating_invariants": "Substrate's storage models for snapshots must enable users to grant granular access control when needed for system administration, rather than only supporting broad access across all actors.", + "suggested_concrete_mitigations": "Customer managed encryption keys for snapshots, similar to K8s kmsplugin. Storage layout that maps well to per-actor identity.", + "notes": "" + }, + { + "threat_id": "T-40", + "priority": "Medium", + "threat": "Insider access to disks, e.g. for substrate backend DB", + "mitigating_invariants": "Support envelope encryption of sensitive data, or avoid storing sensitive data.", + "suggested_concrete_mitigations": "Use FDE by default (most cloud providers already do this, nonspecific to Substrate). If secrets are ever stored in Substrate's DB, support envelope encryption using HSMs, similar to kms providers in K8s.", + "notes": "" + }, + { + "threat_id": "T-41", + "priority": "High", + "threat": "Malicious actions are not recorded for forensics.", + "mitigating_invariants": "Enable audit logging for all Substrate components that serve APIs, including node-local components.", + "suggested_concrete_mitigations": "Enable audit logging for snapshot/GCS buckets Set up audit logging for ateapi requests", + "notes": "" + }, + { + "threat_id": "T-42", + "priority": "Medium", + "threat": "Malicious activity proceeds undetected.", + "mitigating_invariants": "Enable integration with threat detections systems.", + "suggested_concrete_mitigations": "Support for pluggable threat detection integrations, making telemetry from Substrate API and node components, as well as sandboxes, available for continuous analysis. Consider supporting pluggable per-node or per-worker security sidecars that can gather telemetry, etc.", + "notes": "" + }, + { + "threat_id": "T-43", + "priority": "Medium", + "threat": "Detected malicious actor cannot be contained.", + "mitigating_invariants": "Quarantine or suspension option.", + "suggested_concrete_mitigations": "Support dynamic policy updates for quick quarantine. Automatically trigger snapshot/suspend on malicious activity, but taint the snapshot so that it can't automatically be resumed in production.", + "notes": "" + } +] diff --git a/hack/export-threats.py b/hack/export-threats.py new file mode 100755 index 000000000..f7e055b23 --- /dev/null +++ b/hack/export-threats.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import json +import argparse + +def extract(md_path): + with open(md_path, 'r') as f: + lines = f.readlines() + + threats = [] + headers = {} + in_table = False + for line in lines: + line = line.strip() + if not line.startswith('|'): + in_table = False + continue + + cells = [c.strip() for c in line.split('|')[1:-1]] + if not in_table: + lower_cells = [c.lower() for c in cells] + if 'threat' in lower_cells: + in_table = True + headers = {} + for i, col in enumerate(lower_cells): + if col == 'threat': headers['threat'] = i + elif col == 'threat id': headers['threat_id'] = i + elif col == 'priority': headers['priority'] = i + elif col == 'mitigating invariants': headers['mitigating_invariants'] = i + elif col == 'suggested concrete mitigations': headers['suggested_concrete_mitigations'] = i + elif col == 'notes': headers['notes'] = i + continue + + if all(c.replace('-', '').replace(':', '') == '' for c in cells): + continue + + if in_table and 'threat' in headers and len(cells) > headers['threat']: + threat = cells[headers['threat']] + if threat: + t_obj = {} + if 'threat_id' in headers and len(cells) > headers['threat_id']: + t_obj['threat_id'] = cells[headers['threat_id']] + if 'priority' in headers and len(cells) > headers['priority']: + t_obj['priority'] = cells[headers['priority']] + + t_obj['threat'] = threat + + if 'mitigating_invariants' in headers and len(cells) > headers['mitigating_invariants']: + t_obj['mitigating_invariants'] = cells[headers['mitigating_invariants']] + if 'suggested_concrete_mitigations' in headers and len(cells) > headers['suggested_concrete_mitigations']: + t_obj['suggested_concrete_mitigations'] = cells[headers['suggested_concrete_mitigations']] + if 'notes' in headers and len(cells) > headers['notes']: + t_obj['notes'] = cells[headers['notes']] + + threats.append(t_obj) + return threats + +def main(): + parser = argparse.ArgumentParser(description="Export threats from threat-model.md to JSON") + parser.add_argument("--md", type=str, default="docs/threat-model.md", help="Path to threat-model.md") + parser.add_argument("--out", type=str, default="docs/threats.json", help="Output JSON path") + args = parser.parse_args() + + if not os.path.exists(args.md): + print(f"Error: {args.md} not found.", file=sys.stderr) + sys.exit(1) + + threats = extract(args.md) + with open(args.out, 'w') as f: + json.dump(threats, f, indent=2) + f.write('\n') + print(f"Extracted {len(threats)} threats from {args.md} to {args.out}") + +if __name__ == '__main__': + main() diff --git a/hack/verify/verify-threats.sh b/hack/verify/verify-threats.sh new file mode 100755 index 000000000..f23c2de00 --- /dev/null +++ b/hack/verify/verify-threats.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +hack/export-threats.py --md docs/threat-model.md --out "${TMP_DIR}/threats.json" > /dev/null + +if ! diff -u docs/threats.json "${TMP_DIR}/threats.json"; then + echo "FAIL: docs/threats.json is out of date." >&2 + echo "Please run 'hack/export-threats.py' to regenerate it." >&2 + exit 1 +fi