From 707a033632c61b6a6153a7c9320b0c289ce7d7a2 Mon Sep 17 00:00:00 2001 From: Hannes Zietsman Date: Sat, 12 Sep 2026 15:39:45 +0200 Subject: [PATCH 1/4] Add optional Vast service routing and preserved-config migration --- README.md | 23 + landing-page/index.html | 23 +- scripts/health-api.py | 104 ++ scripts/vpm-custom-config-migrate.py | 71 ++ src/cryptolabs_proxy/auth.py | 14 + src/cryptolabs_proxy/cli.py | 222 +++- src/cryptolabs_proxy/config.py | 5 + src/cryptolabs_proxy/custom_config.py | 120 ++ src/cryptolabs_proxy/migration.py | 622 ++++++++++ src/cryptolabs_proxy/services.py | 12 + src/cryptolabs_proxy/templates/nginx.conf.j2 | 8 +- .../templates/vast-price-manager.conf.j2 | 38 + tests/test_vast_price_manager.py | 1008 +++++++++++++++++ 13 files changed, 2238 insertions(+), 32 deletions(-) create mode 100644 scripts/vpm-custom-config-migrate.py create mode 100644 src/cryptolabs_proxy/custom_config.py create mode 100644 src/cryptolabs_proxy/migration.py create mode 100644 src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 create mode 100644 tests/test_vast_price_manager.py diff --git a/README.md b/README.md index 73b5af4..d420a58 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,29 @@ Config files are stored in `/etc/cryptolabs-proxy/`: └── ssl/ # SSL certificates ``` +The image ships a bootstrap Nginx configuration. Service lifecycle commands +regenerate the configuration above and require it to be mounted at +`/etc/nginx/nginx.conf` in the running proxy container. This is how optional +Vast Price Manager routing is enabled and removed; the CLI refuses to claim a +route change when the container is still using the bundled configuration. +The bootstrap configuration intentionally contains no Vast Price Manager route. +Before an installer enables or disables VPM, it must first create the registry +configuration, render `/etc/cryptolabs-proxy/nginx.conf`, bind-mount that file +as `/etc/nginx/nginx.conf`, and verify the proxy is using it. The subsequent +`cryptolabs-proxy register` or `unregister` command then validates and reloads +that mounted configuration atomically with the registry update. + +### Existing custom proxy configuration + +Do not use the generic CLI alone to migrate a proxy that already runs a custom +unmounted `nginx.conf`: it regenerates the full configuration. Use the reviewed +`scripts/vpm-custom-config-migrate.py` helper first. Its read-only `plan` mode +derives a migration ID from the source container and config hash. Its explicit +`apply` stores root-only backups, mounts a byte-preserved custom baseline with +only the canonical managed VPM block added, and can perform a proxy-only +`rollback`. Custom-config mode then permits only normal VPM +`register`/`unregister`; it rejects changes to every other service. + User authentication data is stored in `/data/auth/`: ``` diff --git a/landing-page/index.html b/landing-page/index.html index 243ab70..e72a428 100644 --- a/landing-page/index.html +++ b/landing-page/index.html @@ -839,6 +839,16 @@

⚙️ System Updates

exporterName: 'vastai', keyPlaceholder: 'Your Vast.ai API Key' }, + 'vast-price-manager': { + displayName: 'Vast Price Manager', + icon: '💰', + description: 'Optional pricing-management service for Vast. It keeps its own login, CSRF protection, and password reauthentication.', + path: '/vast-pricing/', + productUrl: 'https://github.com/cryptolabsza/vast-price-manager', + isCryptoLabs: true, + installNote: 'Install and manage this service from the Vast setup in Server Manager. Fleet access is restricted to administrators.', + lifecycleManager: 'dc-overview' + }, 'dc-watchdog': { displayName: 'DC Watchdog', icon: '', @@ -900,6 +910,7 @@

⚙️ System Updates

for (const [key, service] of Object.entries(DEFAULT_SERVICES)) { const serviceHealth = health[key] || {}; const isRunning = serviceHealth.running === true; + const isUnconfigured = serviceHealth.state === 'unconfigured'; const isExternalUrl = service.externalUrl ? true : false; // Determine if this service belongs in "Services" or "Available Products" @@ -912,7 +923,7 @@

⚙️ System Updates

if (isRunning || (isExternalUrl && isConfigured && !service.needsApiKey)) { runningCount++; - runningHtml += renderServiceCard(key, service, isExternalUrl ? 'external' : 'running', health); + runningHtml += renderServiceCard(key, service, isExternalUrl ? 'external' : (isUnconfigured ? 'unconfigured' : 'running'), health); } else { availableHtml += renderServiceCard(key, service, isRunning ? 'running' : 'not-installed', health); } @@ -947,6 +958,10 @@

⚙️ System Updates

statusClass = 'running'; statusText = 'Running'; statusDot = 'running'; + } else if (status === 'unconfigured') { + statusClass = 'not-installed'; + statusText = 'Setup Required'; + statusDot = 'not-installed'; } else { statusClass = 'not-installed'; statusText = 'Not Installed'; @@ -956,7 +971,7 @@

⚙️ System Updates

let actions = ''; let installSection = ''; - if (status === 'running' || status === 'external') { + if (status === 'running' || status === 'unconfigured' || status === 'external') { // Active service - show Open button const openUrl = service.externalUrl || service.path; const target = service.externalUrl ? ' target="_blank"' : ''; @@ -1250,7 +1265,9 @@

${service.displayName}

const tagMatches = currentTag === expectedTag || currentTag === targetBranch; let updateBtn = ''; - if (!info.running) { + if (info.lifecycle_manager) { + updateBtn = `Managed by Server Manager`; + } else if (!info.running) { const isExporter = name === 'vastai-exporter' || name === 'runpod-exporter'; const prometheusUp = serviceHealth['prometheus']?.running === true; const grafanaUp = serviceHealth['grafana']?.running === true; diff --git a/scripts/health-api.py b/scripts/health-api.py index 222b3b4..aaf0490 100644 --- a/scripts/health-api.py +++ b/scripts/health-api.py @@ -11,13 +11,16 @@ import threading import os import re +from urllib.error import HTTPError, URLError from urllib.parse import urlparse, parse_qs +from urllib.request import Request, urlopen from pathlib import Path PORT = 8080 BUILD_INFO_FILE = '/app/BUILD_INFO' SETTINGS_FILE = '/data/auth/update-settings.json' SHARED_CONFIG_FILE = '/data/auth/shared-config.json' +VPM_READY_URL = 'http://vast-price-manager:8088/readyz' # Internal Docker network subnet - only allow requests from this range INTERNAL_NETWORK = '172.30.' @@ -42,6 +45,15 @@ 'grafana': {'container': 'grafana', 'port': 3000, 'image': 'grafana/grafana'}, 'prometheus': {'container': 'prometheus', 'port': 9090, 'image': 'prom/prometheus'}, 'vastai-exporter': {'container': 'vastai-exporter', 'port': 8622, 'image': 'ghcr.io/cryptolabsza/vastai-exporter'}, + # The optional VPM service is installed and lifecycle-managed by + # dc-overview. The proxy only reports its status and proxies its UI. + 'vast-price-manager': { + 'container': 'vast-price-manager', + 'port': 8088, + 'image': '', + 'lifecycle_manager': 'dc-overview', + 'update_supported': False, + }, 'runpod-exporter': {'container': 'runpod-exporter', 'port': 8623, 'image': 'ghcr.io/cryptolabsza/runpod-exporter'}, } @@ -231,6 +243,67 @@ def check_container_running(container_name): return False +def get_vast_price_manager_allowed_host(): + """Read only VPM's non-secret public-host setting from Docker inspect.""" + try: + result = subprocess.run( + ['docker', 'inspect', '--format', '{{range .Config.Env}}{{println .}}{{end}}', 'vast-price-manager'], + capture_output=True, text=True, timeout=5, + ) + if result.returncode != 0: + return None + for env_line in result.stdout.splitlines(): + if env_line.startswith('VPM_ALLOWED_HOSTS='): + host = env_line.split('=', 1)[1].split(',', 1)[0].strip() + return host or None + except (OSError, subprocess.SubprocessError): + pass + return None + + +def get_vast_price_manager_docker_health(): + """Return Docker's VPM healthcheck result without treating running as healthy.""" + try: + result = subprocess.run( + ['docker', 'inspect', 'vast-price-manager'], capture_output=True, text=True, timeout=5, + ) + if result.returncode != 0: + return None + data = json.loads(result.stdout) + return data[0].get('State', {}).get('Health', {}).get('Status') if data else None + except (OSError, subprocess.SubprocessError, json.JSONDecodeError, IndexError): + return None + + +def get_vast_price_manager_readiness(): + """Report VPM account setup separately from Docker liveness. + + VPM's Docker healthcheck uses ``/healthz``. Its ``/readyz`` endpoint is + surfaced for Fleet status only, so an unconfigured account never creates a + liveness restart loop. + """ + allowed_host = get_vast_price_manager_allowed_host() + if not allowed_host: + return 'unavailable' + try: + request = Request(VPM_READY_URL, headers={'Host': allowed_host}) + with urlopen(request, timeout=2) as response: + return 'ready' if response.status == 200 else 'unavailable' + except HTTPError as error: + return 'unconfigured' if error.code == 503 else 'unavailable' + except (URLError, OSError, TimeoutError): + return 'unavailable' + + +def service_action_error(service_name): + """Return an error when lifecycle is owned outside generic proxy updates.""" + config = SERVICES.get(service_name, {}) + if config.get('update_supported') is False: + display_name = 'Vast Price Manager' if service_name == 'vast-price-manager' else service_name + return f"{display_name} lifecycle is managed by {config['lifecycle_manager']}." + return None + + def get_container_version(container_name): """Get version info for a container from its image, labels, and environment variables.""" try: @@ -546,6 +619,21 @@ def get_all_service_status(include_versions=False): 'image': config.get('image', ''), 'self': config.get('self', False), } + + if config.get('lifecycle_manager'): + service_info['lifecycle_manager'] = config['lifecycle_manager'] + service_info['update_supported'] = config.get('update_supported', True) + + if name == 'vast-price-manager': + readiness = get_vast_price_manager_readiness() if running else 'not-installed' + docker_health = get_vast_price_manager_docker_health() if running else None + service_info.update({ + 'healthy': docker_health == 'healthy', + 'docker_health': docker_health, + 'readiness': readiness, + 'configured': readiness == 'ready', + 'state': 'running' if readiness == 'ready' else readiness, + }) if include_versions and running: version_info = get_container_version(container) @@ -575,6 +663,9 @@ def get_all_versions(): 'image': config.get('image', ''), 'self': config.get('self', False), } + if config.get('lifecycle_manager'): + version_info['lifecycle_manager'] = config['lifecycle_manager'] + version_info['update_supported'] = config.get('update_supported', True) if running: v = get_container_version(container) @@ -763,6 +854,8 @@ def do_POST(self): if service == 'all': # Update all services for name, config in SERVICES.items(): + if service_action_error(name): + continue if config.get('self'): # Handle self-update last continue @@ -781,6 +874,10 @@ def do_POST(self): results[service] = {'success': success, 'message': msg} elif service in SERVICES: + action_error = service_action_error(service) + if action_error: + self.send_json({'error': action_error}, 400) + return config = SERVICES[service] tag = 'dev' if target_branch == 'dev' else 'latest' success, msg = update_container(service, config, tag) @@ -804,6 +901,13 @@ def do_POST(self): if name not in SERVICES: results[name] = {'success': False, 'message': 'Unknown service'} continue + + action_error = service_action_error(name) + if action_error: + if service == 'all': + continue + self.send_json({'error': action_error}, 400) + return config = SERVICES[name] tag = 'dev' if target_branch == 'dev' else 'latest' diff --git a/scripts/vpm-custom-config-migrate.py b/scripts/vpm-custom-config-migrate.py new file mode 100644 index 0000000..c5a701e --- /dev/null +++ b/scripts/vpm-custom-config-migrate.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Run the reviewed VPM custom-proxy migration helper on its target host.""" + +import argparse +import json +from pathlib import Path +import sys + +from cryptolabs_proxy.migration import PROXY_READY_TIMEOUT, CustomConfigMigrator, MigrationError + + +def parser(): + command = argparse.ArgumentParser(description="VPM custom proxy migration (no default apply)") + command.add_argument("--config-dir", type=Path, default=Path("/etc/cryptolabs-proxy")) + command.add_argument("--backup-root", type=Path, default=Path("/etc/cryptolabs-proxy/migrations")) + command.add_argument("--transport-timeout", type=int, default=15, help="per Docker/HTTP transport timeout in seconds") + command.add_argument( + "--readiness-timeout", + type=int, + default=PROXY_READY_TIMEOUT, + help="Docker health and local auth readiness deadline in seconds", + ) + subcommands = command.add_subparsers(dest="action", required=True) + + plan = subcommands.add_parser("plan", help="read-only sanitized migration plan") + plan.add_argument("--container", default="cryptolabs-proxy") + plan.add_argument("--proxy-image", required=True) + + apply = subcommands.add_parser("apply", help="explicit proxy-only switch with rollback") + apply.add_argument("--container", default="cryptolabs-proxy") + apply.add_argument("--proxy-image", required=True) + apply.add_argument("--migration-id", required=True) + apply.add_argument("--enable-vpm", action="store_true", help="write the canonical VPM registry entry and route") + + rollback = subcommands.add_parser("rollback", help="restore only the named proxy migration") + rollback.add_argument("--container", default="cryptolabs-proxy") + rollback.add_argument("--migration-id", required=True) + return command + + +def main(): + args = parser().parse_args() + migrator = CustomConfigMigrator( + args.config_dir, + args.backup_root, + timeout=args.transport_timeout, + readiness_timeout=args.readiness_timeout, + ) + try: + if args.action == "plan": + print(json.dumps(migrator.plan(args.container, args.proxy_image).sanitized_manifest(), sort_keys=True)) + elif args.action == "apply": + outcome = migrator.apply(args.container, args.proxy_image, args.migration_id, args.enable_vpm) + print(json.dumps({"migration_id": args.migration_id, "state": outcome.state, "detail": outcome.detail})) + if outcome.state == "candidate_retained": + # The candidate is serving, but its fallback did not meet the + # required health contract. Surface that operator action is + # required without letting generic exception cleanup rewrite + # the live managed configuration. + return 2 + else: + outcome = migrator.rollback(args.container, args.migration_id) + print(json.dumps({"migration_id": args.migration_id, "state": outcome.state, "detail": outcome.detail})) + except MigrationError as error: + print(f"migration failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cryptolabs_proxy/auth.py b/src/cryptolabs_proxy/auth.py index def0707..5612abd 100644 --- a/src/cryptolabs_proxy/auth.py +++ b/src/cryptolabs_proxy/auth.py @@ -1442,6 +1442,20 @@ def get_headers(): return response return '', 401 + + @app.route('/auth/vast-price-manager/authorize') + def authorize_vast_price_manager(): + """Authorize the Fleet administrator-only VPM proxy route. + + This endpoint is used only by Nginx's internal ``auth_request`` + subrequest. It reads the signed-in Flask session and deliberately + ignores incoming role headers, which a client could forge. + """ + if not session.get('logged_in') or session.get('require_password_change'): + return '', 401 + if session.get('role') != 'admin': + return '', 403 + return '', 204 # API endpoints for programmatic access @app.route('/auth/api/users', methods=['GET']) diff --git a/src/cryptolabs_proxy/cli.py b/src/cryptolabs_proxy/cli.py index 23a47a7..1d3dedc 100644 --- a/src/cryptolabs_proxy/cli.py +++ b/src/cryptolabs_proxy/cli.py @@ -1,6 +1,9 @@ """CryptoLabs Proxy CLI - Setup and manage the unified reverse proxy.""" import click +import copy +from contextlib import contextmanager +import fcntl import os import subprocess import sys @@ -14,10 +17,22 @@ from . import __version__ from .config import CONFIG_DIR, get_jinja_env, generate_nginx_config, generate_docker_compose -from .services import ServiceRegistry +from .custom_config import ( + CustomConfigError, + custom_config_settings, + ensure_vpm_only_change, + load_verified_baseline, + render_managed_vpm_config, +) +from .services import DEFAULT_SERVICES, ServiceRegistry console = Console() +# A registry mutation can invoke the proxy container, but each command is +# bounded so a stalled Docker daemon cannot indefinitely block other lifecycle +# operations waiting for the registry lock. +NGINX_COMMAND_TIMEOUT = 15 + custom_style = questionary.Style([ ('qmark', 'fg:cyan bold'), ('question', 'bold'), @@ -90,6 +105,140 @@ def get_local_ip() -> str: return "127.0.0.1" +def validate_nginx_config(): + """Validate the mounted proxy configuration before asking Nginx to reload.""" + try: + result = subprocess.run( + ["docker", "exec", "cryptolabs-proxy", "nginx", "-t"], + capture_output=True, + text=True, + timeout=NGINX_COMMAND_TIMEOUT, + ) + return result.returncode == 0, result.stderr or result.stdout + except (OSError, subprocess.SubprocessError) as error: + return False, str(error) + + +def reload_nginx_config(): + """Reload Nginx and return its output to the caller.""" + try: + result = subprocess.run( + ["docker", "exec", "cryptolabs-proxy", "nginx", "-s", "reload"], + capture_output=True, + text=True, + timeout=NGINX_COMMAND_TIMEOUT, + ) + return result.returncode == 0, result.stderr or result.stdout + except (OSError, subprocess.SubprocessError) as error: + return False, str(error) + + +def proxy_uses_generated_config(config_path: Path) -> bool: + """Confirm the active container sees exactly the generated host config. + + The image's bundled nginx.conf is a bootstrap configuration. Lifecycle + registration is supported only for deployments that mount the generated + config into /etc/nginx/nginx.conf; otherwise a reload would not change the + active routes. + """ + try: + result = subprocess.run( + ["docker", "exec", "cryptolabs-proxy", "cat", "/etc/nginx/nginx.conf"], + capture_output=True, + text=True, + timeout=NGINX_COMMAND_TIMEOUT, + ) + return result.returncode == 0 and config_path.read_text() == result.stdout + except (OSError, subprocess.SubprocessError): + return False + + +@contextmanager +def registry_lock(config_dir: Path): + """Serialize lifecycle changes to one generated proxy configuration. + + The lock deliberately covers registry loading through validation, reload, + and rollback. Docker calls made during that interval use a timeout so a + competing install or disable action is never held behind an unbounded + command. + """ + config_dir.mkdir(parents=True, exist_ok=True) + lock_path = config_dir / ".registry.lock" + with lock_path.open("a+") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def _snapshot_files(paths): + return {path: path.read_bytes() if path.exists() else None for path in paths} + + +def _restore_files(snapshot): + for path, content in snapshot.items(): + if content is None: + path.unlink(missing_ok=True) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + + +def apply_registry_change(registry: ServiceRegistry, services: dict): + """Persist, validate, and reload a registry mutation with file rollback. + + The route remains active until Nginx successfully reloads. If validation or + reload fails, registry/settings/config files are restored; a failed reload + also triggers an old-config reload attempt to make the active state match + the restored files. + """ + nginx_path = registry.config_dir / "nginx.conf" + snapshot = _snapshot_files([registry.services_file, registry.config_file, nginx_path]) + try: + if custom_config_settings(registry.config): + ensure_vpm_only_change(registry.services, services) + baseline = load_verified_baseline(registry.config) + rendered_config = render_managed_vpm_config( + baseline, + enabled="vast-price-manager" in services, + ) + else: + rendered_config = None + registry.services = services + registry.save() + if rendered_config is None: + generate_nginx_config( + registry.config_dir, + domain=registry.config.get("domain", get_local_ip()), + letsencrypt=registry.config.get("letsencrypt", False), + services=registry.services, + ) + else: + nginx_path.write_bytes(rendered_config) + if not proxy_uses_generated_config(nginx_path): + raise RuntimeError( + "Proxy is not using the generated /etc/cryptolabs-proxy/nginx.conf; " + "route change was not applied." + ) + valid, validation_output = validate_nginx_config() + if not valid: + raise RuntimeError(f"Nginx configuration validation failed: {validation_output.strip()}") + reloaded, reload_output = reload_nginx_config() + if reloaded: + return True, "" + + _restore_files(snapshot) + rollback_ok, rollback_output = reload_nginx_config() + detail = f"Nginx reload failed: {reload_output.strip()}" + if not rollback_ok: + detail += f"; rollback reload also failed: {rollback_output.strip()}" + return False, detail + except (OSError, RuntimeError, CustomConfigError) as error: + _restore_files(snapshot) + return False, str(error) + + # Docker network subnet for UFW rules DOCKER_NETWORK_SUBNET = "172.30.0.0/16" @@ -385,36 +534,53 @@ def setup(): def register(service_name, container_name, path, port, display_name, icon, description): """Register a service with the proxy.""" check_root() - - registry = ServiceRegistry(CONFIG_DIR) - - if path is None: - path = f"/{service_name}/" - - registry.add_service( - name=service_name, - container_name=container_name, - path=path, - port=port, - display_name=display_name or service_name.replace("-", " ").title(), - icon=icon, - description=description - ) - - # Regenerate nginx config - generate_nginx_config( - CONFIG_DIR, - domain=registry.config.get("domain", get_local_ip()), - letsencrypt=registry.config.get("letsencrypt", False), - services=registry.services - ) - - # Reload nginx - subprocess.run(["docker", "exec", "cryptolabs-proxy", "nginx", "-s", "reload"], capture_output=True) - + with registry_lock(CONFIG_DIR): + registry = ServiceRegistry(CONFIG_DIR) + + if path is None: + path = f"/{service_name}/" + services = copy.deepcopy(registry.services) + service = copy.deepcopy(services.get(service_name, {})) + defaults = DEFAULT_SERVICES.get(service_name, {}) + service.update({ + "container_name": container_name, + "path": path, + "port": port, + "display_name": display_name or defaults.get("display_name") or service_name.replace("-", " ").title(), + "icon": icon if icon != "🔧" else defaults.get("icon", icon), + "description": description or defaults.get("description", ""), + }) + # VPM registration is intentionally narrow: the route fragment has one + # fixed container/path/port contract, so retain every canonical field + # even when the installer invokes the generic CLI command. + if service_name == "vast-price-manager": + service = copy.deepcopy(defaults) + services[service_name] = service + applied, message = apply_registry_change(registry, services) + if not applied: + raise click.ClickException(message) console.print(f"[green]✓[/green] Registered {service_name} at {path}") +@main.command() +@click.argument("service_name") +def unregister(service_name): + """Remove a service route while preserving unrelated proxy settings.""" + check_root() + with registry_lock(CONFIG_DIR): + registry = ServiceRegistry(CONFIG_DIR) + if service_name not in registry.services: + console.print(f"[yellow]•[/yellow] {service_name} is not registered") + return + + services = copy.deepcopy(registry.services) + del services[service_name] + applied, message = apply_registry_change(registry, services) + if not applied: + raise click.ClickException(message) + console.print(f"[green]✓[/green] Unregistered {service_name}") + + @main.command() def status(): """Show status of all services.""" diff --git a/src/cryptolabs_proxy/config.py b/src/cryptolabs_proxy/config.py index f95e1f5..3d6ab83 100644 --- a/src/cryptolabs_proxy/config.py +++ b/src/cryptolabs_proxy/config.py @@ -31,6 +31,11 @@ def generate_nginx_config(config_dir: Path, domain: str, letsencrypt: bool = Fal (config_dir / "nginx.conf").write_text(content) +def render_vpm_fragment() -> str: + """Render the exact VPM route shared by generated and custom configs.""" + return get_jinja_env().get_template("vast-price-manager.conf.j2").render() + + def generate_docker_compose(config_dir: Path, domain: str = None, use_letsencrypt: bool = False): """Generate docker-compose.yml from template.""" env = get_jinja_env() diff --git a/src/cryptolabs_proxy/custom_config.py b/src/cryptolabs_proxy/custom_config.py new file mode 100644 index 0000000..c815feb --- /dev/null +++ b/src/cryptolabs_proxy/custom_config.py @@ -0,0 +1,120 @@ +"""VPM-only management for a preserved, custom Nginx baseline.""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path + +from .config import render_vpm_fragment + + +CUSTOM_CONFIG_KEY = "custom_config" +CUSTOM_CONFIG_MODE = "vpm-managed" +MANAGED_BEGIN = b"# BEGIN CRYPTOLABS MANAGED VPM" +MANAGED_END = b"# END CRYPTOLABS MANAGED VPM" +_ANCHOR = re.compile(rb"^[ \t]*location[ \t]+@login_redirect[ \t]*\{") + + +class CustomConfigError(RuntimeError): + """The preserved baseline cannot safely receive a VPM-only mutation.""" + + +def configure_custom_config_mode(config: dict, baseline_path: Path) -> None: + """Store the immutable baseline identity in existing registry settings.""" + baseline = baseline_path.read_bytes() + config[CUSTOM_CONFIG_KEY] = { + "mode": CUSTOM_CONFIG_MODE, + "baseline_path": str(baseline_path), + "baseline_sha256": hashlib.sha256(baseline).hexdigest(), + } + + +def custom_config_settings(config: dict) -> dict | None: + settings = config.get(CUSTOM_CONFIG_KEY) + if settings and settings.get("mode") == CUSTOM_CONFIG_MODE: + return settings + return None + + +def load_verified_baseline(config: dict) -> bytes: + """Read the root-owned baseline only when its configured hash still matches.""" + settings = custom_config_settings(config) + if not settings: + raise CustomConfigError("custom-config mode is not configured") + path = Path(settings["baseline_path"]) + baseline = path.read_bytes() + actual = hashlib.sha256(baseline).hexdigest() + if actual != settings.get("baseline_sha256"): + raise CustomConfigError("custom-config baseline SHA-256 changed; refusing mutation") + return baseline + + +def _strip_comments_and_strings(line: bytes) -> bytes: + """Keep braces structural while ignoring simple quoted Nginx directive text.""" + result = bytearray() + quote = None + escaped = False + for char in line: + if quote: + if escaped: + escaped = False + elif char == ord("\\"): + escaped = True + elif char == quote: + quote = None + continue + if char in (ord("'"), ord('"')): + quote = char + elif char == ord("#"): + break + else: + result.append(char) + return bytes(result) + + +def _anchor_end_offset(baseline: bytes) -> int: + lines = baseline.splitlines(keepends=True) + anchors = [index for index, line in enumerate(lines) if _ANCHOR.match(line)] + if len(anchors) != 1: + raise CustomConfigError("custom-config baseline must contain exactly one location @login_redirect anchor") + + anchor = anchors[0] + depth = 0 + for line in lines[:anchor]: + structural = _strip_comments_and_strings(line) + depth += structural.count(b"{") - structural.count(b"}") + target_depth = depth + offset = sum(len(line) for line in lines[:anchor]) + for line in lines[anchor:]: + structural = _strip_comments_and_strings(line) + depth += structural.count(b"{") - structural.count(b"}") + offset += len(line) + if depth == target_depth: + return offset + raise CustomConfigError("custom-config login redirect anchor has unbalanced Nginx braces") + + +def render_managed_vpm_config(baseline: bytes, enabled: bool) -> bytes: + """Return baseline bytes unchanged or splice the reviewed fragment after its anchor.""" + if not enabled: + return baseline + if MANAGED_BEGIN in baseline or MANAGED_END in baseline: + raise CustomConfigError("custom-config baseline already contains a managed VPM delimiter") + if b"upstream auth_server" not in baseline or b"location @service_unavailable" not in baseline: + raise CustomConfigError("custom-config baseline lacks required Fleet auth or service-unavailable handlers") + offset = _anchor_end_offset(baseline) + fragment = render_vpm_fragment().encode() + block = MANAGED_BEGIN + b"\n" + fragment.rstrip() + b"\n" + MANAGED_END + b"\n" + return baseline[:offset] + block + baseline[offset:] + + +def ensure_vpm_only_change(previous: dict, requested: dict) -> None: + """Custom mode never rewrites routes for a service other than VPM.""" + changed = { + name + for name in set(previous) | set(requested) + if previous.get(name) != requested.get(name) + } + if changed - {"vast-price-manager"}: + raise CustomConfigError("custom-config mode permits only vast-price-manager lifecycle mutations") diff --git a/src/cryptolabs_proxy/migration.py b/src/cryptolabs_proxy/migration.py new file mode 100644 index 0000000..7b7e62c --- /dev/null +++ b/src/cryptolabs_proxy/migration.py @@ -0,0 +1,622 @@ +"""Dry-run planning and bounded state helpers for custom VPM proxy migration. + +This module intentionally does not contact Docker at import time. The calling +host helper owns Docker execution; these functions build deterministic plans +and create payloads without printing inspected configuration or environment. +""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +import hashlib +import http.client +import json +import os +from pathlib import Path +import socket +import subprocess +import time +from typing import Any +from urllib.parse import quote + +from .custom_config import configure_custom_config_mode, render_managed_vpm_config + + +class MigrationError(RuntimeError): + """A custom-config migration precondition or compensated switch failed.""" + + +@dataclass(frozen=True) +class MigrationOutcome: + """The serving state reached by a completed compensating switch.""" + + state: str + detail: str = "" + + @classmethod + def applied(cls) -> "MigrationOutcome": + return cls("applied") + + @classmethod + def candidate_retained(cls, detail: str) -> "MigrationOutcome": + return cls("candidate_retained", detail) + + +# Keep this in sync with the image HEALTHCHECK in Dockerfile. A candidate can +# remain in Docker's `starting` state until the start period and a scheduled +# health check have elapsed; the Engine request timeout is deliberately a +# separate, short transport bound. +DOCKER_HEALTH_START_PERIOD = 10 +DOCKER_HEALTH_INTERVAL = 30 +DOCKER_HEALTH_TIMEOUT = 10 +DOCKER_HEALTH_RETRIES = 3 +PROXY_READY_TIMEOUT = ( + DOCKER_HEALTH_START_PERIOD + + DOCKER_HEALTH_INTERVAL * DOCKER_HEALTH_RETRIES + + DOCKER_HEALTH_TIMEOUT + + 10 # bounded local auth-probe allowance +) + + +@dataclass(frozen=True) +class MigrationPlan: + container_id: str + baseline_sha256: str + image: str + migration_id: str + + @classmethod + def from_source(cls, container_id: str, baseline: bytes, image: str) -> "MigrationPlan": + baseline_sha256 = hashlib.sha256(baseline).hexdigest() + material = f"{container_id}:{baseline_sha256}:{image}".encode() + return cls(container_id, baseline_sha256, image, hashlib.sha256(material).hexdigest()[:20]) + + def sanitized_manifest(self) -> dict[str, str]: + return { + "migration_id": self.migration_id, + "container_id": self.container_id, + "baseline_sha256": self.baseline_sha256, + "image": self.image, + } + + +def _mount_request(mount: dict[str, Any]) -> dict[str, Any]: + """Convert inspect mount data to the Docker create API representation.""" + mount_type = mount["Type"] + if mount_type not in ("bind", "volume"): + raise MigrationError(f"unsupported mount type {mount_type}; refusing to drop its options") + source = mount.get("Name") if mount_type == "volume" else mount.get("Source") + if not source: + raise MigrationError(f"cannot preserve {mount_type} mount without a source") + result = { + "Type": mount_type, + "Source": source, + "Target": mount["Destination"], + "ReadOnly": not mount.get("RW", True), + } + if mount_type == "bind" and mount.get("Propagation"): + result["BindOptions"] = {"Propagation": mount["Propagation"]} + return result + + +def endpoint_configurations(inspect: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Return only user-configured endpoint fields accepted by Docker create.""" + source_networks = inspect.get("NetworkSettings", {}).get("Networks", {}) + # Inspect also reports Engine-assigned endpoint IDs, IP addresses, and + # gateways. They are not valid create inputs; retain every user-configured + # endpoint setting while allowing Docker to allocate those runtime fields. + endpoint_fields = ("Aliases", "Links", "IPAMConfig", "MacAddress", "DriverOpts", "GwPriority") + networks = { + name: {field: deepcopy(endpoint[field]) for field in endpoint_fields if field in endpoint} + for name, endpoint in source_networks.items() + } + if not networks: + raise MigrationError("source proxy has no inspectable network endpoints") + return networks + + +def build_recreate_request(inspect: dict[str, Any], image: str, candidate_config_path: str) -> dict[str, Any]: + """Clone inspected Engine settings and add only the read-only config bind.""" + config = deepcopy(inspect["Config"]) + host_config = deepcopy(inspect["HostConfig"]) + config["Image"] = image + mounts = [_mount_request(mount) for mount in inspect.get("Mounts", [])] + if any(mount["Target"] == "/etc/nginx/nginx.conf" for mount in mounts): + raise MigrationError("source proxy already has an nginx.conf mount") + mounts.append({ + "Type": "bind", + "Source": candidate_config_path, + "Target": "/etc/nginx/nginx.conf", + "ReadOnly": True, + }) + # Docker accepts Mounts as the declarative form. Removing Binds prevents a + # duplicate mount while every source mount is reconstructed above. + host_config.pop("Binds", None) + host_config["Mounts"] = mounts + networks = endpoint_configurations(inspect) + return { + "Config": config, + "HostConfig": host_config, + "NetworkingConfig": {"EndpointsConfig": networks}, + } + + +def prepare_custom_registry(registry, baseline_path: Path, enable_vpm: bool) -> bytes: + """Configure custom mode and return the candidate config without reloading.""" + configure_custom_config_mode(registry.config, baseline_path) + if enable_vpm: + from .services import DEFAULT_SERVICES + registry.services["vast-price-manager"] = deepcopy(DEFAULT_SERVICES["vast-price-manager"]) + else: + registry.services.pop("vast-price-manager", None) + return render_managed_vpm_config(baseline_path.read_bytes(), enable_vpm) + + +class SwitchController: + """Compensating swap sequence with an injected Engine adapter for testing.""" + + def __init__(self, engine, health_check, rollback_health_check=None): + self.engine = engine + self.health_check = health_check + self.rollback_health_check = rollback_health_check or health_check + + def apply( + self, + source_id: str, + migration_id: str, + request: dict[str, Any], + candidate_name: str | None = None, + ) -> MigrationOutcome: + candidate_name = candidate_name or source_id + rollback_name = f"{candidate_name}.rollback-{migration_id}" + held_candidate_name = f"{candidate_name}.candidate-{migration_id}" + endpoints = request["NetworkingConfig"]["EndpointsConfig"] + candidate_created = False + renamed = False + stopped = False + detached_source_networks: list[str] = [] + try: + self.engine.stop(source_id) + stopped = True + for network in endpoints: + self.engine.disconnect(source_id, network) + detached_source_networks.append(network) + self.engine.rename(source_id, rollback_name) + renamed = True + self.engine.create(candidate_name, request) + candidate_created = True + self.engine.start(candidate_name) + if not self.health_check(candidate_name): + raise MigrationError("candidate proxy health check failed") + except Exception as error: + try: + if candidate_created: + self.engine.stop(candidate_name, ignore_missing=True) + for network in endpoints: + self.engine.disconnect(candidate_name, network) + self.engine.rename(candidate_name, held_candidate_name) + if renamed: + self.engine.rename(rollback_name, candidate_name) + for network, endpoint in endpoints.items(): + self.engine.connect(candidate_name, network, endpoint) + self.engine.start(candidate_name) + if not self.rollback_health_check(candidate_name): + if candidate_created: + self.engine.stop(candidate_name, ignore_missing=True) + for network in endpoints: + self.engine.disconnect(candidate_name, network) + self.engine.rename(candidate_name, rollback_name) + self.engine.rename(held_candidate_name, candidate_name) + for network, endpoint in endpoints.items(): + self.engine.connect(candidate_name, network, endpoint) + self.engine.start(candidate_name) + if not self.health_check(candidate_name): + raise MigrationError("candidate and restored proxy health checks both failed") + return MigrationOutcome.candidate_retained( + "source restoration health check failed; recovered candidate is serving" + ) + raise MigrationError("restored proxy health check failed") + if candidate_created: + self.engine.remove(held_candidate_name, ignore_missing=True) + elif stopped: + # A failed disconnect or rename leaves the source under + # its original name but stopped. Restore only endpoints + # whose detach request completed, preserving static IPs + # and aliases before restarting it. + for network in detached_source_networks: + self.engine.connect(source_id, network, endpoints[network]) + self.engine.start(source_id) + except Exception as rollback_error: + raise MigrationError( + f"custom proxy migration failed: {error}; rollback also failed: {rollback_error}" + ) from rollback_error + if isinstance(error, MigrationError): + raise + raise MigrationError(f"custom proxy migration failed: {error}") from error + return MigrationOutcome.applied() + + +def wait_for_proxy_ready(engine, container: str, auth_probe, timeout: int, interval: float = 0.25) -> bool: + """Wait through Docker health startup and require anonymous auth rejection.""" + deadline = time.monotonic() + timeout + while True: + state = engine.inspect(container).get("State", {}) + health = state.get("Health", {}).get("Status") + if not state.get("Running") or health == "unhealthy": + return False + if health == "healthy": + try: + return bool(auth_probe()) + except OSError: + return False + if time.monotonic() >= deadline: + return False + time.sleep(interval) + + +def wait_for_container_healthy(engine, container: str, timeout: int, interval: float = 0.25) -> bool: + """Bounded health wait used when restoring the pre-VPM proxy image.""" + deadline = time.monotonic() + timeout + while True: + state = engine.inspect(container).get("State", {}) + health = state.get("Health", {}).get("Status") + if not state.get("Running") or health == "unhealthy": + return False + if health == "healthy": + return True + if time.monotonic() >= deadline: + return False + time.sleep(interval) + + +class _UnixHTTPConnection(http.client.HTTPConnection): + """Small standard-library Docker Engine API client over its Unix socket.""" + + def __init__(self, socket_path: str, timeout: int = 15): + super().__init__("localhost", timeout=timeout) + self.socket_path = socket_path + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect(self.socket_path) + + +class DockerEngine: + """Bounded Engine actions that keep inspect/config values out of stdout.""" + + def __init__(self, socket_path: str = "/var/run/docker.sock", timeout: int = 15): + self.socket_path = socket_path + self.timeout = timeout + + def _request(self, method: str, path: str, payload: dict | None = None, allowed=(200, 201, 204)): + connection = _UnixHTTPConnection(self.socket_path, self.timeout) + body = json.dumps(payload).encode() if payload is not None else None + headers = {"Content-Type": "application/json"} if body else {} + try: + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + content = response.read() + except OSError as error: + raise MigrationError(f"Docker Engine request failed: {error}") from error + finally: + connection.close() + if response.status not in allowed: + # Engine failures can include sensitive Config data; retain only + # the status, never echo its body into command output. + raise MigrationError(f"Docker Engine {method} {path} returned HTTP {response.status}") + return content + + def inspect(self, container: str) -> dict[str, Any]: + return json.loads(self._request("GET", f"/containers/{quote(container, safe='')}/json")) + + def stop(self, container: str, ignore_missing: bool = False): + allowed = (204, 304, 404) if ignore_missing else (204, 304) + # Leave the Unix-socket read timeout enough headroom to receive the + # Engine response after its graceful-stop period. + grace = max(1, self.timeout - 2) + self._request("POST", f"/containers/{quote(container, safe='')}/stop?t={grace}", allowed=allowed) + + def rename(self, old: str, new: str): + self._request("POST", f"/containers/{quote(old, safe='')}/rename?name={quote(new, safe='')}", allowed=(204,)) + + def create(self, name: str, request: dict[str, Any]): + payload = deepcopy(request["Config"]) + payload["HostConfig"] = request["HostConfig"] + payload["NetworkingConfig"] = request["NetworkingConfig"] + self._request("POST", f"/containers/create?name={quote(name, safe='')}", payload, allowed=(201,)) + + def start(self, container: str): + self._request("POST", f"/containers/{quote(container, safe='')}/start", allowed=(204, 304)) + + def remove(self, container: str, ignore_missing: bool = False): + allowed = (204, 404) if ignore_missing else (204,) + self._request("DELETE", f"/containers/{quote(container, safe='')}?force=1", allowed=allowed) + + def disconnect(self, container: str, network: str): + self._request( + "POST", + f"/networks/{quote(network, safe='')}/disconnect", + {"Container": container, "Force": True}, + allowed=(200,), + ) + + def connect(self, container: str, network: str, endpoint: dict[str, Any]): + self._request( + "POST", + f"/networks/{quote(network, safe='')}/connect", + {"Container": container, "EndpointConfig": endpoint}, + allowed=(200,), + ) + + def healthy(self, container: str) -> bool: + state = self.inspect(container).get("State", {}) + health = state.get("Health", {}).get("Status") + return state.get("Running") and health in (None, "healthy") + + +class CustomConfigMigrator: + """Explicit plan/apply/rollback workflow for one custom proxy container.""" + + def __init__( + self, + config_dir: Path, + backup_root: Path, + engine: DockerEngine | None = None, + timeout: int = 15, + readiness_timeout: int = PROXY_READY_TIMEOUT, + ): + self.config_dir = config_dir + self.backup_root = backup_root + self.engine = engine or DockerEngine(timeout=timeout) + # `timeout` bounds each Unix socket/subprocess/HTTP operation. It is + # not the lifecycle deadline, because Docker's own health cadence is + # intentionally much longer. + self.timeout = timeout + if readiness_timeout < PROXY_READY_TIMEOUT: + raise MigrationError( + f"readiness timeout must be at least {PROXY_READY_TIMEOUT} seconds for the proxy healthcheck" + ) + self.readiness_timeout = readiness_timeout + + def _read_active_config(self, container: str) -> bytes: + try: + result = subprocess.run( + ["docker", "exec", container, "cat", "/etc/nginx/nginx.conf"], + capture_output=True, + timeout=self.timeout, + ) + except (OSError, subprocess.SubprocessError) as error: + raise MigrationError(f"cannot read active proxy config: {error}") from error + if result.returncode: + raise MigrationError("cannot read active proxy config") + return result.stdout + + def plan(self, container: str, image: str) -> MigrationPlan: + """Read only: return a deterministic, sanitized migration identity.""" + if "@sha256:" not in image: + raise MigrationError("proxy image must be an immutable sha256 digest") + inspect = self.engine.inspect(container) + return MigrationPlan.from_source(inspect["Id"], self._read_active_config(container), image) + + def _backup_path(self, migration_id: str) -> Path: + return self.backup_root / migration_id + + @staticmethod + def _write_private(path: Path, content: bytes): + path.write_bytes(content) + path.chmod(0o600) + + def _validate_candidate(self, image: str, candidate: Path, inspect: dict[str, Any], migration_id: str): + ssl_mount = next((m for m in inspect.get("Mounts", []) if m.get("Destination") == "/etc/nginx/ssl"), None) + if not ssl_mount: + raise MigrationError("source proxy has no /etc/nginx/ssl mount for candidate validation") + validation_name = f"cryptolabs-vpm-validate-{migration_id}"[:63] + command = [ + "docker", "run", "--name", validation_name, + "--label", f"cryptolabs.migration.validation={migration_id}", + "--network", "none", "--entrypoint", "nginx", + "-v", f"{candidate}:/etc/nginx/nginx.conf:ro", + "-v", f"{ssl_mount['Source']}:/etc/nginx/ssl:ro", + image, "-t", + ] + try: + result = subprocess.run(command, capture_output=True, timeout=self.timeout) + except (OSError, subprocess.SubprocessError) as error: + raise MigrationError(f"candidate Nginx validation could not run: {error}") from error + finally: + # The image entrypoint may ignore argv or outlive a killed Docker + # client. Explicitly remove only our unique labeled container. + subprocess.run( + ["docker", "rm", "-f", validation_name], + capture_output=True, + timeout=self.timeout, + ) + if result.returncode: + raise MigrationError("candidate Nginx validation failed") + + def _anonymous_auth_is_rejected(self) -> bool: + """The replacement must expose the new endpoint without a session.""" + connection = http.client.HTTPConnection("127.0.0.1", 80, timeout=self.timeout) + try: + connection.request("GET", "/auth/vast-price-manager/authorize") + return connection.getresponse().status == 401 + except OSError: + return False + finally: + connection.close() + + def apply(self, container: str, image: str, migration_id: str, enable_vpm: bool) -> MigrationOutcome: + """Snapshot privately, validate, then perform a compensating proxy-only swap.""" + if os.geteuid() != 0: + raise MigrationError("custom proxy migration must run as root") + from .cli import registry_lock + with registry_lock(self.config_dir): + # Rebuild the read-only plan while holding the same lock that + # serializes later VPM register/unregister writes. + plan = self.plan(container, image) + if plan.migration_id != migration_id: + raise MigrationError("migration ID does not match the current source container/config/image") + backup = self._backup_path(migration_id) + if backup.exists(): + raise MigrationError("migration backup already exists") + self.backup_root.mkdir(parents=True, mode=0o700) + self.backup_root.chmod(0o700) + backup.mkdir(parents=True, mode=0o700) + backup.chmod(0o700) + inspect = self.engine.inspect(container) + baseline = self._read_active_config(container) + current = MigrationPlan.from_source(inspect["Id"], baseline, image) + if current != plan: + raise MigrationError("source container ID or config changed before switch") + self._write_private(backup / "baseline.nginx.conf", baseline) + self._write_private(backup / "inspect.json", json.dumps(inspect, sort_keys=True).encode()) + from .services import ServiceRegistry + registry = ServiceRegistry(self.config_dir) + previous = _snapshot_registry_files(registry) + _write_registry_backup(backup, previous) + candidate = self.config_dir / "nginx.conf" + previous_candidate = candidate.read_bytes() if candidate.exists() else None + try: + candidate_content = prepare_custom_registry(registry, backup / "baseline.nginx.conf", enable_vpm) + self._write_private(candidate, candidate_content) + registry.save() + self._validate_candidate(image, candidate, inspect, migration_id) + request = build_recreate_request(inspect, image, str(candidate)) + outcome = SwitchController( + self.engine, + lambda name: wait_for_proxy_ready( + self.engine, + name, + self._anonymous_auth_is_rejected, + self.readiness_timeout, + ), + rollback_health_check=lambda name: wait_for_container_healthy( + self.engine, + name, + self.readiness_timeout, + ), + ).apply(plan.container_id, migration_id, request, candidate_name=container) + except Exception: + _restore_registry_files(previous) + if previous_candidate is None: + candidate.unlink(missing_ok=True) + else: + self._write_private(candidate, previous_candidate) + raise + # A retained candidate is the live recovery path. Marker writing + # is diagnostic only and must never re-enter generic cleanup that + # would unlink/revert the configuration it is serving. + if outcome.state == "candidate_retained": + try: + self._write_private( + backup / "outcome.json", + json.dumps({"state": outcome.state, "detail": outcome.detail}, sort_keys=True).encode(), + ) + except OSError: + return MigrationOutcome.candidate_retained( + f"{outcome.detail}; recovery marker could not be written" + ) + return outcome + + def rollback(self, container: str, migration_id: str) -> MigrationOutcome: + """Restore only this proxy without discarding its healthy replacement.""" + if os.geteuid() != 0: + raise MigrationError("custom proxy rollback must run as root") + from .cli import registry_lock + # Match lifecycle register/unregister: the same lock covers runtime + # changes and the registry/config restore so no command observes or + # persists a split state. + with registry_lock(self.config_dir): + return self._rollback_locked(container, migration_id) + + def _rollback_locked(self, container: str, migration_id: str) -> MigrationOutcome: + backup = self._backup_path(migration_id) + if not backup.is_dir(): + raise MigrationError("migration backup does not exist") + rollback_name = f"{container}.rollback-{migration_id}" + held_candidate = f"{container}.candidate-{migration_id}" + source = json.loads((backup / "inspect.json").read_text()) + restored = self.engine.inspect(rollback_name) + if restored.get("Id") != source.get("Id"): + raise MigrationError("rollback source does not match the private backup") + # Prove the replacement works before it is stopped or renamed. + if not wait_for_proxy_ready(self.engine, container, self._anonymous_auth_is_rejected, self.readiness_timeout): + raise MigrationError("current replacement is not healthy; refusing to remove its recovery path") + source_endpoints = endpoint_configurations(source) + candidate_endpoints = endpoint_configurations(self.engine.inspect(container)) + candidate_held = False + source_promoted = False + detached_candidate_networks: list[str] = [] + try: + self.engine.stop(container) + for network in candidate_endpoints: + self.engine.disconnect(container, network) + detached_candidate_networks.append(network) + self.engine.rename(container, held_candidate) + candidate_held = True + self.engine.rename(rollback_name, container) + source_promoted = True + for network, endpoint in source_endpoints.items(): + self.engine.connect(container, network, endpoint) + self.engine.start(container) + if not wait_for_container_healthy(self.engine, container, self.readiness_timeout): + raise MigrationError("restored proxy health check failed") + except Exception as error: + try: + if source_promoted: + self.engine.stop(container, ignore_missing=True) + for network in source_endpoints: + self.engine.disconnect(container, network) + self.engine.rename(container, rollback_name) + if candidate_held: + self.engine.rename(held_candidate, container) + for network, endpoint in candidate_endpoints.items(): + self.engine.connect(container, network, endpoint) + self.engine.start(container) + if not wait_for_proxy_ready( + self.engine, + container, + self._anonymous_auth_is_rejected, + self.readiness_timeout, + ): + raise MigrationError("candidate recovery health check failed") + else: + # Candidate was never renamed, so restore exactly the + # networks detached before the failed rename/disconnect. + for network in detached_candidate_networks: + self.engine.connect(container, network, candidate_endpoints[network]) + self.engine.start(container) + except Exception as recovery_error: + raise MigrationError( + f"rollback failed: {error}; candidate recovery also failed: {recovery_error}" + ) from recovery_error + raise MigrationError("rollback failed; healthy replacement was restored") from error + self.engine.remove(held_candidate, ignore_missing=True) + _restore_registry_files(_read_registry_backup(backup)) + return MigrationOutcome("rolled_back") + + +def _snapshot_registry_files(registry) -> dict[Path, bytes | None]: + paths = [registry.services_file, registry.config_file, registry.config_dir / "nginx.conf"] + return {path: path.read_bytes() if path.exists() else None for path in paths} + + +def _write_registry_backup(backup: Path, snapshot: dict[Path, bytes | None]): + encoded = {str(path): content.decode("latin1") if content is not None else None for path, content in snapshot.items()} + CustomConfigMigrator._write_private(backup / "registry-snapshot.json", json.dumps(encoded).encode()) + + +def _read_registry_backup(backup: Path) -> dict[Path, bytes | None]: + encoded = json.loads((backup / "registry-snapshot.json").read_text()) + return {Path(path): content.encode("latin1") if content is not None else None for path, content in encoded.items()} + + +def _restore_registry_files(snapshot: dict[Path, bytes | None]): + for path, content in snapshot.items(): + if content is None: + path.unlink(missing_ok=True) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) diff --git a/src/cryptolabs_proxy/services.py b/src/cryptolabs_proxy/services.py index 72d406f..4922913 100644 --- a/src/cryptolabs_proxy/services.py +++ b/src/cryptolabs_proxy/services.py @@ -71,6 +71,18 @@ "icon": "💎", "description": "Prometheus exporter for Vast.ai metrics", }, + "vast-price-manager": { + "container_name": "vast-price-manager", + "path": "/vast-pricing/", + "port": 8088, + "display_name": "Vast Price Manager", + "icon": "💰", + "description": "Optional Vast pricing management service with its own secure login.", + "product_url": "https://github.com/cryptolabsza/vast-price-manager", + "admin_only": True, + "lifecycle_manager": "dc-overview", + "optional": True, + }, "runpod-exporter": { "container_name": "runpod-exporter", "path": "/runpod-metrics/", diff --git a/src/cryptolabs_proxy/templates/nginx.conf.j2 b/src/cryptolabs_proxy/templates/nginx.conf.j2 index 496d1e0..b3d5344 100644 --- a/src/cryptolabs_proxy/templates/nginx.conf.j2 +++ b/src/cryptolabs_proxy/templates/nginx.conf.j2 @@ -158,8 +158,14 @@ http { return 302 /auth/login?next=$request_uri; } + {% if 'vast-price-manager' in services %} + {% include "vast-price-manager.conf.j2" %} + {% endif %} + {% for name, service in services.items() %} - {% if not service.path %} + {% if name == 'vast-price-manager' %} + # Vast Price Manager uses its dedicated admin-only route above. + {% elif not service.path %} # {{ service.get('display_name', name) }} - external service, no local proxy route {% else %} # {{ service.get('display_name', name) }} - uses variable for runtime DNS resolution diff --git a/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 b/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 new file mode 100644 index 0000000..c4d2ef6 --- /dev/null +++ b/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 @@ -0,0 +1,38 @@ + # Vast Price Manager has a dedicated admin-only route. It strips the + # prefix once and explicitly preserves the remainder and query string. + location = /_vast_pricing_admin { + internal; + proxy_pass http://127.0.0.1:8081/auth/vast-price-manager/authorize; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header Cookie $http_cookie; + proxy_set_header X-Original-URI $request_uri; + } + + location = /vast-pricing { + return 308 /vast-pricing/; + } + + location ~ ^/vast-pricing(?/.*)$ { + auth_request /_vast_pricing_admin; + error_page 401 = @login_redirect; + + # Runtime Docker DNS resolution permits startup before this opt-in + # service has been installed. + set $upstream_vast_price_manager vast-price-manager; + proxy_pass http://$upstream_vast_price_manager:8088$vpm_upstream_path$is_args$args; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + # VPM retains its own login. Never pass client-supplied Fleet + # identity headers to it. + proxy_set_header X-Fleet-Auth-User ""; + proxy_set_header X-Fleet-Auth-Role ""; + proxy_set_header X-Fleet-Auth-Token ""; + proxy_set_header X-Fleet-Authenticated ""; + proxy_intercept_errors on; + error_page 502 503 504 = @service_unavailable; + } diff --git a/tests/test_vast_price_manager.py b/tests/test_vast_price_manager.py new file mode 100644 index 0000000..ab9d45e --- /dev/null +++ b/tests/test_vast_price_manager.py @@ -0,0 +1,1008 @@ +"""Contract tests for the optional Vast Price Manager Fleet integration.""" + +import importlib.util +from copy import deepcopy +import hashlib +from http.server import BaseHTTPRequestHandler, HTTPServer +from multiprocessing import get_context +import os +from pathlib import Path +import threading + +from click.testing import CliRunner +import pytest + + +REPOSITORY = Path(__file__).resolve().parents[1] + + +def _register_service_in_process(config_dir, service_name, ready, start, result_queue): + """Run a real CLI registration in a separate process for lock coverage.""" + import cryptolabs_proxy.cli as cli + + cli.CONFIG_DIR = Path(config_dir) + cli.check_root = lambda: None + cli.proxy_uses_generated_config = lambda path: True + cli.validate_nginx_config = lambda: (True, "") + cli.reload_nginx_config = lambda: (True, "") + ready.set() + start.wait(5) + result = CliRunner().invoke( + cli.main, + ["register", service_name, service_name, "--path", f"/{service_name}/", "--port", "9010"], + ) + result_queue.put((result.exit_code, result.output)) + + +def _register_service_after_rollback_lock(config_dir, entered, result_queue): + """Attempt a real lifecycle command and mark only after its registry lock.""" + import cryptolabs_proxy.cli as cli + + cli.CONFIG_DIR = Path(config_dir) + cli.check_root = lambda: None + cli.proxy_uses_generated_config = lambda path: entered.set() or True + cli.validate_nginx_config = lambda: (True, "") + cli.reload_nginx_config = lambda: (True, "") + result = CliRunner().invoke( + cli.main, + ["register", "service-after-rollback", "service-after-rollback", "--path", "/after/", "--port", "9011"], + ) + result_queue.put((result.exit_code, result.output)) + + +def load_health_api(): + """Load the standalone health API script without starting its server.""" + spec = importlib.util.spec_from_file_location( + "health_api", REPOSITORY / "scripts" / "health-api.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_vast_price_manager_registry_contract(): + from cryptolabs_proxy.services import DEFAULT_SERVICES + + service = DEFAULT_SERVICES["vast-price-manager"] + assert service["container_name"] == "vast-price-manager" + assert service["path"] == "/vast-pricing/" + assert service["port"] == 8088 + assert service["display_name"] == "Vast Price Manager" + assert service["product_url"] == "https://github.com/cryptolabsza/vast-price-manager" + assert service["admin_only"] is True + assert service["lifecycle_manager"] == "dc-overview" + + +def test_rendered_vpm_route_authorizes_session_admin_and_preserves_uri(tmp_path): + from cryptolabs_proxy.config import generate_nginx_config + from cryptolabs_proxy.services import DEFAULT_SERVICES + + generate_nginx_config(tmp_path, "fleet.example.test", services={}) + config = (tmp_path / "nginx.conf").read_text() + + assert "location = /vast-pricing {" not in config + + generate_nginx_config( + tmp_path, + "fleet.example.test", + services={"vast-price-manager": deepcopy(DEFAULT_SERVICES["vast-price-manager"])}, + ) + config = (tmp_path / "nginx.conf").read_text() + + assert "location = /vast-pricing {" in config + assert "return 308 /vast-pricing/;" in config + assert "location = /_vast_pricing_admin {" in config + assert "proxy_pass http://127.0.0.1:8081/auth/vast-price-manager/authorize;" in config + assert "location ~ ^/vast-pricing(?/.*)$ {" in config + vpm_route = config.split("location ~ ^/vast-pricing(?/.*)$ {")[1].split("\n }", 1)[0] + assert "auth_request /_vast_pricing_admin;" in vpm_route + assert "proxy_pass http://$upstream_vast_price_manager:8088$vpm_upstream_path$is_args$args;" in vpm_route + assert "proxy_set_header X-Forwarded-Host $host;" in vpm_route + assert "proxy_set_header X-Forwarded-Proto $scheme;" in vpm_route + assert 'proxy_set_header X-Fleet-Auth-Role "";' in vpm_route + + +def test_cli_register_unregister_toggles_generated_vpm_route_and_preserves_metadata(monkeypatch, tmp_path): + import cryptolabs_proxy.cli as cli + from cryptolabs_proxy.services import ServiceRegistry + + monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) + monkeypatch.setattr(cli, "check_root", lambda: None) + monkeypatch.setattr(cli, "proxy_uses_generated_config", lambda path: True) + monkeypatch.setattr(cli, "validate_nginx_config", lambda: (True, "")) + monkeypatch.setattr(cli, "reload_nginx_config", lambda: (True, "")) + + original = ServiceRegistry(tmp_path) + original.config = {"domain": "fleet.example.test", "letsencrypt": False, "keep": "setting"} + original.add_service("existing-service", "existing-service", "/existing/", 9010, description="keep me") + + runner = CliRunner() + register = runner.invoke( + cli.main, + ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"], + ) + assert register.exit_code == 0, register.output + + registry = ServiceRegistry(tmp_path) + service = registry.get_service("vast-price-manager") + assert service["lifecycle_manager"] == "dc-overview" + assert service["admin_only"] is True + assert "location = /vast-pricing {" in (tmp_path / "nginx.conf").read_text() + assert registry.get_service("existing-service")["description"] == "keep me" + assert registry.config["keep"] == "setting" + + unregister = runner.invoke(cli.main, ["unregister", "vast-price-manager"]) + assert unregister.exit_code == 0, unregister.output + assert ServiceRegistry(tmp_path).get_service("vast-price-manager") is None + assert "location = /vast-pricing {" not in (tmp_path / "nginx.conf").read_text() + assert ServiceRegistry(tmp_path).get_service("existing-service")["path"] == "/existing/" + assert ServiceRegistry(tmp_path).config["keep"] == "setting" + + reregister = runner.invoke( + cli.main, + ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"], + ) + assert reregister.exit_code == 0, reregister.output + assert ServiceRegistry(tmp_path).get_service("vast-price-manager")["lifecycle_manager"] == "dc-overview" + assert "location = /vast-pricing {" in (tmp_path / "nginx.conf").read_text() + + +def test_cli_unregister_rolls_back_registry_and_generated_config_when_reload_fails(monkeypatch, tmp_path): + import cryptolabs_proxy.cli as cli + from cryptolabs_proxy.config import generate_nginx_config + from cryptolabs_proxy.services import DEFAULT_SERVICES, ServiceRegistry + + registry = ServiceRegistry(tmp_path) + registry.config = {"domain": "fleet.example.test", "letsencrypt": False, "keep": "setting"} + registry.services = {"vast-price-manager": deepcopy(DEFAULT_SERVICES["vast-price-manager"])} + registry.save() + generate_nginx_config(tmp_path, "fleet.example.test", services=registry.services) + previous_registry = registry.services_file.read_text() + previous_config = (tmp_path / "nginx.conf").read_text() + + monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) + monkeypatch.setattr(cli, "check_root", lambda: None) + monkeypatch.setattr(cli, "proxy_uses_generated_config", lambda path: True) + monkeypatch.setattr(cli, "validate_nginx_config", lambda: (True, "")) + reload_calls = [] + monkeypatch.setattr(cli, "reload_nginx_config", lambda: reload_calls.append(True) or (False, "reload failed")) + + result = CliRunner().invoke(cli.main, ["unregister", "vast-price-manager"]) + + assert result.exit_code != 0 + assert "reload failed" in result.output + assert registry.services_file.read_text() == previous_registry + assert (tmp_path / "nginx.conf").read_text() == previous_config + assert "location = /vast-pricing {" in (tmp_path / "nginx.conf").read_text() + assert len(reload_calls) == 2 + + +def test_default_nginx_config_has_no_vpm_route_before_generated_config_migration(): + config = (REPOSITORY / "nginx" / "nginx.conf").read_text() + + assert "location = /vast-pricing {" not in config + assert "location = /_vast_pricing_admin {" not in config + assert "location ~ ^/vast-pricing(?/.*)$ {" not in config + + +def test_concurrent_cli_registrations_preserve_both_services(tmp_path): + context = get_context("fork") + start = context.Event() + first_ready = context.Event() + second_ready = context.Event() + result_queue = context.Queue() + first = context.Process( + target=_register_service_in_process, + args=(str(tmp_path), "service-one", first_ready, start, result_queue), + ) + second = context.Process( + target=_register_service_in_process, + args=(str(tmp_path), "service-two", second_ready, start, result_queue), + ) + first.start() + second.start() + assert first_ready.wait(5) + assert second_ready.wait(5) + start.set() + first.join(10) + second.join(10) + + assert first.exitcode == 0 + assert second.exitcode == 0 + assert [result_queue.get(timeout=2)[0] for _ in range(2)] == [0, 0] + + from cryptolabs_proxy.services import ServiceRegistry + + assert set(ServiceRegistry(tmp_path).services) == {"service-one", "service-two"} + + +def test_vpm_admin_authorization_is_session_backed_not_header_backed(client, admin_user): + anonymous = client.get("/auth/vast-price-manager/authorize") + assert anonymous.status_code == 401 + + forged = client.get( + "/auth/vast-price-manager/authorize", + headers={"X-Fleet-Auth-Role": "admin", "X-Fleet-Auth-User": "forged"}, + ) + assert forged.status_code == 401 + + client.post("/auth/login", data={"username": admin_user["username"], "password": admin_user["password"]}) + admin = client.get("/auth/vast-price-manager/authorize") + assert admin.status_code == 204 + + +def test_vpm_admin_authorization_rejects_readonly_and_readwrite(client, readonly_user, readwrite_user): + for user in (readonly_user, readwrite_user): + client.post("/auth/login", data={"username": user["username"], "password": user["password"]}) + response = client.get("/auth/vast-price-manager/authorize") + assert response.status_code == 403 + client.get("/auth/logout") + + +def test_vpm_health_status_distinguishes_unconfigured_from_healthy(monkeypatch): + health_api = load_health_api() + monkeypatch.setattr(health_api, "check_container_running", lambda name: name == "vast-price-manager") + monkeypatch.setattr(health_api, "get_vast_price_manager_readiness", lambda: "unconfigured") + monkeypatch.setattr(health_api, "get_vast_price_manager_docker_health", lambda: "healthy") + + status = health_api.get_all_service_status()["vast-price-manager"] + + assert status["running"] is True + assert status["healthy"] is True + assert status["state"] == "unconfigured" + assert status["configured"] is False + assert status["lifecycle_manager"] == "dc-overview" + + +def test_vpm_health_does_not_treat_a_running_unhealthy_container_as_healthy(monkeypatch): + health_api = load_health_api() + monkeypatch.setattr(health_api, "check_container_running", lambda name: name == "vast-price-manager") + monkeypatch.setattr(health_api, "get_vast_price_manager_readiness", lambda: "ready") + monkeypatch.setattr(health_api, "get_vast_price_manager_docker_health", lambda: "unhealthy") + + status = health_api.get_all_service_status()["vast-price-manager"] + + assert status["running"] is True + assert status["healthy"] is False + assert status["docker_health"] == "unhealthy" + assert status["configured"] is True + + +def test_vpm_allowed_host_reads_only_the_nonsecret_container_setting(monkeypatch): + health_api = load_health_api() + + class Result: + returncode = 0 + stdout = "VPM_ALLOWED_HOSTS=fleet.example.test, alternative.example.test\nVPM_MASTER_KEY=not-read\n" + + monkeypatch.setattr(health_api.subprocess, "run", lambda *args, **kwargs: Result()) + + assert health_api.get_vast_price_manager_allowed_host() == "fleet.example.test" + + +@pytest.mark.skipif( + os.environ.get("PROXY_NETWORK_TEST") != "1", + reason="requires a local HTTP listener for probe transport verification", +) +def test_vpm_readiness_probe_uses_allowed_public_host_over_http_transport(monkeypatch): + observed = {} + + class VpmReadinessHandler(BaseHTTPRequestHandler): + def do_GET(self): + observed["path"] = self.path + observed["host"] = self.headers.get("Host") + self.send_response(503 if observed["host"] == "fleet.example.test" else 400) + self.end_headers() + + def log_message(self, format, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), VpmReadinessHandler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + health_api = load_health_api() + monkeypatch.setattr(health_api, "VPM_READY_URL", f"http://127.0.0.1:{server.server_port}/readyz", raising=False) + monkeypatch.setattr(health_api, "get_vast_price_manager_allowed_host", lambda: "fleet.example.test", raising=False) + + assert health_api.get_vast_price_manager_readiness() == "unconfigured" + assert observed == {"path": "/readyz", "host": "fleet.example.test"} + finally: + server.shutdown() + thread.join() + + +def test_vpm_lifecycle_actions_are_rejected_by_generic_update_api(): + health_api = load_health_api() + assert health_api.service_action_error("vast-price-manager") == ( + "Vast Price Manager lifecycle is managed by dc-overview." + ) + + +CUSTOM_PROXY_CONFIG = b'''worker_processes auto; +events { worker_connections 1024; } +http { + upstream auth_server { server 127.0.0.1:8081; } + server { listen 80; location /legacy/ { proxy_pass http://legacy; } } + server { + listen 443 ssl; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + location @service_unavailable { return 503; } + location @login_redirect { + return 302 /auth/login?next=$request_uri; + } + location /auth/ { proxy_pass http://auth_server/auth/; } + location /unrelated/ { default_type application/json; return 200 '{"keep":"bytes"}'; } + } + server { listen 8080; location /health { return 200; } } +} +''' + + +def test_custom_config_render_preserves_baseline_bytes_and_uses_canonical_fragment(tmp_path): + from cryptolabs_proxy.custom_config import render_managed_vpm_config + from cryptolabs_proxy.config import render_vpm_fragment + + rendered = render_managed_vpm_config(CUSTOM_PROXY_CONFIG, enabled=True) + + assert b'# BEGIN CRYPTOLABS MANAGED VPM' in rendered + assert render_vpm_fragment().encode() in rendered + before, managed = rendered.split(b'# BEGIN CRYPTOLABS MANAGED VPM', 1) + managed, after = managed.split(b'# END CRYPTOLABS MANAGED VPM', 1) + assert before == CUSTOM_PROXY_CONFIG[:len(before)] + assert after == CUSTOM_PROXY_CONFIG[len(CUSTOM_PROXY_CONFIG) - len(after):] + assert b'location /unrelated/ { default_type application/json; return 200 \'{"keep":"bytes"}\'; }' in rendered + assert render_managed_vpm_config(CUSTOM_PROXY_CONFIG, enabled=False) == CUSTOM_PROXY_CONFIG + + +@pytest.mark.parametrize( + "baseline, expected", + [ + (CUSTOM_PROXY_CONFIG.replace(b"location @login_redirect", b"location @not_login_redirect"), "exactly one"), + (CUSTOM_PROXY_CONFIG.replace(b"location @login_redirect", b"location @login_redirect", 1) + b"\nlocation @login_redirect { return 302 /; }\n", "exactly one"), + (CUSTOM_PROXY_CONFIG + b"# BEGIN CRYPTOLABS MANAGED VPM\n", "managed VPM"), + (CUSTOM_PROXY_CONFIG.replace(b"location @service_unavailable", b"location @not_service_unavailable"), "required Fleet auth"), + ], +) +def test_custom_config_render_rejects_missing_duplicate_or_previously_managed_anchors(baseline, expected): + from cryptolabs_proxy.custom_config import CustomConfigError, render_managed_vpm_config + + with pytest.raises(CustomConfigError, match=expected): + render_managed_vpm_config(baseline, enabled=True) + + +def test_cli_vpm_lifecycle_preserves_custom_baseline_and_rejects_unrelated_routes(monkeypatch, tmp_path): + import cryptolabs_proxy.cli as cli + from cryptolabs_proxy.custom_config import configure_custom_config_mode + from cryptolabs_proxy.services import ServiceRegistry + + baseline_path = tmp_path / "baselines" / "proxy.conf" + baseline_path.parent.mkdir() + baseline_path.write_bytes(CUSTOM_PROXY_CONFIG) + registry = ServiceRegistry(tmp_path) + registry.services = {"existing-service": {"container_name": "existing", "path": "/existing/", "port": 9000}} + configure_custom_config_mode(registry.config, baseline_path) + registry.save() + + monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) + monkeypatch.setattr(cli, "check_root", lambda: None) + monkeypatch.setattr(cli, "proxy_uses_generated_config", lambda path: True) + monkeypatch.setattr(cli, "validate_nginx_config", lambda: (True, "")) + monkeypatch.setattr(cli, "reload_nginx_config", lambda: (True, "")) + runner = CliRunner() + + enabled = runner.invoke(cli.main, ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"]) + assert enabled.exit_code == 0, enabled.output + candidate = (tmp_path / "nginx.conf").read_bytes() + assert b"location /unrelated/" in candidate + assert b"# BEGIN CRYPTOLABS MANAGED VPM" in candidate + assert ServiceRegistry(tmp_path).get_service("vast-price-manager")["path"] == "/vast-pricing/" + assert ServiceRegistry(tmp_path).get_service("vast-price-manager")["port"] == 8088 + assert ServiceRegistry(tmp_path).get_service("existing-service")["path"] == "/existing/" + + rejected = runner.invoke(cli.main, ["register", "unrelated-service", "unrelated", "--path", "/unrelated-new/", "--port", "9001"]) + assert rejected.exit_code != 0 + assert "custom-config mode permits only vast-price-manager" in rejected.output + assert (tmp_path / "nginx.conf").read_bytes() == candidate + + disabled = runner.invoke(cli.main, ["unregister", "vast-price-manager"]) + assert disabled.exit_code == 0, disabled.output + assert (tmp_path / "nginx.conf").read_bytes() == CUSTOM_PROXY_CONFIG + assert ServiceRegistry(tmp_path).config["custom_config"]["baseline_sha256"] == hashlib.sha256(CUSTOM_PROXY_CONFIG).hexdigest() + + +def test_custom_mode_refuses_changed_baseline_before_mutating_registry(monkeypatch, tmp_path): + import cryptolabs_proxy.cli as cli + from cryptolabs_proxy.custom_config import configure_custom_config_mode + from cryptolabs_proxy.services import ServiceRegistry + + baseline_path = tmp_path / "baseline.conf" + baseline_path.write_bytes(CUSTOM_PROXY_CONFIG) + registry = ServiceRegistry(tmp_path) + configure_custom_config_mode(registry.config, baseline_path) + registry.save() + baseline_path.write_bytes(CUSTOM_PROXY_CONFIG + b"# changed") + + monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) + monkeypatch.setattr(cli, "check_root", lambda: None) + result = CliRunner().invoke(cli.main, ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"]) + + assert result.exit_code != 0 + assert "baseline SHA-256 changed" in result.output + assert ServiceRegistry(tmp_path).get_service("vast-price-manager") is None + + +def test_engine_clone_preserves_full_settings_and_adds_only_candidate_config_mount(): + from cryptolabs_proxy.migration import build_recreate_request + + inspect = { + "Config": {"Image": "old@sha256:old", "Env": ["SECRET=not-printed"], "Labels": {"keep": "label"}, "Cmd": ["nginx"]}, + "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}, "PortBindings": {"443/tcp": [{"HostPort": "443"}]}, "LogConfig": {"Type": "json-file"}, "Binds": []}, + "Mounts": [ + {"Type": "volume", "Name": "fleet-auth-data", "Source": "/var/lib/docker/volumes/fleet-auth-data/_data", "Destination": "/data/auth", "RW": True}, + {"Type": "bind", "Source": "/etc/cryptolabs-proxy/ssl", "Destination": "/etc/nginx/ssl", "RW": False}, + ], + "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["cryptolabs-proxy"], "DriverOpts": {"keep": "value"}, "EndpointID": "engine-assigned", "IPAddress": "172.30.0.2"}}}, + } + + request = build_recreate_request(inspect, "new@sha256:reviewed", "/etc/cryptolabs-proxy/nginx.conf") + + assert request["Config"]["Image"] == "new@sha256:reviewed" + assert request["Config"]["Env"] == inspect["Config"]["Env"] + assert request["Config"]["Labels"] == inspect["Config"]["Labels"] + assert request["HostConfig"]["RestartPolicy"] == inspect["HostConfig"]["RestartPolicy"] + assert request["HostConfig"]["PortBindings"] == inspect["HostConfig"]["PortBindings"] + assert request["HostConfig"]["LogConfig"] == inspect["HostConfig"]["LogConfig"] + assert request["NetworkingConfig"]["EndpointsConfig"]["cryptolabs"]["Aliases"] == ["cryptolabs-proxy"] + assert "EndpointID" not in request["NetworkingConfig"]["EndpointsConfig"]["cryptolabs"] + assert "IPAddress" not in request["NetworkingConfig"]["EndpointsConfig"]["cryptolabs"] + mounts = request["HostConfig"]["Mounts"] + assert any(m["Type"] == "volume" and m["Source"] == "fleet-auth-data" and m["Target"] == "/data/auth" for m in mounts) + assert any(m["Type"] == "bind" and m["Source"] == "/etc/cryptolabs-proxy/nginx.conf" and m["Target"] == "/etc/nginx/nginx.conf" and m["ReadOnly"] is True for m in mounts) + + +@pytest.mark.parametrize("failure", ["stop", "rename", "create", "start", "health"]) +def test_switch_controller_compensates_each_failed_proxy_swap(failure): + from cryptolabs_proxy.migration import MigrationError, SwitchController + + class Engine: + def __init__(self): + self.calls = [] + self.failed_candidate_start = False + + def stop(self, name, ignore_missing=False): + self.calls.append(("stop", name, ignore_missing)) + if failure == "stop" and name == "cryptolabs-proxy": + raise RuntimeError("stop failed") + + def rename(self, old, new): + self.calls.append(("rename", old, new)) + if failure == "rename" and old == "cryptolabs-proxy": + raise RuntimeError("rename failed") + + def create(self, name, request): + self.calls.append(("create", name)) + if failure == "create": + raise RuntimeError("create failed") + + def start(self, name): + self.calls.append(("start", name)) + if failure == "start" and name == "cryptolabs-proxy" and not self.failed_candidate_start: + self.failed_candidate_start = True + raise RuntimeError("start failed") + + def remove(self, name, ignore_missing=False): + self.calls.append(("remove", name, ignore_missing)) + + def disconnect(self, name, network): + self.calls.append(("disconnect", name, network)) + + def connect(self, name, network, endpoint): + self.calls.append(("connect", name, network, endpoint)) + + engine = Engine() + controller = SwitchController(engine, health_check=lambda name: failure != "health") + + with pytest.raises(MigrationError): + controller.apply("cryptolabs-proxy", "test", {"candidate": True, "NetworkingConfig": {"EndpointsConfig": {}}}) + + if failure == "stop": + assert engine.calls == [("stop", "cryptolabs-proxy", False)] + else: + assert ("start", "cryptolabs-proxy") in engine.calls + if failure in {"create", "start", "health"}: + assert ("rename", "cryptolabs-proxy.rollback-test", "cryptolabs-proxy") in engine.calls + if failure == "start": + assert ("remove", "cryptolabs-proxy.candidate-test", True) in engine.calls + if failure == "health": + assert ("remove", "cryptolabs-proxy.candidate-test", True) not in engine.calls + + +def test_migration_plan_is_deterministic_and_exposes_no_environment_data(): + from cryptolabs_proxy.migration import MigrationPlan + + plan = MigrationPlan.from_source("container-id", CUSTOM_PROXY_CONFIG, "proxy@sha256:reviewed") + manifest = plan.sanitized_manifest() + + assert manifest["migration_id"] == MigrationPlan.from_source("container-id", CUSTOM_PROXY_CONFIG, "proxy@sha256:reviewed").migration_id + assert manifest["baseline_sha256"] == hashlib.sha256(CUSTOM_PROXY_CONFIG).hexdigest() + assert "Env" not in manifest + + +def test_candidate_validation_uses_nginx_entrypoint_and_always_removes_owned_container(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationError + + commands = [] + + class Result: + returncode = 0 + + def run(command, **kwargs): + commands.append(command) + if command[:3] == ["docker", "rm", "-f"]: + return Result() + return Result() + + monkeypatch.setattr("cryptolabs_proxy.migration.subprocess.run", run) + candidate = tmp_path / "nginx.conf" + candidate.write_bytes(CUSTOM_PROXY_CONFIG) + inspect = {"Mounts": [{"Destination": "/etc/nginx/ssl", "Source": "/ssl"}]} + + CustomConfigMigrator(tmp_path, tmp_path / "backups")._validate_candidate("proxy@sha256:reviewed", candidate, inspect, "migration-id") + + validation = commands[0] + assert validation[:3] == ["docker", "run", "--name"] + assert "--entrypoint" in validation + assert validation[validation.index("--entrypoint") + 1] == "nginx" + assert "--label" in validation + assert validation[-1:] == ["-t"] + assert commands[-1][:3] == ["docker", "rm", "-f"] + + +def test_candidate_validation_cleans_owned_container_when_nginx_fails(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationError + + commands = [] + + class Result: + def __init__(self, returncode): + self.returncode = returncode + + def run(command, **kwargs): + commands.append(command) + return Result(1 if command[:2] == ["docker", "run"] else 0) + + monkeypatch.setattr("cryptolabs_proxy.migration.subprocess.run", run) + candidate = tmp_path / "nginx.conf" + candidate.write_bytes(CUSTOM_PROXY_CONFIG) + inspect = {"Mounts": [{"Destination": "/etc/nginx/ssl", "Source": "/ssl"}]} + + with pytest.raises(MigrationError, match="validation failed"): + CustomConfigMigrator(tmp_path, tmp_path / "backups")._validate_candidate("proxy@sha256:reviewed", candidate, inspect, "migration-id") + + assert commands[-1][:3] == ["docker", "rm", "-f"] + + +def test_engine_clone_preserves_static_ip_and_rejects_tmpfs_mounts(): + from cryptolabs_proxy.migration import MigrationError, build_recreate_request + + inspect = { + "Config": {"Image": "old", "Env": ["SECRET=redacted"]}, + "HostConfig": {"PortBindings": {"80/tcp": [{"HostPort": "80"}]}}, + "Mounts": [], + "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["stable-alias"], "IPAMConfig": {"IPv4Address": "172.30.0.10"}}}}, + } + request = build_recreate_request(inspect, "new@sha256:reviewed", "/candidate.conf") + endpoint = request["NetworkingConfig"]["EndpointsConfig"]["cryptolabs"] + assert endpoint["Aliases"] == ["stable-alias"] + assert endpoint["IPAMConfig"] == {"IPv4Address": "172.30.0.10"} + assert request["HostConfig"]["PortBindings"] == inspect["HostConfig"]["PortBindings"] + + inspect["Mounts"] = [{"Type": "tmpfs", "Destination": "/run/cache", "RW": True}] + with pytest.raises(MigrationError, match="unsupported mount type"): + build_recreate_request(inspect, "new@sha256:reviewed", "/candidate.conf") + + +def test_startup_waits_through_starting_then_requires_unauthenticated_auth_401(): + from cryptolabs_proxy.migration import wait_for_proxy_ready + + states = iter([ + {"Running": True, "Health": {"Status": "starting"}}, + {"Running": True, "Health": {"Status": "healthy"}}, + ]) + + class Engine: + def inspect(self, container): + return {"State": next(states)} + + probes = [] + ready = wait_for_proxy_ready( + Engine(), + "candidate", + auth_probe=lambda: probes.append(True) or True, + timeout=2, + interval=0, + ) + + assert ready is True + assert probes == [True] + + +def test_startup_rejects_healthy_proxy_when_unauthenticated_auth_does_not_return_401(): + from cryptolabs_proxy.migration import wait_for_proxy_ready + + class Engine: + def inspect(self, container): + return {"State": {"Running": True, "Health": {"Status": "healthy"}}} + + assert wait_for_proxy_ready(Engine(), "candidate", auth_probe=lambda: False, timeout=0, interval=0) is False + + +def test_manual_rollback_keeps_and_restores_candidate_when_old_proxy_is_unhealthy(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationError + + migration_id = "migration" + backup = tmp_path / "backups" / migration_id + backup.mkdir(parents=True) + source = {"Id": "old-id", "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["old"], "IPAMConfig": {"IPv4Address": "172.30.0.10"}}}}} + (backup / "inspect.json").write_text(__import__("json").dumps(source)) + (backup / "registry-snapshot.json").write_text("{}") + + class Engine: + def __init__(self): + self.calls = [] + + def inspect(self, name): + if name == "cryptolabs-proxy.rollback-migration": + return source + return {"Id": "candidate-id", "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["candidate"], "IPAMConfig": {"IPv4Address": "172.30.0.10"}}}}} + + def stop(self, *args, **kwargs): self.calls.append(("stop", args[0])) + def disconnect(self, *args): self.calls.append(("disconnect", args[0], args[1])) + def rename(self, *args): self.calls.append(("rename", *args)) + def connect(self, *args): self.calls.append(("connect", args[0], args[1])) + def start(self, *args): self.calls.append(("start", args[0])) + def remove(self, *args, **kwargs): self.calls.append(("remove", args[0])) + + monkeypatch.setattr("cryptolabs_proxy.migration.os.geteuid", lambda: 0) + monkeypatch.setattr("cryptolabs_proxy.migration.wait_for_proxy_ready", lambda *args: True) + monkeypatch.setattr("cryptolabs_proxy.migration.wait_for_container_healthy", lambda *args: False) + engine = Engine() + + with pytest.raises(MigrationError, match="healthy replacement was restored"): + CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=engine).rollback("cryptolabs-proxy", migration_id) + + assert ("rename", "cryptolabs-proxy", "cryptolabs-proxy.candidate-migration") in engine.calls + assert ("rename", "cryptolabs-proxy.candidate-migration", "cryptolabs-proxy") in engine.calls + assert not any(call[0] == "remove" for call in engine.calls) + + +@pytest.mark.parametrize("failure", ["disconnect-second", "rename"]) +def test_switch_controller_reconnects_only_detached_source_endpoints_before_restart(failure): + from cryptolabs_proxy.migration import MigrationError, SwitchController + + endpoints = { + "cryptolabs": {"Aliases": ["proxy"], "IPAMConfig": {"IPv4Address": "172.30.0.10"}}, + "monitoring": {"Aliases": ["metrics"]}, + } + + class Engine: + def __init__(self): + self.calls = [] + + def stop(self, name, ignore_missing=False): self.calls.append(("stop", name)) + def start(self, name): self.calls.append(("start", name)) + def disconnect(self, name, network): + self.calls.append(("disconnect", name, network)) + if failure == "disconnect-second" and network == "monitoring": + raise RuntimeError("second disconnect failed") + def rename(self, old, new): + self.calls.append(("rename", old, new)) + if failure == "rename": + raise RuntimeError("rename failed") + def create(self, *args): raise AssertionError("candidate must not be created") + def connect(self, name, network, endpoint): self.calls.append(("connect", name, network, endpoint)) + + engine = Engine() + with pytest.raises(MigrationError): + SwitchController(engine, health_check=lambda _: True).apply( + "source-id", "repair", {"NetworkingConfig": {"EndpointsConfig": endpoints}}, "cryptolabs-proxy" + ) + + reconnects = [call for call in engine.calls if call[0] == "connect"] + expected_networks = ["cryptolabs"] if failure == "disconnect-second" else ["cryptolabs", "monitoring"] + assert [call[2] for call in reconnects] == expected_networks + assert [call[3] for call in reconnects] == [endpoints[network] for network in expected_networks] + assert engine.calls[-1] == ("start", "source-id") + + +def test_readiness_budget_exceeds_image_health_cadence_and_transport_timeout_is_separate(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import ( + CustomConfigMigrator, + PROXY_READY_TIMEOUT, + wait_for_proxy_ready, + ) + + assert PROXY_READY_TIMEOUT >= 110 # Docker start period + three intervals + probe allowance. + now = [0.0] + + class Engine: + def __init__(self): + self.states = iter([ + {"Running": True, "Health": {"Status": "starting"}}, + {"Running": True, "Health": {"Status": "starting"}}, + {"Running": True, "Health": {"Status": "healthy"}}, + ]) + + def inspect(self, container): + return {"State": next(self.states)} + + monkeypatch.setattr("cryptolabs_proxy.migration.time.monotonic", lambda: now[0]) + monkeypatch.setattr("cryptolabs_proxy.migration.time.sleep", lambda seconds: now.__setitem__(0, now[0] + seconds)) + assert wait_for_proxy_ready(Engine(), "candidate", lambda: True, timeout=PROXY_READY_TIMEOUT, interval=16) is True + + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", timeout=7) + assert migrator.engine.timeout == 7 + assert migrator.readiness_timeout == PROXY_READY_TIMEOUT + with pytest.raises(Exception, match="readiness timeout must be at least"): + CustomConfigMigrator(tmp_path, tmp_path / "backups", readiness_timeout=15) + + +def test_apply_uses_engine_health_polling_with_the_real_switch_controller(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationPlan + + image = "proxy@sha256:reviewed" + source = { + "Id": "source-id", + "Config": {"Image": "old"}, + "HostConfig": {"Binds": []}, + "Mounts": [], + "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["proxy"], "IPAMConfig": {"IPv4Address": "172.30.0.10"}}}}, + } + + class Engine: + def __init__(self): + self.calls = [] + self.inspect_calls = [] + + def inspect(self, name): + self.inspect_calls.append(name) + if name == "cryptolabs-proxy" and self.inspect_calls.count(name) > 1: + return {"State": {"Running": True, "Health": {"Status": "healthy"}}} + return source + + def stop(self, name, ignore_missing=False): self.calls.append(("stop", name)) + def disconnect(self, name, network): self.calls.append(("disconnect", name, network)) + def rename(self, old, new): self.calls.append(("rename", old, new)) + def create(self, name, request): self.calls.append(("create", name, request)) + def start(self, name): self.calls.append(("start", name)) + def connect(self, name, network, endpoint): self.calls.append(("connect", name, network, endpoint)) + def remove(self, name, ignore_missing=False): self.calls.append(("remove", name)) + + engine = Engine() + plan = MigrationPlan.from_source("source-id", CUSTOM_PROXY_CONFIG, image) + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=engine) + monkeypatch.setattr("cryptolabs_proxy.migration.os.geteuid", lambda: 0) + monkeypatch.setattr(migrator, "plan", lambda *args: plan) + monkeypatch.setattr(migrator, "_read_active_config", lambda *args: CUSTOM_PROXY_CONFIG) + monkeypatch.setattr(migrator, "_validate_candidate", lambda *args: None) + monkeypatch.setattr(migrator, "_anonymous_auth_is_rejected", lambda: True) + + outcome = migrator.apply("cryptolabs-proxy", image, plan.migration_id, enable_vpm=True) + + assert outcome.state == "applied" + assert any(call[:2] == ("create", "cryptolabs-proxy") for call in engine.calls) + assert "cryptolabs-proxy" in engine.inspect_calls + + +def test_apply_retains_registry_and_candidate_config_for_recovered_candidate(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationOutcome, MigrationPlan, SwitchController + from cryptolabs_proxy.services import ServiceRegistry + + image = "proxy@sha256:reviewed" + source = { + "Id": "source-id", + "Config": {"Image": "old"}, + "HostConfig": {"Binds": []}, + "Mounts": [], + "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["proxy"], "IPAMConfig": {"IPv4Address": "172.30.0.10"}}}}, + } + + class Engine: + def inspect(self, container): return source + + plan = MigrationPlan.from_source("source-id", CUSTOM_PROXY_CONFIG, image) + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=Engine()) + (tmp_path / "nginx.conf").write_bytes(b"old candidate file") + monkeypatch.setattr("cryptolabs_proxy.migration.os.geteuid", lambda: 0) + monkeypatch.setattr(migrator, "plan", lambda *args: plan) + monkeypatch.setattr(migrator, "_read_active_config", lambda *args: CUSTOM_PROXY_CONFIG) + monkeypatch.setattr(migrator, "_validate_candidate", lambda *args: None) + monkeypatch.setattr( + SwitchController, + "apply", + lambda *args, **kwargs: MigrationOutcome.candidate_retained("old proxy restoration was unhealthy"), + ) + + outcome = migrator.apply("cryptolabs-proxy", image, plan.migration_id, enable_vpm=True) + + assert outcome.state == "candidate_retained" + assert ServiceRegistry(tmp_path).get_service("vast-price-manager") is not None + assert b"BEGIN CRYPTOLABS MANAGED VPM" in (tmp_path / "nginx.conf").read_bytes() + marker = tmp_path / "backups" / plan.migration_id / "outcome.json" + assert marker.exists() + assert __import__("json").loads(marker.read_text())["state"] == "candidate_retained" + + +def test_apply_keeps_recovered_candidate_when_outcome_marker_cannot_be_written(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationOutcome, MigrationPlan, SwitchController + from cryptolabs_proxy.services import ServiceRegistry + + image = "proxy@sha256:reviewed" + source = { + "Id": "source-id", + "Config": {"Image": "old"}, + "HostConfig": {"Binds": []}, + "Mounts": [], + "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["proxy"]}}}, + } + + class Engine: + def inspect(self, container): return source + + plan = MigrationPlan.from_source("source-id", CUSTOM_PROXY_CONFIG, image) + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=Engine()) + original_write = migrator._write_private + + def write_private(path, content): + if path.name == "outcome.json": + raise OSError("backup marker is unavailable") + original_write(path, content) + + monkeypatch.setattr("cryptolabs_proxy.migration.os.geteuid", lambda: 0) + monkeypatch.setattr(migrator, "plan", lambda *args: plan) + monkeypatch.setattr(migrator, "_read_active_config", lambda *args: CUSTOM_PROXY_CONFIG) + monkeypatch.setattr(migrator, "_validate_candidate", lambda *args: None) + monkeypatch.setattr(migrator, "_write_private", write_private) + monkeypatch.setattr( + SwitchController, + "apply", + lambda *args, **kwargs: MigrationOutcome.candidate_retained("old proxy restoration was unhealthy"), + ) + + outcome = migrator.apply("cryptolabs-proxy", image, plan.migration_id, enable_vpm=True) + + assert outcome.state == "candidate_retained" + assert "marker could not be written" in outcome.detail + assert ServiceRegistry(tmp_path).get_service("vast-price-manager") is not None + assert b"BEGIN CRYPTOLABS MANAGED VPM" in (tmp_path / "nginx.conf").read_bytes() + + +def test_manual_rollback_uses_readiness_deadline_for_preflight_restoration_and_recovery(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationError + + migration_id = "deadline-migration" + backup = tmp_path / "backups" / migration_id + backup.mkdir(parents=True) + source = {"Id": "old-id", "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["old"]}}}} + (backup / "inspect.json").write_text(__import__("json").dumps(source)) + (backup / "registry-snapshot.json").write_text("{}") + + class Engine: + def inspect(self, name): + if name == "cryptolabs-proxy.rollback-deadline-migration": + return source + return {"Id": "candidate-id", "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["candidate"]}}}} + def stop(self, *args, **kwargs): pass + def disconnect(self, *args): pass + def rename(self, *args): pass + def connect(self, *args): pass + def start(self, *args): pass + def remove(self, *args, **kwargs): pass + + observed = [] + monkeypatch.setattr("cryptolabs_proxy.migration.os.geteuid", lambda: 0) + monkeypatch.setattr( + "cryptolabs_proxy.migration.wait_for_proxy_ready", + lambda engine, container, probe, timeout: observed.append(("ready", timeout)) or True, + ) + monkeypatch.setattr( + "cryptolabs_proxy.migration.wait_for_container_healthy", + lambda engine, container, timeout: observed.append(("healthy", timeout)) or False, + ) + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=Engine(), timeout=7) + + with pytest.raises(MigrationError, match="healthy replacement was restored"): + migrator.rollback("cryptolabs-proxy", migration_id) + + assert observed == [ + ("ready", migrator.readiness_timeout), + ("healthy", migrator.readiness_timeout), + ("ready", migrator.readiness_timeout), + ] + + +def test_migration_helper_reports_retained_candidate_as_operator_action(monkeypatch, capsys): + from cryptolabs_proxy.migration import MigrationOutcome + + spec = importlib.util.spec_from_file_location( + "vpm_custom_config_migrate", REPOSITORY / "scripts" / "vpm-custom-config-migrate.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + class Migrator: + def __init__(self, *args, **kwargs): pass + def apply(self, *args, **kwargs): + return MigrationOutcome.candidate_retained("source restoration was unhealthy") + + monkeypatch.setattr(module, "CustomConfigMigrator", Migrator) + monkeypatch.setattr( + "sys.argv", + ["vpm-custom-config-migrate.py", "apply", "--proxy-image", "proxy@sha256:reviewed", "--migration-id", "repair"], + ) + + assert module.main() == 2 + assert __import__("json").loads(capsys.readouterr().out)["state"] == "candidate_retained" + + +def test_manual_rollback_blocks_lifecycle_command_until_runtime_and_registry_restore_finish(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator + + migration_id = "lock-migration" + backup = tmp_path / "backups" / migration_id + backup.mkdir(parents=True) + source = {"Id": "old-id", "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["old"]}}}} + (backup / "inspect.json").write_text(__import__("json").dumps(source)) + (backup / "registry-snapshot.json").write_text("{}") + in_switch = threading.Event() + release_switch = threading.Event() + errors = [] + + class Engine: + def inspect(self, name): + if name == "cryptolabs-proxy.rollback-lock-migration": + return source + return {"Id": "candidate-id", "NetworkSettings": {"Networks": {"cryptolabs": {"Aliases": ["candidate"]}}}} + def stop(self, *args, **kwargs): + in_switch.set() + assert release_switch.wait(5) + def disconnect(self, *args): pass + def rename(self, *args): pass + def connect(self, *args): pass + def start(self, *args): pass + def remove(self, *args, **kwargs): pass + + monkeypatch.setattr("cryptolabs_proxy.migration.os.geteuid", lambda: 0) + monkeypatch.setattr("cryptolabs_proxy.migration.wait_for_proxy_ready", lambda *args: True) + monkeypatch.setattr("cryptolabs_proxy.migration.wait_for_container_healthy", lambda *args: True) + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=Engine()) + rollback_thread = threading.Thread( + target=lambda: _rollback_in_thread(migrator, migration_id, errors), + daemon=True, + ) + rollback_thread.start() + assert in_switch.wait(2) + + context = get_context("fork") + entered = context.Event() + results = context.Queue() + process = context.Process(target=_register_service_after_rollback_lock, args=(str(tmp_path), entered, results)) + process.start() + assert not entered.wait(0.4) + + release_switch.set() + rollback_thread.join(5) + process.join(5) + assert not errors + assert process.exitcode == 0 + assert entered.is_set() + assert results.get(timeout=2)[0] == 0 + + +def _rollback_in_thread(migrator, migration_id, errors): + try: + migrator.rollback("cryptolabs-proxy", migration_id) + except Exception as error: # pragma: no cover - asserted by caller + errors.append(error) From 614f4b70589c0109f3ed1871f5e83fc8836e4624 Mon Sep 17 00:00:00 2001 From: Hannes Zietsman Date: Sat, 12 Sep 2026 18:33:28 +0200 Subject: [PATCH 2/4] Share Fleet login with Vast Price Manager and preserve custom proxy routes --- README.md | 27 ++- landing-page/index.html | 2 +- src/cryptolabs_proxy/auth.py | 106 +++++++++++- src/cryptolabs_proxy/config.py | 5 + src/cryptolabs_proxy/custom_config.py | 101 +++++++++++- src/cryptolabs_proxy/migration.py | 21 ++- src/cryptolabs_proxy/templates/nginx.conf.j2 | 2 + .../vast-price-manager-auth-blocks.conf.j2 | 7 + .../templates/vast-price-manager.conf.j2 | 11 +- tests/test_offline_image.py | 96 +++++++++++ tests/test_routes.py | 154 ++++++++++++++++++ tests/test_vast_price_manager.py | 81 ++++++++- 12 files changed, 586 insertions(+), 27 deletions(-) create mode 100644 src/cryptolabs_proxy/templates/vast-price-manager-auth-blocks.conf.j2 create mode 100644 tests/test_offline_image.py diff --git a/README.md b/README.md index d420a58..0c79c24 100644 --- a/README.md +++ b/README.md @@ -145,9 +145,32 @@ unmounted `nginx.conf`: it regenerates the full configuration. Use the reviewed `scripts/vpm-custom-config-migrate.py` helper first. Its read-only `plan` mode derives a migration ID from the source container and config hash. Its explicit `apply` stores root-only backups, mounts a byte-preserved custom baseline with -only the canonical managed VPM block added, and can perform a proxy-only +only the canonical managed blocks added, and can perform a proxy-only `rollback`. Custom-config mode then permits only normal VPM -`register`/`unregister`; it rejects changes to every other service. +`register`/`unregister`; it rejects changes to every other service. Its managed +Fleet auth deny block remains installed in every HTTP server block after VPM is unregistered, so the +always-running internal VPM session and reauthentication endpoints cannot fall +through a preserved public /auth/ route. Rollback restores the +original stored baseline exactly. + +### Vast Price Manager Fleet sign-in + +When Vast Price Manager runs in Fleet mode, it uses the existing Fleet +`fleet_session` cookie and does not create a second VPM account or login. VPM +introspects the current Fleet session on the Docker network at +`http://cryptolabs-proxy:8081/auth/vast-price-manager/session`; the public +Nginx configuration explicitly returns `404` for that endpoint and its +reauthentication companion, including their trailing-slash variants. + +The internal session response is limited to the current enabled Fleet admin's +username, role, and two purpose-separated HMAC values derived from the signed +cookie: a subject and a CSRF token. It never returns the cookie, password, +password hash, or signing key. The proxy rereads the user record for every +session, reauthentication, and VPM proxy authorization check, so a disabled, +deleted, demoted, or password-change-required account loses VPM access +immediately. Sensitive VPM confirmation calls +`POST /auth/vast-price-manager/reauth` internally with the Fleet password and +the returned CSRF token; it uses the normal Fleet password throttle. User authentication data is stored in `/data/auth/`: diff --git a/landing-page/index.html b/landing-page/index.html index e72a428..08ee580 100644 --- a/landing-page/index.html +++ b/landing-page/index.html @@ -842,7 +842,7 @@

⚙️ System Updates

'vast-price-manager': { displayName: 'Vast Price Manager', icon: '💰', - description: 'Optional pricing-management service for Vast. It keeps its own login, CSRF protection, and password reauthentication.', + description: 'Manage Vast pricing with your existing Fleet login.', path: '/vast-pricing/', productUrl: 'https://github.com/cryptolabsza/vast-price-manager', isCryptoLabs: true, diff --git a/src/cryptolabs_proxy/auth.py b/src/cryptolabs_proxy/auth.py index 5612abd..daf704d 100644 --- a/src/cryptolabs_proxy/auth.py +++ b/src/cryptolabs_proxy/auth.py @@ -1121,6 +1121,55 @@ def decorated(*args, **kwargs): return f(*args, **kwargs) return decorated + def current_vast_price_manager_user(): + """Return the live Fleet administrator behind this signed session. + + Flask has already verified ``fleet_session`` before exposing ``session``. + The database lookup is deliberately repeated for every VPM request so a + role change, disablement, deletion, or forced password change takes + effect immediately instead of trusting values cached in the cookie. + """ + if not session.get('logged_in') or session.get('require_password_change'): + return None, 401 + + username = session.get('username') + if not isinstance(username, str) or not username: + return None, 401 + + user = get_user(username) + if not user or not user.get('enabled', True) or user.get('role') != 'admin': + return None, 403 + if user.get('require_password_change', False): + return None, 401 + return user, 200 + + def vast_price_manager_session_payload(): + """Build the bounded VPM authority response from the signed cookie.""" + user, status = current_vast_price_manager_user() + if not user: + return None, status + + signed_cookie = request.cookies.get(app.config['SESSION_COOKIE_NAME']) + if not signed_cookie: + return None, 401 + + cookie_bytes = signed_cookie.encode('utf-8') + + def derive(purpose: bytes) -> str: + return hmac.new( + AUTH_SECRET_KEY.encode('utf-8'), purpose + cookie_bytes, hashlib.sha256 + ).hexdigest() + + return { + 'authenticated': True, + 'username': user['username'], + 'role': 'admin', + # Purpose separation prevents either stable value from standing in + # for the other while avoiding disclosure of the signed cookie. + 'subject': derive(b'cryptolabs/vpm/subject/v1:'), + 'csrf_token': derive(b'cryptolabs/vpm/csrf/v1:'), + }, 200 + # ========================================================================= # ROUTES # ========================================================================= @@ -1451,11 +1500,60 @@ def authorize_vast_price_manager(): subrequest. It reads the signed-in Flask session and deliberately ignores incoming role headers, which a client could forge. """ - if not session.get('logged_in') or session.get('require_password_change'): - return '', 401 - if session.get('role') != 'admin': - return '', 403 + _, status = current_vast_price_manager_user() + if status != 200: + return '', status return '', 204 + + @app.route('/auth/vast-price-manager/session') + def vast_price_manager_session(): + """Expose current Fleet authority only to VPM's Docker-network client.""" + payload, status = vast_price_manager_session_payload() + if not payload: + return '', status + return jsonify(payload) + + @app.route('/auth/vast-price-manager/reauth', methods=['POST']) + def reauthenticate_vast_price_manager(): + """Recheck the current Fleet password for one sensitive VPM action.""" + payload, status = vast_price_manager_session_payload() + if not payload: + return '', status + + data = request.get_json(silent=True) + if not isinstance(data, dict): + return '', 400 + password = data.get('password') + csrf_token = data.get('csrf_token') + if ( + not isinstance(password, str) + or not isinstance(csrf_token, str) + or len(password) > 4096 + or len(csrf_token) != 64 + ): + return '', 400 + if not hmac.compare_digest(csrf_token, payload['csrf_token']): + return '', 403 + + verified = verify_user(payload['username'], password) + if not verified: + # ``verify_user`` records failures and applies the existing Fleet + # lockout policy. Surface a rate limit only once it is observable. + current = get_user(payload['username']) + try: + locked = current and current.get('locked_until') and ( + datetime.fromisoformat(current['locked_until']) > datetime.utcnow() + ) + except (TypeError, ValueError): + locked = False + return '', 429 if locked else 401 + + # Re-read after password verification so a concurrent state change + # cannot turn this endpoint into a stale authority grant. + payload, status = vast_price_manager_session_payload() + if not payload: + return '', status + return jsonify(payload) # API endpoints for programmatic access @app.route('/auth/api/users', methods=['GET']) diff --git a/src/cryptolabs_proxy/config.py b/src/cryptolabs_proxy/config.py index 3d6ab83..9447f04 100644 --- a/src/cryptolabs_proxy/config.py +++ b/src/cryptolabs_proxy/config.py @@ -36,6 +36,11 @@ def render_vpm_fragment() -> str: return get_jinja_env().get_template("vast-price-manager.conf.j2").render() +def render_vpm_internal_auth_blocks() -> str: + """Render public-Nginx blocks for Docker-only VPM Fleet auth endpoints.""" + return get_jinja_env().get_template("vast-price-manager-auth-blocks.conf.j2").render() + + def generate_docker_compose(config_dir: Path, domain: str = None, use_letsencrypt: bool = False): """Generate docker-compose.yml from template.""" env = get_jinja_env() diff --git a/src/cryptolabs_proxy/custom_config.py b/src/cryptolabs_proxy/custom_config.py index c815feb..271313e 100644 --- a/src/cryptolabs_proxy/custom_config.py +++ b/src/cryptolabs_proxy/custom_config.py @@ -6,14 +6,18 @@ import re from pathlib import Path -from .config import render_vpm_fragment +from .config import render_vpm_fragment, render_vpm_internal_auth_blocks CUSTOM_CONFIG_KEY = "custom_config" CUSTOM_CONFIG_MODE = "vpm-managed" MANAGED_BEGIN = b"# BEGIN CRYPTOLABS MANAGED VPM" MANAGED_END = b"# END CRYPTOLABS MANAGED VPM" +AUTH_DENIES_BEGIN = b"# BEGIN CRYPTOLABS MANAGED INTERNAL AUTH DENIES" +AUTH_DENIES_END = b"# END CRYPTOLABS MANAGED INTERNAL AUTH DENIES" _ANCHOR = re.compile(rb"^[ \t]*location[ \t]+@login_redirect[ \t]*\{") +_HTTP = re.compile(rb"^[ \t]*http[ \t]*\{") +_SERVER = re.compile(rb"^[ \t]*server[ \t]*\{") class CustomConfigError(RuntimeError): @@ -73,6 +77,29 @@ def _strip_comments_and_strings(line: bytes) -> bytes: return bytes(result) +def _structural_brace_events(line: bytes) -> list[tuple[int, int]]: + """Return brace positions that are outside quoted strings and comments.""" + events = [] + quote = None + escaped = False + for index, char in enumerate(line): + if quote: + if escaped: + escaped = False + elif char == ord("\\"): + escaped = True + elif char == quote: + quote = None + continue + if char in (ord("'"), ord('"')): + quote = char + elif char == ord("#"): + break + elif char in (ord("{"), ord("}")): + events.append((index, char)) + return events + + def _anchor_end_offset(baseline: bytes) -> int: lines = baseline.splitlines(keepends=True) anchors = [index for index, line in enumerate(lines) if _ANCHOR.match(line)] @@ -95,18 +122,74 @@ def _anchor_end_offset(baseline: bytes) -> int: raise CustomConfigError("custom-config login redirect anchor has unbalanced Nginx braces") +def _http_server_end_offsets(baseline: bytes) -> list[int]: + """Locate every HTTP server closing brace without parsing quoted braces.""" + depth = 0 + http_depth = None + server_parents = [] + offsets = [] + offset = 0 + + for line in baseline.splitlines(keepends=True): + events = _structural_brace_events(line) + http_open = None + server_open = None + if http_depth is None and _HTTP.match(line): + http_open = next((index for index, char in events if char == ord("{")), None) + elif http_depth is not None and depth == http_depth + 1 and _SERVER.match(line): + server_open = next((index for index, char in events if char == ord("{")), None) + + for index, char in events: + if char == ord("{"): + if index == http_open: + http_depth = depth + if index == server_open: + server_parents.append(depth) + depth += 1 + else: + depth -= 1 + if server_parents and depth == server_parents[-1]: + server_parents.pop() + offsets.append(offset + index) + if http_depth is not None and depth == http_depth: + http_depth = None + offset += len(line) + + if http_depth is not None or server_parents: + raise CustomConfigError("custom-config HTTP server blocks have unbalanced Nginx braces") + if not offsets: + raise CustomConfigError("custom-config baseline must contain at least one HTTP server block") + return offsets + + def render_managed_vpm_config(baseline: bytes, enabled: bool) -> bytes: - """Return baseline bytes unchanged or splice the reviewed fragment after its anchor.""" - if not enabled: - return baseline - if MANAGED_BEGIN in baseline or MANAGED_END in baseline: + """Splice auth denies and, when enabled, the VPM route after the anchor. + + The auth server is always installed, so its VPM-only authority endpoints + must remain private even after VPM itself is unregistered. The source + baseline is never changed: each render starts from its recorded bytes, + which keeps a migration rollback byte-exact. + """ + if any(marker in baseline for marker in ( + MANAGED_BEGIN, MANAGED_END, AUTH_DENIES_BEGIN, AUTH_DENIES_END, + )): raise CustomConfigError("custom-config baseline already contains a managed VPM delimiter") if b"upstream auth_server" not in baseline or b"location @service_unavailable" not in baseline: raise CustomConfigError("custom-config baseline lacks required Fleet auth or service-unavailable handlers") - offset = _anchor_end_offset(baseline) - fragment = render_vpm_fragment().encode() - block = MANAGED_BEGIN + b"\n" + fragment.rstrip() + b"\n" + MANAGED_END + b"\n" - return baseline[:offset] + block + baseline[offset:] + anchor_offset = _anchor_end_offset(baseline) + server_end_offsets = _http_server_end_offsets(baseline) + auth_denies = render_vpm_internal_auth_blocks().encode().rstrip() + auth_block = b"\n" + AUTH_DENIES_BEGIN + b"\n" + auth_denies + b"\n" + AUTH_DENIES_END + b"\n" + insertions = [(offset, auth_block) for offset in server_end_offsets] + if enabled: + fragment = render_vpm_fragment().encode().rstrip() + vpm_block = MANAGED_BEGIN + b"\n" + fragment + b"\n" + MANAGED_END + b"\n" + insertions.append((anchor_offset, vpm_block)) + + rendered = baseline + for offset, block in sorted(insertions, reverse=True): + rendered = rendered[:offset] + block + rendered[offset:] + return rendered def ensure_vpm_only_change(previous: dict, requested: dict) -> None: diff --git a/src/cryptolabs_proxy/migration.py b/src/cryptolabs_proxy/migration.py index 7b7e62c..d02454c 100644 --- a/src/cryptolabs_proxy/migration.py +++ b/src/cryptolabs_proxy/migration.py @@ -14,6 +14,7 @@ import json import os from pathlib import Path +import re import socket import subprocess import time @@ -23,6 +24,9 @@ from .custom_config import configure_custom_config_mode, render_managed_vpm_config +_LOCAL_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$") + + class MigrationError(RuntimeError): """A custom-config migration precondition or compensated switch failed.""" @@ -395,7 +399,7 @@ def _read_active_config(self, container: str) -> bytes: def plan(self, container: str, image: str) -> MigrationPlan: """Read only: return a deterministic, sanitized migration identity.""" - if "@sha256:" not in image: + if "@sha256:" not in image and _LOCAL_IMAGE_ID.fullmatch(image) is None: raise MigrationError("proxy image must be an immutable sha256 digest") inspect = self.engine.inspect(container) return MigrationPlan.from_source(inspect["Id"], self._read_active_config(container), image) @@ -409,6 +413,21 @@ def _write_private(path: Path, content: bytes): path.chmod(0o600) def _validate_candidate(self, image: str, candidate: Path, inspect: dict[str, Any], migration_id: str): + if _LOCAL_IMAGE_ID.fullmatch(image) is not None: + # Docker otherwise treats an absent sha256: argument as a + # repository/tag candidate for `docker run`. Verify the exact + # locally loaded content ID before creating any validator. + try: + local = subprocess.run( + ["docker", "image", "inspect", "--format", "{{.Id}}", image], + capture_output=True, + text=True, + timeout=self.timeout, + ) + except (OSError, subprocess.SubprocessError) as error: + raise MigrationError(f"offline image missing: exact local immutable image ID is unavailable ({error})") from error + if local.returncode != 0 or local.stdout.strip() != image: + raise MigrationError("offline image missing: exact local immutable image ID is unavailable") ssl_mount = next((m for m in inspect.get("Mounts", []) if m.get("Destination") == "/etc/nginx/ssl"), None) if not ssl_mount: raise MigrationError("source proxy has no /etc/nginx/ssl mount for candidate validation") diff --git a/src/cryptolabs_proxy/templates/nginx.conf.j2 b/src/cryptolabs_proxy/templates/nginx.conf.j2 index b3d5344..fa001fc 100644 --- a/src/cryptolabs_proxy/templates/nginx.conf.j2 +++ b/src/cryptolabs_proxy/templates/nginx.conf.j2 @@ -158,6 +158,8 @@ http { return 302 /auth/login?next=$request_uri; } + {% include "vast-price-manager-auth-blocks.conf.j2" %} + {% if 'vast-price-manager' in services %} {% include "vast-price-manager.conf.j2" %} {% endif %} diff --git a/src/cryptolabs_proxy/templates/vast-price-manager-auth-blocks.conf.j2 b/src/cryptolabs_proxy/templates/vast-price-manager-auth-blocks.conf.j2 new file mode 100644 index 0000000..5c4653c --- /dev/null +++ b/src/cryptolabs_proxy/templates/vast-price-manager-auth-blocks.conf.j2 @@ -0,0 +1,7 @@ + # VPM calls these endpoints only over the Docker network on port 8081. + # Exact locations keep the generic public /auth/ forwarding route from + # exposing the session authority or password reauthentication API. + location = /auth/vast-price-manager/session { return 404; } + location = /auth/vast-price-manager/session/ { return 404; } + location = /auth/vast-price-manager/reauth { return 404; } + location = /auth/vast-price-manager/reauth/ { return 404; } diff --git a/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 b/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 index c4d2ef6..bee6989 100644 --- a/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 +++ b/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 @@ -27,12 +27,17 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Proto $scheme; - # VPM retains its own login. Never pass client-supplied Fleet - # identity headers to it. + # VPM directly introspects the signed Fleet session on its + # configured internal origin. Never pass client identity headers. proxy_set_header X-Fleet-Auth-User ""; proxy_set_header X-Fleet-Auth-Role ""; proxy_set_header X-Fleet-Auth-Token ""; proxy_set_header X-Fleet-Authenticated ""; - proxy_intercept_errors on; + # Preserve the browser Fleet cookie for VPM's direct introspection. + proxy_set_header Cookie $http_cookie; + # Auth-request failures still use the redirect above. Do not + # intercept VPM's own 401 reauthentication result, which must + # reach its caller as an error rather than a Fleet login redirect. + proxy_intercept_errors off; error_page 502 503 504 = @service_unavailable; } diff --git a/tests/test_offline_image.py b/tests/test_offline_image.py new file mode 100644 index 0000000..cd1f7d5 --- /dev/null +++ b/tests/test_offline_image.py @@ -0,0 +1,96 @@ +"""Offline immutable local image-ID coverage for custom proxy migration.""" + +from pathlib import Path + +import pytest + +from cryptolabs_proxy.migration import ( + CustomConfigMigrator, + MigrationError, + build_recreate_request, +) + + +LOCAL_ID = "sha256:" + "b" * 64 + + +def test_migration_plan_accepts_full_canonical_local_image_id(monkeypatch, tmp_path: Path): + class Engine: + def inspect(self, _container): + return {"Id": "source-id"} + + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=Engine()) + monkeypatch.setattr(migrator, "_read_active_config", lambda _container: b"baseline") + + assert migrator.plan("cryptolabs-proxy", LOCAL_ID).image == LOCAL_ID + + +@pytest.mark.parametrize("image", ["sha256:" + "a" * 63, "sha256:" + "A" * 64]) +def test_migration_plan_rejects_truncated_or_noncanonical_local_image_ids(image, tmp_path: Path): + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=object()) + + with pytest.raises(MigrationError, match="immutable"): + migrator.plan("cryptolabs-proxy", image) + + +def test_candidate_validation_and_create_request_keep_exact_local_image_id(monkeypatch, tmp_path: Path): + commands = [] + + class Result: + returncode = 0 + + def run(command, **_kwargs): + commands.append(command) + if command[:4] == ["docker", "image", "inspect", "--format"]: + return type("InspectResult", (), {"returncode": 0, "stdout": f"{LOCAL_ID}\n", "stderr": ""})() + return Result() + + monkeypatch.setattr("cryptolabs_proxy.migration.subprocess.run", run) + candidate = tmp_path / "nginx.conf" + candidate.write_text("events {}") + inspect = {"Mounts": [{"Destination": "/etc/nginx/ssl", "Source": "/ssl"}]} + + CustomConfigMigrator(tmp_path, tmp_path / "backups")._validate_candidate( + LOCAL_ID, candidate, inspect, "offline-id" + ) + request = build_recreate_request( + {"Config": {}, "HostConfig": {}, "Mounts": [], "NetworkSettings": {"Networks": {"cryptolabs": {}}}}, + LOCAL_ID, + str(candidate), + ) + + assert commands[0] == ["docker", "image", "inspect", "--format", "{{.Id}}", LOCAL_ID] + assert LOCAL_ID in commands[1] + assert request["Config"]["Image"] == LOCAL_ID + + +@pytest.mark.parametrize( + "inspect_result", + [ + (1, "", "No such image"), + (0, "sha256:" + "c" * 64 + "\n", ""), + ], +) +def test_local_image_id_missing_or_mismatched_rejects_before_candidate_validator_runs(monkeypatch, tmp_path: Path, inspect_result): + commands = [] + returncode, stdout, stderr = inspect_result + + class Result: + def __init__(self, returncode, stdout="", stderr=""): + self.returncode, self.stdout, self.stderr = returncode, stdout, stderr + + def run(command, **_kwargs): + commands.append(command) + return Result(returncode, stdout, stderr) + + monkeypatch.setattr("cryptolabs_proxy.migration.subprocess.run", run) + candidate = tmp_path / "nginx.conf" + candidate.write_text("events {}") + inspect = {"Mounts": [{"Destination": "/etc/nginx/ssl", "Source": "/ssl"}]} + + with pytest.raises(MigrationError, match="offline image missing"): + CustomConfigMigrator(tmp_path, tmp_path / "backups")._validate_candidate( + LOCAL_ID, candidate, inspect, "offline-id" + ) + + assert commands == [["docker", "image", "inspect", "--format", "{{.Id}}", LOCAL_ID]] diff --git a/tests/test_routes.py b/tests/test_routes.py index 3f88a23..9812ba4 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -1,6 +1,9 @@ """Tests for Flask routes: login/logout, user management, settings, API endpoints.""" +import hashlib +import hmac import json +from datetime import timedelta import pytest @@ -111,6 +114,157 @@ def test_logout_clears_session(self, logged_in_admin): assert 'logged_in' not in sess +# --------------------------------------------------------------------------- +# Vast Price Manager Fleet session authority +# --------------------------------------------------------------------------- + +class TestVastPriceManagerFleetSession: + def _login(self, client, admin_user): + response = client.post('/auth/login', data={ + 'username': admin_user['username'], + 'password': admin_user['password'], + }) + assert response.status_code == 302 + + def _vpm_session(self, client): + """Model VPM's direct call without forwarding Fleet's Set-Cookie back.""" + incoming_cookie = client.get_cookie('fleet_session') + response = client.get('/auth/vast-price-manager/session') + if incoming_cookie: + client.set_cookie('fleet_session', incoming_cookie.value) + return response + + def test_session_uses_real_fleet_cookie_and_returns_only_contract_fields(self, client, admin_user): + import cryptolabs_proxy.auth as auth + + self._login(client, admin_user) + signed_cookie = client.get_cookie('fleet_session').value.encode('utf-8') + + response = self._vpm_session(client) + + assert response.status_code == 200 + assert response.get_json() == { + 'authenticated': True, + 'username': admin_user['username'], + 'role': 'admin', + 'subject': response.get_json()['subject'], + 'csrf_token': response.get_json()['csrf_token'], + } + payload = response.get_json() + assert len(payload['subject']) == 64 + assert len(payload['csrf_token']) == 64 + assert payload['subject'] != payload['csrf_token'] + assert all(character in '0123456789abcdef' for character in payload['subject']) + assert all(character in '0123456789abcdef' for character in payload['csrf_token']) + assert payload['subject'] == hmac.new( + auth.AUTH_SECRET_KEY.encode('utf-8'), + b'cryptolabs/vpm/subject/v1:' + signed_cookie, + hashlib.sha256, + ).hexdigest() + assert payload['csrf_token'] == hmac.new( + auth.AUTH_SECRET_KEY.encode('utf-8'), + b'cryptolabs/vpm/csrf/v1:' + signed_cookie, + hashlib.sha256, + ).hexdigest() + assert b'fleet_session=' not in response.data + assert b'password' not in response.data + + def test_session_rechecks_current_user_state_instead_of_cached_session_role(self, client, admin_user): + import cryptolabs_proxy.auth as auth + + self._login(client, admin_user) + auth.update_user(admin_user['username'], role='readonly') + + assert client.get('/auth/vast-price-manager/session').status_code == 403 + assert client.get('/auth/vast-price-manager/authorize').status_code == 403 + + @pytest.mark.parametrize('change', [ + lambda auth, username: auth.update_user(username, enabled=False), + lambda auth, username: auth.update_user(username, require_password_change=True), + lambda auth, username: auth.delete_user(username), + ]) + def test_session_rejects_disabled_forced_change_or_deleted_user(self, client, admin_user, change): + import cryptolabs_proxy.auth as auth + + # Keep a second admin so the delete operation is valid. + auth.create_user('other-admin', 'otherpass', role='admin') + self._login(client, admin_user) + change(auth, admin_user['username']) + + expected = 401 if auth.get_user(admin_user['username']) and auth.get_user(admin_user['username']).get('require_password_change') else 403 + assert client.get('/auth/vast-price-manager/session').status_code == expected + + def test_session_requires_a_signed_logged_in_fleet_session(self, client, admin_user): + assert client.get('/auth/vast-price-manager/session').status_code == 401 + client.set_cookie('fleet_session', 'forged-session') + assert client.get('/auth/vast-price-manager/session').status_code == 401 + + def test_session_rejects_an_expired_signed_fleet_session(self, client, admin_user): + self._login(client, admin_user) + client.application.config['PERMANENT_SESSION_LIFETIME'] = timedelta(seconds=-1) + + assert client.get('/auth/vast-price-manager/session').status_code == 401 + + def test_reauth_validates_csrf_and_current_fleet_password(self, client, admin_user): + self._login(client, admin_user) + session = self._vpm_session(client).get_json() + + invalid_csrf = client.post('/auth/vast-price-manager/reauth', json={ + 'password': admin_user['password'], 'csrf_token': '0' * 64, + }) + assert invalid_csrf.status_code == 403 + + wrong_password = client.post('/auth/vast-price-manager/reauth', json={ + 'password': 'incorrect', 'csrf_token': session['csrf_token'], + }) + assert wrong_password.status_code == 401 + + session = self._vpm_session(client).get_json() + valid = client.post('/auth/vast-price-manager/reauth', json={ + 'password': admin_user['password'], 'csrf_token': session['csrf_token'], + }) + assert valid.status_code == 200 + assert valid.get_json() == session + + def test_reauth_rejects_unbounded_or_non_json_input(self, client, admin_user): + self._login(client, admin_user) + session = client.get('/auth/vast-price-manager/session').get_json() + + assert client.post('/auth/vast-price-manager/reauth', data='not-json').status_code == 400 + assert client.post('/auth/vast-price-manager/reauth', json={ + 'password': 'x' * 4097, 'csrf_token': session['csrf_token'], + }).status_code == 400 + + def test_reauth_reports_detectable_password_throttling(self, client, admin_user): + import cryptolabs_proxy.auth as auth + + auth.save_settings({'max_login_attempts': 3, 'lockout_duration_minutes': 15}) + self._login(client, admin_user) + for _ in range(2): + csrf_token = self._vpm_session(client).get_json()['csrf_token'] + assert client.post('/auth/vast-price-manager/reauth', json={ + 'password': 'incorrect', 'csrf_token': csrf_token, + }).status_code == 401 + + csrf_token = self._vpm_session(client).get_json()['csrf_token'] + limited = client.post('/auth/vast-price-manager/reauth', json={ + 'password': 'incorrect', 'csrf_token': csrf_token, + }) + assert limited.status_code == 429 + + def test_reauth_rechecks_live_user_state(self, client, admin_user): + import cryptolabs_proxy.auth as auth + + self._login(client, admin_user) + csrf_token = self._vpm_session(client).get_json()['csrf_token'] + auth.update_user(admin_user['username'], enabled=False) + + response = client.post('/auth/vast-price-manager/reauth', json={ + 'password': admin_user['password'], 'csrf_token': csrf_token, + }) + assert response.status_code == 403 + + # --------------------------------------------------------------------------- # Change password route # --------------------------------------------------------------------------- diff --git a/tests/test_vast_price_manager.py b/tests/test_vast_price_manager.py index ab9d45e..71bfda4 100644 --- a/tests/test_vast_price_manager.py +++ b/tests/test_vast_price_manager.py @@ -81,6 +81,8 @@ def test_rendered_vpm_route_authorizes_session_admin_and_preserves_uri(tmp_path) config = (tmp_path / "nginx.conf").read_text() assert "location = /vast-pricing {" not in config + assert 'location = /auth/vast-price-manager/session { return 404; }' in config + assert 'location = /auth/vast-price-manager/reauth { return 404; }' in config generate_nginx_config( tmp_path, @@ -100,6 +102,13 @@ def test_rendered_vpm_route_authorizes_session_admin_and_preserves_uri(tmp_path) assert "proxy_set_header X-Forwarded-Host $host;" in vpm_route assert "proxy_set_header X-Forwarded-Proto $scheme;" in vpm_route assert 'proxy_set_header X-Fleet-Auth-Role "";' in vpm_route + assert 'location = /auth/vast-price-manager/session { return 404; }' in config + assert 'location = /auth/vast-price-manager/session/ { return 404; }' in config + assert 'location = /auth/vast-price-manager/reauth { return 404; }' in config + assert 'location = /auth/vast-price-manager/reauth/ { return 404; }' in config + assert 'proxy_set_header Cookie $http_cookie;' in vpm_route + assert 'proxy_intercept_errors off;' in vpm_route + assert 'error_page 502 503 504 = @service_unavailable;' in vpm_route def test_cli_register_unregister_toggles_generated_vpm_route_and_preserves_metadata(monkeypatch, tmp_path): @@ -342,18 +351,56 @@ def test_vpm_lifecycle_actions_are_rejected_by_generic_update_api(): def test_custom_config_render_preserves_baseline_bytes_and_uses_canonical_fragment(tmp_path): from cryptolabs_proxy.custom_config import render_managed_vpm_config - from cryptolabs_proxy.config import render_vpm_fragment + from cryptolabs_proxy.config import render_vpm_fragment, render_vpm_internal_auth_blocks rendered = render_managed_vpm_config(CUSTOM_PROXY_CONFIG, enabled=True) + auth_denies = render_vpm_internal_auth_blocks().encode().rstrip() + managed_auth_block = ( + b"\n# BEGIN CRYPTOLABS MANAGED INTERNAL AUTH DENIES\n" + + auth_denies + + b"\n# END CRYPTOLABS MANAGED INTERNAL AUTH DENIES\n" + ) + managed_vpm_block = ( + b"# BEGIN CRYPTOLABS MANAGED VPM\n" + + render_vpm_fragment().encode().rstrip() + + b"\n# END CRYPTOLABS MANAGED VPM\n" + ) assert b'# BEGIN CRYPTOLABS MANAGED VPM' in rendered assert render_vpm_fragment().encode() in rendered - before, managed = rendered.split(b'# BEGIN CRYPTOLABS MANAGED VPM', 1) - managed, after = managed.split(b'# END CRYPTOLABS MANAGED VPM', 1) - assert before == CUSTOM_PROXY_CONFIG[:len(before)] - assert after == CUSTOM_PROXY_CONFIG[len(CUSTOM_PROXY_CONFIG) - len(after):] + assert rendered.count(b'# BEGIN CRYPTOLABS MANAGED INTERNAL AUTH DENIES') == 3 + assert rendered.replace(managed_vpm_block, b'').replace(managed_auth_block, b'') == CUSTOM_PROXY_CONFIG assert b'location /unrelated/ { default_type application/json; return 200 \'{"keep":"bytes"}\'; }' in rendered - assert render_managed_vpm_config(CUSTOM_PROXY_CONFIG, enabled=False) == CUSTOM_PROXY_CONFIG + assert b'location = /auth/vast-price-manager/session { return 404; }' in rendered + assert b'location = /auth/vast-price-manager/session/ { return 404; }' in rendered + assert b'location = /auth/vast-price-manager/reauth { return 404; }' in rendered + assert b'location = /auth/vast-price-manager/reauth/ { return 404; }' in rendered + disabled = render_managed_vpm_config(CUSTOM_PROXY_CONFIG, enabled=False) + assert b'# BEGIN CRYPTOLABS MANAGED VPM' not in disabled + assert b'location = /auth/vast-price-manager/session { return 404; }' in disabled + assert b'location = /auth/vast-price-manager/session/ { return 404; }' in disabled + assert b'location = /auth/vast-price-manager/reauth { return 404; }' in disabled + assert b'location = /auth/vast-price-manager/reauth/ { return 404; }' in disabled + assert b'location = /vast-pricing {' not in disabled + assert b'proxy_pass http://$upstream_vast_price_manager:8088' not in disabled + assert disabled.count(b'# BEGIN CRYPTOLABS MANAGED INTERNAL AUTH DENIES') == 3 + assert disabled.replace(managed_auth_block, b'') == CUSTOM_PROXY_CONFIG + + +def test_custom_config_blocks_internal_vpm_auth_in_each_http_server_block(): + from cryptolabs_proxy.custom_config import render_managed_vpm_config + + multi_server_baseline = CUSTOM_PROXY_CONFIG.replace( + b"server { listen 80; location /legacy/ { proxy_pass http://legacy; } }", + b"server { listen 80; location /auth/ { proxy_pass http://auth_server/auth/; } location /legacy/ { proxy_pass http://legacy; } }", + ) + + disabled = render_managed_vpm_config(multi_server_baseline, enabled=False) + + assert b"location /auth/ { proxy_pass http://auth_server/auth/; }" in disabled + assert disabled.count(b"location = /auth/vast-price-manager/session { return 404; }") == 3 + assert disabled.count(b"location = /auth/vast-price-manager/reauth { return 404; }") == 3 + assert b"proxy_pass http://$upstream_vast_price_manager:8088" not in disabled @pytest.mark.parametrize( @@ -408,9 +455,29 @@ def test_cli_vpm_lifecycle_preserves_custom_baseline_and_rejects_unrelated_route disabled = runner.invoke(cli.main, ["unregister", "vast-price-manager"]) assert disabled.exit_code == 0, disabled.output - assert (tmp_path / "nginx.conf").read_bytes() == CUSTOM_PROXY_CONFIG + disabled_config = (tmp_path / "nginx.conf").read_bytes() + assert b"location /unrelated/" in disabled_config + assert b"# BEGIN CRYPTOLABS MANAGED VPM" not in disabled_config + assert b"location = /auth/vast-price-manager/session { return 404; }" in disabled_config + assert b"location = /auth/vast-price-manager/session/ { return 404; }" in disabled_config + assert b"location = /auth/vast-price-manager/reauth { return 404; }" in disabled_config + assert b"location = /auth/vast-price-manager/reauth/ { return 404; }" in disabled_config + assert b"location = /vast-pricing {" not in disabled_config + assert b"proxy_pass http://$upstream_vast_price_manager:8088" not in disabled_config assert ServiceRegistry(tmp_path).config["custom_config"]["baseline_sha256"] == hashlib.sha256(CUSTOM_PROXY_CONFIG).hexdigest() + reenabled = runner.invoke(cli.main, ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"]) + assert reenabled.exit_code == 0, reenabled.output + reenabled_config = (tmp_path / "nginx.conf").read_bytes() + assert reenabled_config.count(b"# BEGIN CRYPTOLABS MANAGED INTERNAL AUTH DENIES") == 3 + assert reenabled_config.count(b"# BEGIN CRYPTOLABS MANAGED VPM") == 1 + + disabled_again = runner.invoke(cli.main, ["unregister", "vast-price-manager"]) + assert disabled_again.exit_code == 0, disabled_again.output + disabled_again_config = (tmp_path / "nginx.conf").read_bytes() + assert disabled_again_config.count(b"# BEGIN CRYPTOLABS MANAGED INTERNAL AUTH DENIES") == 3 + assert b"# BEGIN CRYPTOLABS MANAGED VPM" not in disabled_again_config + def test_custom_mode_refuses_changed_baseline_before_mutating_registry(monkeypatch, tmp_path): import cryptolabs_proxy.cli as cli From 80eff6a245cbebdde045e2e8f7db44791178c262 Mon Sep 17 00:00:00 2001 From: Hannes Zietsman Date: Sat, 12 Sep 2026 20:50:45 +0200 Subject: [PATCH 3/4] Gate new Vast Price Manager services on connected exporter setup --- README.md | 12 +++ landing-page/index.html | 25 ++++- scripts/health-api.py | 20 ++++ src/cryptolabs_proxy/cli.py | 11 +++ src/cryptolabs_proxy/migration.py | 15 ++- src/cryptolabs_proxy/vpm_prerequisite.py | 100 +++++++++++++++++++ tests/test_vast_price_manager.py | 119 +++++++++++++++++++++++ tests/test_vpm_prerequisite.py | 77 +++++++++++++++ 8 files changed, 375 insertions(+), 4 deletions(-) create mode 100644 src/cryptolabs_proxy/vpm_prerequisite.py create mode 100644 tests/test_vpm_prerequisite.py diff --git a/README.md b/README.md index 0c79c24..b2045b9 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,18 @@ original stored baseline exactly. ### Vast Price Manager Fleet sign-in +### Vast Price Manager prerequisite + +Before a new Vast Price Manager route can be added, the local Vast.ai exporter +must be running and report at least one connected account. Complete setup at +`/vastai/`. Existing VPM installations remain available for maintenance if the +exporter later loses connectivity; they are never automatically disabled. + +The proxy checks the exporter from inside its container using its internal +management token, which must be provisioned by the exporter setup. It records +only whether setup is ready and the count of connected accounts; API keys, +account names, balances, and tokens are not returned. + When Vast Price Manager runs in Fleet mode, it uses the existing Fleet `fleet_session` cookie and does not create a second VPM account or login. VPM introspects the current Fleet session on the Docker network at diff --git a/landing-page/index.html b/landing-page/index.html index 08ee580..3079cae 100644 --- a/landing-page/index.html +++ b/landing-page/index.html @@ -846,7 +846,7 @@

⚙️ System Updates

path: '/vast-pricing/', productUrl: 'https://github.com/cryptolabsza/vast-price-manager', isCryptoLabs: true, - installNote: 'Install and manage this service from the Vast setup in Server Manager. Fleet access is restricted to administrators.', + installNote: 'Install and manage this service from the Vast setup in Server Manager.', lifecycleManager: 'dc-overview' }, 'dc-watchdog': { @@ -1002,7 +1002,28 @@

⚙️ System Updates

} } else { // Local service not installed - if (service.deployable) { + const vpmPrerequisite = key === 'vast-price-manager' + ? health['vast-price-manager']?.prerequisite + : null; + if (key === 'vast-price-manager' && !vpmPrerequisite?.configured) { + installSection = ` +
+

Requires Vast.ai setup with a connected account.

+
+ `; + actions = ` + Set up Vast.ai + `; + } else if (key === 'vast-price-manager') { + installSection = ` +
+

Available to install from the Vast.ai setup in Server Manager.

+
+ `; + actions = ` + Learn More + `; + } else if (service.deployable) { // Deployable exporter — check if Prometheus/Grafana are running first const prometheusUp = health['prometheus']?.running === true; const grafanaUp = health['grafana']?.running === true; diff --git a/scripts/health-api.py b/scripts/health-api.py index aaf0490..1c7e101 100644 --- a/scripts/health-api.py +++ b/scripts/health-api.py @@ -6,21 +6,28 @@ import json import subprocess +import time import http.server import socketserver import threading import os import re +import sys from urllib.error import HTTPError, URLError from urllib.parse import urlparse, parse_qs from urllib.request import Request, urlopen from pathlib import Path +sys.path.insert(0, '/app/src') +from cryptolabs_proxy.vpm_prerequisite import get_vpm_prerequisite + PORT = 8080 BUILD_INFO_FILE = '/app/BUILD_INFO' SETTINGS_FILE = '/data/auth/update-settings.json' SHARED_CONFIG_FILE = '/data/auth/shared-config.json' VPM_READY_URL = 'http://vast-price-manager:8088/readyz' +VPM_PREREQUISITE_CACHE_TTL_SECONDS = 30 +_VPM_PREREQUISITE_CACHE = {'expires_at': 0, 'value': None} # Internal Docker network subnet - only allow requests from this range INTERNAL_NETWORK = '172.30.' @@ -633,6 +640,7 @@ def get_all_service_status(include_versions=False): 'readiness': readiness, 'configured': readiness == 'ready', 'state': 'running' if readiness == 'ready' else readiness, + 'prerequisite': get_vpm_prerequisite_for_display(), }) if include_versions and running: @@ -648,6 +656,18 @@ def get_all_service_status(include_versions=False): return status +def get_vpm_prerequisite_for_display(): + """Cache the sanitized prerequisite only for status-page polling.""" + now = time.monotonic() + cached = _VPM_PREREQUISITE_CACHE + if cached['value'] is not None and now < cached['expires_at']: + return cached['value'] + value = get_vpm_prerequisite() + _VPM_PREREQUISITE_CACHE['value'] = value + _VPM_PREREQUISITE_CACHE['expires_at'] = now + VPM_PREREQUISITE_CACHE_TTL_SECONDS + return value + + def get_all_versions(): """Get version info for all services.""" versions = {} diff --git a/src/cryptolabs_proxy/cli.py b/src/cryptolabs_proxy/cli.py index 1d3dedc..5d16a2c 100644 --- a/src/cryptolabs_proxy/cli.py +++ b/src/cryptolabs_proxy/cli.py @@ -25,6 +25,7 @@ render_managed_vpm_config, ) from .services import DEFAULT_SERVICES, ServiceRegistry +from .vpm_prerequisite import get_vpm_prerequisite console = Console() @@ -537,6 +538,16 @@ def register(service_name, container_name, path, port, display_name, icon, descr with registry_lock(CONFIG_DIR): registry = ServiceRegistry(CONFIG_DIR) + # Only the first registration creates VPM access. Existing instances + # remain manageable when the exporter later becomes unavailable. + if service_name == "vast-price-manager" and service_name not in registry.services: + prerequisite = get_vpm_prerequisite() + if not prerequisite["configured"]: + raise click.ClickException( + "Requires Vast.ai setup with at least one connected account. " + "Open /vastai/ to finish setup." + ) + if path is None: path = f"/{service_name}/" services = copy.deepcopy(registry.services) diff --git a/src/cryptolabs_proxy/migration.py b/src/cryptolabs_proxy/migration.py index d02454c..66065fa 100644 --- a/src/cryptolabs_proxy/migration.py +++ b/src/cryptolabs_proxy/migration.py @@ -22,6 +22,7 @@ from urllib.parse import quote from .custom_config import configure_custom_config_mode, render_managed_vpm_config +from .vpm_prerequisite import get_vpm_prerequisite _LOCAL_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$") @@ -471,7 +472,19 @@ def apply(self, container: str, image: str, migration_id: str, enable_vpm: bool) if os.geteuid() != 0: raise MigrationError("custom proxy migration must run as root") from .cli import registry_lock + from .services import ServiceRegistry with registry_lock(self.config_dir): + registry = ServiceRegistry(self.config_dir) + # Existing registered VPM services remain maintainable when their + # exporter later goes down. A first custom enable needs fresh proof + # before backups, candidate files, or Docker changes are created. + if enable_vpm and registry.get_service("vast-price-manager") is None: + prerequisite = get_vpm_prerequisite() + if not prerequisite["configured"]: + raise MigrationError( + "Requires Vast.ai setup with at least one connected account. " + "Open /vastai/ to finish setup." + ) # Rebuild the read-only plan while holding the same lock that # serializes later VPM register/unregister writes. plan = self.plan(container, image) @@ -491,8 +504,6 @@ def apply(self, container: str, image: str, migration_id: str, enable_vpm: bool) raise MigrationError("source container ID or config changed before switch") self._write_private(backup / "baseline.nginx.conf", baseline) self._write_private(backup / "inspect.json", json.dumps(inspect, sort_keys=True).encode()) - from .services import ServiceRegistry - registry = ServiceRegistry(self.config_dir) previous = _snapshot_registry_files(registry) _write_registry_backup(backup, previous) candidate = self.config_dir / "nginx.conf" diff --git a/src/cryptolabs_proxy/vpm_prerequisite.py b/src/cryptolabs_proxy/vpm_prerequisite.py new file mode 100644 index 0000000..d684cb7 --- /dev/null +++ b/src/cryptolabs_proxy/vpm_prerequisite.py @@ -0,0 +1,100 @@ +"""Fresh, secret-safe readiness check for enabling Vast Price Manager.""" + +import json +import subprocess + + +EXPORTER_CONTAINER = "vastai-exporter" +PROBE_TIMEOUT_SECONDS = 15 + +# The command intentionally executes inside the exporter. That is the only +# place the management token is read; stdout is limited to a connected count. +_EXPORTER_PROBE = r""" +import json +import os +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener + +token = os.environ.get("MGMT_TOKEN") +if not token: + raise SystemExit(1) +request = Request( + "http://127.0.0.1:8622/api/accounts", + headers={"X-Mgmt-Token": token}, +) +try: + class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, request, fp, code, msg, headers, newurl): + return None + + # Ignore inherited HTTP(S)_PROXY values and reject redirects: the token is + # valid only for the exporter's loopback management API. + opener = build_opener(ProxyHandler({}), NoRedirect()) + with opener.open(request, timeout=5) as response: + if response.status != 200: + raise SystemExit(1) + payload = json.load(response) +except (HTTPError, URLError, OSError, ValueError): + raise SystemExit(1) + +accounts = payload.get("accounts") if isinstance(payload, dict) else None +if not isinstance(accounts, list): + raise SystemExit(1) +count = sum( + 1 for account in accounts + if isinstance(account, dict) and account.get("status") == "connected" +) +print(json.dumps({"connected_account_count": count})) +""" + + +def _result(configured: bool, reason: str, count: int = 0) -> dict: + """Return the sole public prerequisite representation.""" + return { + "configured": configured, + "reason": reason, + "connected_account_count": count, + } + + +def get_vpm_prerequisite() -> dict: + """Check exporter availability and connected accounts without exposing secrets. + + This function is deliberately uncached. Lifecycle callers use its current + result immediately before their first VPM registration or custom enable. + """ + try: + running = subprocess.run( + ["docker", "inspect", "--format", "{{.State.Running}}", EXPORTER_CONTAINER], + capture_output=True, + text=True, + timeout=PROBE_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError): + return _result(False, "exporter-not-running") + + if running.returncode != 0 or running.stdout.strip().lower() != "true": + return _result(False, "exporter-not-running") + + try: + probe = subprocess.run( + ["docker", "exec", EXPORTER_CONTAINER, "python3", "-c", _EXPORTER_PROBE], + capture_output=True, + text=True, + timeout=PROBE_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError): + return _result(False, "unavailable") + + if probe.returncode != 0: + return _result(False, "unavailable") + try: + payload = json.loads(probe.stdout) + count = payload["connected_account_count"] + except (TypeError, ValueError, KeyError): + return _result(False, "unavailable") + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + return _result(False, "unavailable") + if count == 0: + return _result(False, "no-connected-account") + return _result(True, "ready", count) diff --git a/tests/test_vast_price_manager.py b/tests/test_vast_price_manager.py index 71bfda4..a3393f1 100644 --- a/tests/test_vast_price_manager.py +++ b/tests/test_vast_price_manager.py @@ -117,6 +117,9 @@ def test_cli_register_unregister_toggles_generated_vpm_route_and_preserves_metad monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) monkeypatch.setattr(cli, "check_root", lambda: None) + monkeypatch.setattr(cli, "get_vpm_prerequisite", lambda: { + "configured": True, "reason": "ready", "connected_account_count": 1, + }) monkeypatch.setattr(cli, "proxy_uses_generated_config", lambda path: True) monkeypatch.setattr(cli, "validate_nginx_config", lambda: (True, "")) monkeypatch.setattr(cli, "reload_nginx_config", lambda: (True, "")) @@ -156,6 +159,107 @@ def test_cli_register_unregister_toggles_generated_vpm_route_and_preserves_metad assert "location = /vast-pricing {" in (tmp_path / "nginx.conf").read_text() +def test_cli_rejects_first_vpm_registration_without_mutating_files(monkeypatch, tmp_path): + import cryptolabs_proxy.cli as cli + + monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) + monkeypatch.setattr(cli, "check_root", lambda: None) + monkeypatch.setattr(cli, "get_vpm_prerequisite", lambda: { + "configured": False, "reason": "no-connected-account", "connected_account_count": 0, + }) + + result = CliRunner().invoke( + cli.main, + ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"], + ) + + assert result.exit_code != 0 + assert "Requires Vast.ai setup" in result.output + assert not (tmp_path / "services.yaml").exists() + assert not (tmp_path / "config.yaml").exists() + assert not (tmp_path / "nginx.conf").exists() + + +def test_cli_allows_existing_vpm_maintenance_when_exporter_is_unavailable(monkeypatch, tmp_path): + import cryptolabs_proxy.cli as cli + from cryptolabs_proxy.services import DEFAULT_SERVICES, ServiceRegistry + + registry = ServiceRegistry(tmp_path) + registry.services["vast-price-manager"] = deepcopy(DEFAULT_SERVICES["vast-price-manager"]) + registry.save() + monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) + monkeypatch.setattr(cli, "check_root", lambda: None) + monkeypatch.setattr(cli, "get_vpm_prerequisite", lambda: { + "configured": False, "reason": "exporter-not-running", "connected_account_count": 0, + }) + monkeypatch.setattr(cli, "proxy_uses_generated_config", lambda path: True) + monkeypatch.setattr(cli, "validate_nginx_config", lambda: (True, "")) + monkeypatch.setattr(cli, "reload_nginx_config", lambda: (True, "")) + + result = CliRunner().invoke( + cli.main, + ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"], + ) + + assert result.exit_code == 0, result.output + assert ServiceRegistry(tmp_path).get_service("vast-price-manager") is not None + + +def test_custom_migration_denies_first_enable_before_files_or_engine_changes(monkeypatch, tmp_path): + from cryptolabs_proxy.migration import CustomConfigMigrator, MigrationError + + class Engine: + def inspect(self, container): + raise AssertionError("migration must not inspect or change the engine") + + migrator = CustomConfigMigrator(tmp_path, tmp_path / "backups", engine=Engine()) + monkeypatch.setattr("cryptolabs_proxy.migration.os.geteuid", lambda: 0) + monkeypatch.setattr("cryptolabs_proxy.migration.get_vpm_prerequisite", lambda: { + "configured": False, "reason": "unavailable", "connected_account_count": 0, + }) + + with pytest.raises(MigrationError, match="Requires Vast.ai setup"): + migrator.apply("cryptolabs-proxy", "proxy@sha256:reviewed", "migration", enable_vpm=True) + + assert not (tmp_path / "backups").exists() + assert not (tmp_path / "nginx.conf").exists() + + +def test_vpm_health_includes_only_the_sanitized_prerequisite(monkeypatch): + health_api = load_health_api() + expected = {"configured": False, "reason": "no-connected-account", "connected_account_count": 0} + monkeypatch.setattr(health_api, "get_vpm_prerequisite", lambda: expected) + + status = health_api.get_all_service_status()["vast-price-manager"] + + assert status["prerequisite"] == expected + assert set(status["prerequisite"]) == {"configured", "reason", "connected_account_count"} + + +def test_vpm_health_caches_the_display_prerequisite_without_affecting_lifecycle_checks(monkeypatch): + health_api = load_health_api() + health_api._VPM_PREREQUISITE_CACHE = {"expires_at": 0, "value": None} + calls = [] + monkeypatch.setattr(health_api.time, "monotonic", lambda: 100) + monkeypatch.setattr(health_api, "get_vpm_prerequisite", lambda: calls.append(True) or { + "configured": True, "reason": "ready", "connected_account_count": 1, + }) + + first = health_api.get_all_service_status()["vast-price-manager"]["prerequisite"] + second = health_api.get_all_service_status()["vast-price-manager"]["prerequisite"] + + assert first == second == {"configured": True, "reason": "ready", "connected_account_count": 1} + assert calls == [True] + + +def test_landing_page_vpm_card_requires_vast_setup_before_installing(): + page = (REPOSITORY / "landing-page" / "index.html").read_text() + + assert "Requires Vast.ai setup" in page + assert 'href="/vastai/"' in page + assert "vpmPrerequisite" in page + + def test_cli_unregister_rolls_back_registry_and_generated_config_when_reload_fails(monkeypatch, tmp_path): import cryptolabs_proxy.cli as cli from cryptolabs_proxy.config import generate_nginx_config @@ -437,6 +541,9 @@ def test_cli_vpm_lifecycle_preserves_custom_baseline_and_rejects_unrelated_route monkeypatch.setattr(cli, "proxy_uses_generated_config", lambda path: True) monkeypatch.setattr(cli, "validate_nginx_config", lambda: (True, "")) monkeypatch.setattr(cli, "reload_nginx_config", lambda: (True, "")) + monkeypatch.setattr(cli, "get_vpm_prerequisite", lambda: { + "configured": True, "reason": "ready", "connected_account_count": 1, + }) runner = CliRunner() enabled = runner.invoke(cli.main, ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"]) @@ -493,6 +600,9 @@ def test_custom_mode_refuses_changed_baseline_before_mutating_registry(monkeypat monkeypatch.setattr(cli, "CONFIG_DIR", tmp_path) monkeypatch.setattr(cli, "check_root", lambda: None) + monkeypatch.setattr(cli, "get_vpm_prerequisite", lambda: { + "configured": True, "reason": "ready", "connected_account_count": 1, + }) result = CliRunner().invoke(cli.main, ["register", "vast-price-manager", "vast-price-manager", "--path", "/vast-pricing/", "--port", "8088"]) assert result.exit_code != 0 @@ -855,6 +965,9 @@ def remove(self, name, ignore_missing=False): self.calls.append(("remove", name) monkeypatch.setattr(migrator, "_read_active_config", lambda *args: CUSTOM_PROXY_CONFIG) monkeypatch.setattr(migrator, "_validate_candidate", lambda *args: None) monkeypatch.setattr(migrator, "_anonymous_auth_is_rejected", lambda: True) + monkeypatch.setattr("cryptolabs_proxy.migration.get_vpm_prerequisite", lambda: { + "configured": True, "reason": "ready", "connected_account_count": 1, + }) outcome = migrator.apply("cryptolabs-proxy", image, plan.migration_id, enable_vpm=True) @@ -886,6 +999,9 @@ def inspect(self, container): return source monkeypatch.setattr(migrator, "plan", lambda *args: plan) monkeypatch.setattr(migrator, "_read_active_config", lambda *args: CUSTOM_PROXY_CONFIG) monkeypatch.setattr(migrator, "_validate_candidate", lambda *args: None) + monkeypatch.setattr("cryptolabs_proxy.migration.get_vpm_prerequisite", lambda: { + "configured": True, "reason": "ready", "connected_account_count": 1, + }) monkeypatch.setattr( SwitchController, "apply", @@ -932,6 +1048,9 @@ def write_private(path, content): monkeypatch.setattr(migrator, "_read_active_config", lambda *args: CUSTOM_PROXY_CONFIG) monkeypatch.setattr(migrator, "_validate_candidate", lambda *args: None) monkeypatch.setattr(migrator, "_write_private", write_private) + monkeypatch.setattr("cryptolabs_proxy.migration.get_vpm_prerequisite", lambda: { + "configured": True, "reason": "ready", "connected_account_count": 1, + }) monkeypatch.setattr( SwitchController, "apply", diff --git a/tests/test_vpm_prerequisite.py b/tests/test_vpm_prerequisite.py new file mode 100644 index 0000000..ab91fec --- /dev/null +++ b/tests/test_vpm_prerequisite.py @@ -0,0 +1,77 @@ +"""Tests for the fresh, sanitized Vast exporter prerequisite probe.""" + +from types import SimpleNamespace + + +def _result(returncode=0, stdout=""): + return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="sensitive diagnostic") + + +def test_prerequisite_requires_running_exporter_before_exec(monkeypatch): + from cryptolabs_proxy import vpm_prerequisite + + monkeypatch.setattr(vpm_prerequisite.subprocess, "run", lambda *args, **kwargs: _result(stdout="false\n")) + + assert vpm_prerequisite.get_vpm_prerequisite() == { + "configured": False, + "reason": "exporter-not-running", + "connected_account_count": 0, + } + + +def test_prerequisite_returns_only_sanitized_connected_account_result(monkeypatch): + from cryptolabs_proxy import vpm_prerequisite + + calls = [] + + def run(command, **kwargs): + calls.append(command) + if command[:2] == ["docker", "inspect"]: + return _result(stdout="true\n") + return _result(stdout='{"connected_account_count": 2}') + + monkeypatch.setattr(vpm_prerequisite.subprocess, "run", run) + + assert vpm_prerequisite.get_vpm_prerequisite() == { + "configured": True, + "reason": "ready", + "connected_account_count": 2, + } + assert calls[1][:4] == ["docker", "exec", "vastai-exporter", "python3"] + assert "MGMT_TOKEN" in calls[1][-1] + assert "key_masked" not in calls[1][-1] + + +def test_exporter_probe_requires_the_provisioned_management_token(): + from cryptolabs_proxy import vpm_prerequisite + + assert "if not token:" in vpm_prerequisite._EXPORTER_PROBE + assert 'headers={"X-Mgmt-Token": token}' in vpm_prerequisite._EXPORTER_PROBE + assert "timeout=5" in vpm_prerequisite._EXPORTER_PROBE + assert vpm_prerequisite.PROBE_TIMEOUT_SECONDS == 15 + + +def test_prerequisite_fails_closed_for_empty_or_malformed_accounts_response(monkeypatch): + from cryptolabs_proxy import vpm_prerequisite + + responses = iter([_result(stdout="true\n"), _result(stdout="not-json")]) + monkeypatch.setattr(vpm_prerequisite.subprocess, "run", lambda *args, **kwargs: next(responses)) + + assert vpm_prerequisite.get_vpm_prerequisite() == { + "configured": False, + "reason": "unavailable", + "connected_account_count": 0, + } + + +def test_prerequisite_reports_no_connected_account_for_valid_empty_result(monkeypatch): + from cryptolabs_proxy import vpm_prerequisite + + responses = iter([_result(stdout="true\n"), _result(stdout='{"connected_account_count": 0}')]) + monkeypatch.setattr(vpm_prerequisite.subprocess, "run", lambda *args, **kwargs: next(responses)) + + assert vpm_prerequisite.get_vpm_prerequisite() == { + "configured": False, + "reason": "no-connected-account", + "connected_account_count": 0, + } From 915d0cd1191250d381d88f73e7672993986c9c4e Mon Sep 17 00:00:00 2001 From: Hannes Zietsman Date: Sun, 20 Sep 2026 19:20:55 +0200 Subject: [PATCH 4/4] Make fleet updates durable with preserved runtime configuration and verified rollback --- landing-page/index.html | 155 +++++------ scripts/health-api.py | 335 ++---------------------- src/cryptolabs_proxy/updates.py | 450 ++++++++++++++++++++++++++++++++ tests/test_updates.py | 358 +++++++++++++++++++++++++ 4 files changed, 907 insertions(+), 391 deletions(-) create mode 100644 src/cryptolabs_proxy/updates.py create mode 100644 tests/test_updates.py diff --git a/landing-page/index.html b/landing-page/index.html index 3079cae..7052e9d 100644 --- a/landing-page/index.html +++ b/landing-page/index.html @@ -1276,34 +1276,20 @@

${service.displayName}

versionText = 'Not running'; } - // --- Update logic --- - // Simple: compare the Docker image TAG against target. - // Target "main" means containers should be on ":latest" tag. - // Target "dev" means containers should be on ":dev" tag. - const targetBranch = settings.target_branch || 'main'; - const currentTag = info.tag || ''; - const expectedTag = targetBranch === 'dev' ? 'dev' : 'latest'; - const tagMatches = currentTag === expectedTag || currentTag === targetBranch; - let updateBtn = ''; if (info.lifecycle_manager) { updateBtn = `Managed by Server Manager`; + } else if (info.pinned) { + updateBtn = 'Pinned image · managed separately'; } else if (!info.running) { - const isExporter = name === 'vastai-exporter' || name === 'runpod-exporter'; - const prometheusUp = serviceHealth['prometheus']?.running === true; - const grafanaUp = serviceHealth['grafana']?.running === true; - - if (isExporter && (!prometheusUp || !grafanaUp)) { - updateBtn = 'Needs dc-overview'; - } else { - updateBtn = ``; - } - } else if (!tagMatches) { - // Tag mismatch (e.g. running :dev but target is main/latest) - updateBtn = ``; + updateBtn = 'Use service setup to start'; + } else if (info.update_available === true) { + updateBtn = ``; updatableCount++; + } else if (info.update_available === false) { + updateBtn = '✓ Downloaded image running'; } else { - updateBtn = '✓ Up to date'; + updateBtn = 'Check for updates first'; } // Self badge for proxy @@ -1349,7 +1335,7 @@

${service.displayName}

updateAllBtn.style.cursor = 'pointer'; } else { updateAllBtn.disabled = true; - updateAllBtn.title = 'All services are up to date or not running'; + updateAllBtn.title = 'Check for updates; pinned and stopped services are preserved'; updateAllBtn.style.opacity = '0.5'; updateAllBtn.style.cursor = 'not-allowed'; } @@ -1374,92 +1360,91 @@

${service.displayName}

}); if (!response.ok) throw new Error('Failed to save'); showUpdateStatus('info', `Target branch set to: ${branch}`); + loadVersions(); } catch (e) { showUpdateStatus('error', `Failed to save branch setting: ${e.message}`); } } - async function checkForUpdates() { - showUpdateStatus('info', '🔍 Checking for updates...'); - const branch = document.getElementById('targetBranch').value; + let activeUpdateJob = null; + let updatePollTimer = null; + const UPDATE_JOB_KEY = 'fleet-update-job'; + + function rememberUpdateJob(id) { try { - const response = await fetch('/api/pull', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ service: 'all', branch }) - }); - const data = await response.json(); - if (data.success) { - showUpdateStatus('success', '✓ Images pulled. Restart containers to apply updates.'); - loadVersions(); - } else { - showUpdateStatus('error', 'Failed to pull updates'); - } - } catch (e) { - showUpdateStatus('error', `Error: ${e.message}`); - } + if (id) localStorage.setItem(UPDATE_JOB_KEY, id); + else localStorage.removeItem(UPDATE_JOB_KEY); + } catch (e) { /* Storage may be disabled; this page can still poll. */ } } - async function updateService(serviceName) { - const branch = document.getElementById('targetBranch').value; - showUpdateStatus('info', `⬆️ Updating ${serviceName}...`); + async function pollUpdateJob(id, failures = 0) { + activeUpdateJob = id; + clearTimeout(updatePollTimer); try { - const response = await fetch('/api/update', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ service: serviceName, branch }) - }); - const data = await response.json(); - if (data.success) { - const result = data.results[serviceName] || {}; - if (result.message === 'self-update-required') { - showUpdateStatus('info', '🔄 Proxy is updating... Page will reload shortly.'); - setTimeout(() => location.reload(), 5000); - } else { - showUpdateStatus('success', `✓ ${serviceName} updated: ${result.message}`); - } - setTimeout(loadVersions, 2000); - } else { - showUpdateStatus('error', `Failed to update ${serviceName}`); + const response = await fetch(`/api/update-status?id=${encodeURIComponent(id)}`); + if (!response.ok) throw new Error('Update status unavailable'); + const job = await response.json(); + if (job.state === 'queued' || job.state === 'running') { + showUpdateStatus('info', 'Update operation in progress. Services restart only after their images are ready.'); + updatePollTimer = setTimeout(() => pollUpdateJob(id), 2000); + return; } + const details = Object.entries(job.results || {}).map(([name, result]) => + `${name}: ${result.message || result.state}`).join(' · '); + showUpdateStatus(job.success === true ? 'success' : 'error', + details || job.message || `Update operation ${job.state}`); + activeUpdateJob = null; + rememberUpdateJob(null); + loadVersions(); + loadServices(); } catch (e) { - showUpdateStatus('error', `Error: ${e.message}`); + if (failures >= 60) { + showUpdateStatus('error', 'Update status is unavailable. Reload to reconnect before starting another update.'); + return; + } + showUpdateStatus('info', 'Reconnecting to update status. The operation may still be running.'); + updatePollTimer = setTimeout(() => pollUpdateJob(id, failures + 1), 3000); } } - async function updateAll() { - if (!confirm('Update all running services to the target branch? This may cause brief downtime. Non-running services will not be started.')) return; - + async function startUpdateJob(action, service) { + if (activeUpdateJob) { + showUpdateStatus('info', 'An update operation is already in progress.'); + return; + } const branch = document.getElementById('targetBranch').value; - showUpdateStatus('info', '⬆️ Updating all services...'); + showUpdateStatus('info', action === 'pull' ? 'Checking images…' : 'Preparing update…'); try { - const response = await fetch('/api/update', { + const response = await fetch(action === 'pull' ? '/api/pull' : '/api/update', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ service: 'all', branch }) + body: JSON.stringify({ service, branch }) }); const data = await response.json(); - if (data.success) { - let msg = 'Updated: '; - for (const [name, result] of Object.entries(data.results || {})) { - msg += `${name}(${result.success ? '✓' : '✗'}) `; - } - if (data.results['cryptolabs-proxy']?.message?.includes('self-update')) { - showUpdateStatus('info', '🔄 Proxy is updating... Page will reload shortly.'); - setTimeout(() => location.reload(), 5000); - } else { - showUpdateStatus('success', msg); - } - setTimeout(loadVersions, 2000); - setTimeout(loadServices, 2000); + if (data.job && (response.ok || response.status === 409)) { + rememberUpdateJob(data.job.id); + await pollUpdateJob(data.job.id); } else { - showUpdateStatus('error', 'Update failed'); + showUpdateStatus('error', data.error || 'Could not start update operation.'); } } catch (e) { - showUpdateStatus('error', `Error: ${e.message}`); + showUpdateStatus('error', 'Could not confirm whether the update started. Refresh status before retrying.'); } } + async function checkForUpdates() { + return startUpdateJob('pull', 'all'); + } + + async function updateService(serviceName) { + return startUpdateJob('update', serviceName); + } + + async function updateAll() { + if (!confirm('Update eligible running services? Services will briefly restart. Pinned images and stopped services are preserved.')) return; + return startUpdateJob('update', 'all'); + } + function showUpdateStatus(type, message) { // Remove any existing status const existingStatus = document.querySelector('.update-status'); @@ -1733,6 +1718,10 @@

${service.displayName}

loadServices(); loadBuildInfo(); loadVersions(); // This calls loadWatchdogStatus() at the end + try { + const pendingUpdate = localStorage.getItem(UPDATE_JOB_KEY); + if (pendingUpdate) pollUpdateJob(pendingUpdate); + } catch (e) { /* Browser storage is optional. */ } // Auto-refresh every 30 seconds setInterval(loadServices, 30000); diff --git a/scripts/health-api.py b/scripts/health-api.py index 1c7e101..ed269c0 100644 --- a/scripts/health-api.py +++ b/scripts/health-api.py @@ -20,6 +20,10 @@ sys.path.insert(0, '/app/src') from cryptolabs_proxy.vpm_prerequisite import get_vpm_prerequisite +from cryptolabs_proxy.updates import ( + SERVICES, UpdateBusy, UpdateError, job_status, + submit_job as submit_update_job, update_status, +) PORT = 8080 BUILD_INFO_FILE = '/app/BUILD_INFO' @@ -44,25 +48,6 @@ INTERNAL_CONFIG_KEYS = {'watchdog_api_key'} NEVER_EXPOSE_KEYS = {'fleet_admin_pass', 'fleet_admin_user', 'auth_secret'} -# Services to check (Docker containers) -SERVICES = { - 'cryptolabs-proxy': {'container': 'cryptolabs-proxy', 'port': 8080, 'image': 'ghcr.io/cryptolabsza/cryptolabs-proxy', 'self': True}, - 'ipmi-monitor': {'container': 'ipmi-monitor', 'port': 5000, 'image': 'ghcr.io/cryptolabsza/ipmi-monitor'}, - 'dc-overview': {'container': 'dc-overview', 'port': 5001, 'image': 'ghcr.io/cryptolabsza/dc-overview'}, - 'grafana': {'container': 'grafana', 'port': 3000, 'image': 'grafana/grafana'}, - 'prometheus': {'container': 'prometheus', 'port': 9090, 'image': 'prom/prometheus'}, - 'vastai-exporter': {'container': 'vastai-exporter', 'port': 8622, 'image': 'ghcr.io/cryptolabsza/vastai-exporter'}, - # The optional VPM service is installed and lifecycle-managed by - # dc-overview. The proxy only reports its status and proxies its UI. - 'vast-price-manager': { - 'container': 'vast-price-manager', - 'port': 8088, - 'image': '', - 'lifecycle_manager': 'dc-overview', - 'update_supported': False, - }, - 'runpod-exporter': {'container': 'runpod-exporter', 'port': 8623, 'image': 'ghcr.io/cryptolabsza/runpod-exporter'}, -} def load_update_settings(): @@ -397,218 +382,6 @@ def get_container_version(container_name): return None -def pull_image(image_name, tag='latest'): - """Pull a Docker image.""" - full_image = f"{image_name}:{tag}" - try: - result = subprocess.run( - ['docker', 'pull', full_image], - capture_output=True, text=True, timeout=300 - ) - return result.returncode == 0, result.stdout + result.stderr - except Exception as e: - return False, str(e) - - -def get_container_config(container_name): - """Get the configuration of a running container for restart.""" - try: - result = subprocess.run( - ['docker', 'inspect', container_name], - capture_output=True, text=True, timeout=10 - ) - if result.returncode != 0: - return None - - data = json.loads(result.stdout) - if not data: - return None - - container = data[0] - config = container.get('Config', {}) - host_config = container.get('HostConfig', {}) - network_settings = container.get('NetworkSettings', {}) - - # Get network names - networks = list(network_settings.get('Networks', {}).keys()) - - # Get port bindings - port_bindings = host_config.get('PortBindings', {}) - - # Get volume bindings - binds = host_config.get('Binds', []) or [] - - # Get environment variables - env_vars = config.get('Env', []) or [] - - # Get restart policy - restart_policy = host_config.get('RestartPolicy', {}).get('Name', 'unless-stopped') - - return { - 'networks': networks, - 'port_bindings': port_bindings, - 'binds': binds, - 'env_vars': env_vars, - 'restart_policy': restart_policy, - } - except Exception as e: - print(f"Error getting container config for {container_name}: {e}") - return None - - -def restart_container(container_name, image_with_tag, container_config): - """Restart a container with the same configuration but new image.""" - if not container_config: - return False, "No container configuration available" - - try: - # Build docker run command - cmd = ['docker', 'run', '-d', '--name', container_name] - - # Restart policy - cmd.extend(['--restart', container_config.get('restart_policy', 'unless-stopped')]) - - # Networks - docker run only supports one --network, use first non-bridge network - # Additional networks will be connected after container starts - networks = container_config.get('networks', ['cryptolabs']) - non_bridge_networks = [n for n in networks if n and n != 'bridge'] - primary_network = non_bridge_networks[0] if non_bridge_networks else None - additional_networks = non_bridge_networks[1:] if len(non_bridge_networks) > 1 else [] - - if primary_network: - cmd.extend(['--network', primary_network]) - - # Port bindings - for container_port, host_bindings in container_config.get('port_bindings', {}).items(): - if host_bindings: - for binding in host_bindings: - host_port = binding.get('HostPort', '') - host_ip = binding.get('HostIp', '') - if host_ip: - cmd.extend(['-p', f"{host_ip}:{host_port}:{container_port.split('/')[0]}"]) - else: - cmd.extend(['-p', f"{host_port}:{container_port.split('/')[0]}"]) - - # Volume bindings - for bind in container_config.get('binds', []): - cmd.extend(['-v', bind]) - - # Environment variables (filter out build-time vars that we'll update) - for env in container_config.get('env_vars', []): - # Skip PATH and other system vars, keep user-defined ones - if env.startswith('PATH=') or env.startswith('HOME='): - continue - cmd.extend(['-e', env]) - - # Image - cmd.append(image_with_tag) - - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode != 0: - return False, f"Failed to start container: {result.stderr}" - - # Connect to additional networks (docker run only supports one --network) - for network in additional_networks: - try: - subprocess.run(['docker', 'network', 'connect', network, container_name], - capture_output=True, timeout=10) - except Exception as e: - print(f"Warning: Failed to connect {container_name} to network {network}: {e}") - - return True, "Container restarted successfully" - except Exception as e: - return False, f"Error restarting container: {e}" - - -def update_container(container_name, service_config, target_branch='main'): - """Update a container to a new image version.""" - image = service_config.get('image', '') - if not image: - return False, "No image configured for service" - - is_self = service_config.get('self', False) - tag = target_branch if target_branch in ['dev', 'main'] else 'latest' - - # Default to latest for all images - if not tag or tag == 'main': - tag = 'latest' - - full_image = f"{image}:{tag}" - - # Pull new image first - success, output = pull_image(image, tag) - if not success: - return False, f"Failed to pull image: {output}" - - if is_self: - # For self-update, we need special handling - # Create a script that will restart the container after we exit - return True, "self-update-required" - - # Get current container configuration before stopping - container_config = get_container_config(container_name) - - # Stop and remove old container - try: - subprocess.run(['docker', 'stop', container_name], capture_output=True, timeout=30) - subprocess.run(['docker', 'rm', container_name], capture_output=True, timeout=10) - except: - pass - - # Restart container with new image - if container_config: - success, msg = restart_container(container_name, full_image, container_config) - if success: - return True, f"Updated to {tag} and restarted" - else: - return False, f"Image pulled but restart failed: {msg}" - else: - # Fallback: container wasn't running or couldn't get config - return True, f"Image pulled. Container needs manual restart (was not running)." - - -def trigger_self_update(target_branch='main'): - """Trigger self-update for the proxy container.""" - # Pull the new image - image = 'ghcr.io/cryptolabsza/cryptolabs-proxy' - tag = target_branch if target_branch in ['dev', 'main'] else 'latest' - - success, output = pull_image(image, tag) - if not success: - return False, f"Failed to pull image: {output}" - - # Create a restart script that runs after the API responds - # This uses docker to restart the container from outside - script = f"""#!/bin/bash -sleep 2 -docker stop cryptolabs-proxy -docker rm cryptolabs-proxy -# The container should be recreated by docker-compose or systemd -# For safety, try to start it using the same command pattern -docker run -d --name cryptolabs-proxy \\ - --restart unless-stopped \\ - --network cryptolabs \\ - -v /var/run/docker.sock:/var/run/docker.sock \\ - -v /data/auth:/data/auth \\ - -p 80:80 -p 443:443 \\ - {image}:{tag} -""" - - # Write script and execute in background - script_path = '/tmp/proxy-update.sh' - try: - with open(script_path, 'w') as f: - f.write(script) - os.chmod(script_path, 0o755) - subprocess.Popen(['/bin/bash', script_path], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - start_new_session=True) - return True, "Self-update initiated. Proxy will restart in a few seconds." - except Exception as e: - return False, f"Failed to initiate self-update: {e}" - - def get_all_service_status(include_versions=False): """Get status of all services.""" status = {} @@ -707,6 +480,8 @@ def get_all_versions(): }) versions[name] = version_info + if running and config.get('update_supported') is not False: + version_info.update(update_status(name, settings.get('branch', 'main'))) # Add configured branch versions['_settings'] = { @@ -780,6 +555,12 @@ def do_GET(self): elif path == '/api/update-settings': settings = load_update_settings() self.send_json(settings) + + elif path == '/api/update-status': + try: + self.send_json(job_status(query.get('id', [''])[0])) + except UpdateError as error: + self.send_json({'error': str(error)}, 404) # ---- Internal Config API (fleet services only) ---- # Security: 4 layers of protection @@ -864,83 +645,21 @@ def do_POST(self): else: self.send_json({'error': 'Failed to save settings'}, 500) - elif path == '/api/update': - # Trigger update for one or all services - service = data.get('service', 'all') - target_branch = data.get('branch', load_update_settings().get('branch', 'main')) - - results = {} - - if service == 'all': - # Update all services - for name, config in SERVICES.items(): - if service_action_error(name): - continue - if config.get('self'): - # Handle self-update last - continue - tag = 'dev' if target_branch == 'dev' else 'latest' - success, msg = update_container(name, config, tag) - results[name] = {'success': success, 'message': msg} - - # Handle proxy self-update last (if requested) - if 'cryptolabs-proxy' in SERVICES: - success, msg = trigger_self_update(target_branch) - results['cryptolabs-proxy'] = {'success': success, 'message': msg} - - elif service == 'cryptolabs-proxy': - # Self-update - success, msg = trigger_self_update(target_branch) - results[service] = {'success': success, 'message': msg} - - elif service in SERVICES: - action_error = service_action_error(service) - if action_error: - self.send_json({'error': action_error}, 400) - return - config = SERVICES[service] - tag = 'dev' if target_branch == 'dev' else 'latest' - success, msg = update_container(service, config, tag) - results[service] = {'success': success, 'message': msg} - - else: - self.send_json({'error': f'Unknown service: {service}'}, 400) - return - - self.send_json({'success': True, 'results': results}) - - elif path == '/api/pull': - # Just pull images without restarting - service = data.get('service', 'all') - target_branch = data.get('branch', load_update_settings().get('branch', 'main')) - - results = {} - services_to_pull = [service] if service != 'all' else list(SERVICES.keys()) - - for name in services_to_pull: - if name not in SERVICES: - results[name] = {'success': False, 'message': 'Unknown service'} - continue - - action_error = service_action_error(name) - if action_error: - if service == 'all': - continue - self.send_json({'error': action_error}, 400) - return - - config = SERVICES[name] - tag = 'dev' if target_branch == 'dev' else 'latest' - - image = config.get('image', '') - if image: - success, msg = pull_image(image, tag) - results[name] = {'success': success, 'message': msg[:200] if len(msg) > 200 else msg} - else: - results[name] = {'success': False, 'message': 'No image configured'} - - self.send_json({'success': True, 'results': results}) - + elif path in ('/api/update', '/api/pull'): + try: + job = submit_update_job( + data.get('service', 'all'), + data.get('branch', load_update_settings().get('branch', 'main')), + 'pull' if path == '/api/pull' else 'update', + ) + self.send_json({'success': True, 'job': job}, 202) + except UpdateBusy as error: + self.send_json({'success': False, 'error': str(error), 'job': error.job}, 409) + except UpdateError as error: + self.send_json({'success': False, 'error': str(error)}, 400) + except Exception: + self.send_json({'success': False, 'error': 'Updater unavailable; no update confirmed.'}, 503) + # ---- Internal Config API: SET values (always requires token) ---- elif path == '/internal/api/config': if not is_internal_request(self.client_address): diff --git a/src/cryptolabs_proxy/updates.py b/src/cryptolabs_proxy/updates.py new file mode 100644 index 0000000..a5f50ff --- /dev/null +++ b/src/cryptolabs_proxy/updates.py @@ -0,0 +1,450 @@ +"""Serialized fleet image updates executed outside the serving proxy container. + +Job files contain only operation state, never container environments or Docker +error bodies. Existing containers remain available as rollback artifacts. +""" +from __future__ import annotations + +import fcntl +import json +import os +import re +import subprocess +import time +import uuid +from contextlib import contextmanager +from copy import deepcopy +from pathlib import Path +from urllib.parse import quote +from urllib.request import urlopen + +from .migration import DockerEngine, MigrationError, _mount_request, endpoint_configurations + + +SERVICES = { + 'cryptolabs-proxy': {'container': 'cryptolabs-proxy', 'port': 8080, 'image': 'ghcr.io/cryptolabsza/cryptolabs-proxy', 'self': True}, + 'ipmi-monitor': {'container': 'ipmi-monitor', 'port': 5000, 'image': 'ghcr.io/cryptolabsza/ipmi-monitor'}, + 'dc-overview': {'container': 'dc-overview', 'port': 5001, 'image': 'ghcr.io/cryptolabsza/dc-overview'}, + 'grafana': {'container': 'grafana', 'port': 3000, 'image': 'grafana/grafana'}, + 'prometheus': {'container': 'prometheus', 'port': 9090, 'image': 'prom/prometheus'}, + 'vastai-exporter': {'container': 'vastai-exporter', 'port': 8622, 'image': 'ghcr.io/cryptolabsza/vastai-exporter'}, + 'vast-price-manager': {'container': 'vast-price-manager', 'port': 8088, 'image': '', + 'lifecycle_manager': 'dc-overview', 'update_supported': False}, + 'runpod-exporter': {'container': 'runpod-exporter', 'port': 8623, 'image': 'ghcr.io/cryptolabsza/runpod-exporter'}, +} +JOBS_DIR = Path('/data/auth/update-jobs') +CHANNEL_LABEL = 'io.cryptolabs.update.channel-image' +IMAGE_LABEL = 'io.cryptolabs.update.image-id' +ACTIVE_STATES = {'queued', 'running'} +BUILD_ENV = {'GIT_COMMIT', 'GIT_BRANCH', 'BUILD_TIME', 'APP_VERSION', 'HOSTNAME'} +# Docker 29 no longer accepts old Engine API clients. 1.45 is supported by +# Docker 29 (minimum 1.44) and retains the endpoint fields used below. +DOCKER_API_VERSION = 'v1.45' + + +class UpdateError(RuntimeError): + """Public, non-sensitive updater failure.""" + + +class UpdateBusy(UpdateError): + def __init__(self, job): + super().__init__('An update is already in progress.') + self.job = job + + +class UpdateEngine(DockerEngine): + def _request(self, method, path, payload=None, allowed=(200, 201, 204)): + """Use an Engine API version accepted by the Docker 29 fleet host.""" + if not path.startswith(f'/{DOCKER_API_VERSION}/'): + path = f'/{DOCKER_API_VERSION}{path}' + return super()._request(method, path, payload, allowed) + + def image(self, reference): + return json.loads(self._request('GET', f'/images/{quote(reference, safe="")}/json')) + + +def target_for(name, source, branch): + """Only floating, first-party channels may follow the branch selector.""" + service = SERVICES[name] + if service.get('update_supported') is False: + return None + config = source.get('Config', {}) + reference = config.get('Image', '') + labels = config.get('Labels') or {} + if reference.startswith('sha256:') and labels.get(IMAGE_LABEL) == source.get('Image'): + reference = labels.get(CHANNEL_LABEL, reference) + base = service['image'] + if reference not in {base, f'{base}:latest', f'{base}:main', f'{base}:dev'}: + return None + tag = ('dev' if branch == 'dev' else 'latest') if base.startswith('ghcr.io/cryptolabsza/') else reference.rsplit(':', 1)[-1] + if tag == base or tag == 'main': + tag = 'latest' + return f'{base}:{tag}' + + +def update_status(name, branch, *, engine=None): + engine = engine or UpdateEngine() + result = {'update_available': None, 'pinned': False} + try: + source = engine.inspect(SERVICES[name]['container']) + target = target_for(name, source, branch) + result.update({'running_image_id': source['Image'], 'target_image': target, 'pinned': target is None}) + if target: + candidate = engine.image(target) + result.update({'target_image_id': candidate['Id'], 'update_available': candidate['Id'] != source['Image']}) + except Exception: + pass # Unknown image identity is never presented as up-to-date. + return result + + +def _env(values): + return dict(value.split('=', 1) for value in (values or []) if '=' in value) + + +def build_update_request(source, old_image, new_image): + """Preserve deploy overrides while allowing target image defaults to change.""" + config = deepcopy(source['Config']) + old_defaults, new_defaults = old_image.get('Config') or {}, new_image.get('Config') or {} + env = _env(new_defaults.get('Env')) + old_env = _env(old_defaults.get('Env')) + for key, value in _env(config.get('Env')).items(): + if key not in BUILD_ENV and (key not in old_env or old_env[key] != value): + env[key] = value + config['Env'] = [f'{key}={value}' for key, value in env.items()] + labels = deepcopy(new_defaults.get('Labels') or {}) + old_labels = old_defaults.get('Labels') or {} + for key, value in (config.get('Labels') or {}).items(): + if not key.startswith('org.opencontainers.image.') and (key not in old_labels or old_labels[key] != value): + labels[key] = value + config['Labels'] = labels + for key in ('Cmd', 'Entrypoint', 'User', 'WorkingDir', 'Healthcheck', 'StopSignal', 'Shell', 'ExposedPorts', 'Volumes'): + if key in old_defaults and config.get(key) == old_defaults[key]: + if key in new_defaults: + config[key] = deepcopy(new_defaults[key]) + else: + config.pop(key, None) + if config.get('Hostname') in (source.get('Id'), source.get('Id', '')[:12]): + config.pop('Hostname', None) + config['Image'] = new_image['Id'] + host = deepcopy(source['HostConfig']) + if host.get('AutoRemove') or host.get('VolumesFrom'): + raise UpdateError('Automatic removal or inherited volumes require a managed update.') + mounts = deepcopy(host.get('Mounts') or []) + targets = {mount.get('Target') for mount in mounts} + targets.update((host.get('Tmpfs') or {}).keys()) + for binding in host.get('Binds') or []: + parts = binding.split(':') + if len(parts) < 2: + raise UpdateError('Cannot preserve an ambiguous mount.') + targets.add(parts[1]) + for mount in source.get('Mounts', []): + if mount['Destination'] not in targets: + mounts.append(_mount_request(mount)) + host['Mounts'] = mounts + endpoints = endpoint_configurations(source) + generated_aliases = {source.get('Id'), source.get('Id', '')[:12]} + for endpoint in endpoints.values(): + if endpoint.get('Aliases'): + endpoint['Aliases'] = [alias for alias in endpoint['Aliases'] if alias not in generated_aliases] + return {'Config': config, 'HostConfig': host, 'NetworkingConfig': {'EndpointsConfig': endpoints}} + + +def _probe(source, name): + """Fallback application probe for containers without Docker healthchecks.""" + service = SERVICES[name] + addresses = [endpoint.get('IPAddress') for endpoint in source.get('NetworkSettings', {}).get('Networks', {}).values()] + if source.get('HostConfig', {}).get('NetworkMode') == 'host': + addresses = ['127.0.0.1'] + paths = {'cryptolabs-proxy': '/api/health', 'dc-overview': '/api/health', + 'ipmi-monitor': '/health', 'grafana': '/api/health', 'prometheus': '/-/ready'} + path = paths.get(name, '/metrics') + if name == 'prometheus': + for argument in source.get('Config', {}).get('Cmd') or []: + if argument.startswith('--web.route-prefix='): + path = argument.split('=', 1)[1].rstrip('/') + '/-/ready' + for address in addresses: + if address: + try: + with urlopen(f'http://{address}:{service["port"]}{path}', timeout=3) as response: + if response.status == 200: + return True + except Exception: + pass + return False + + +def wait_ready(engine, name, image_id, timeout=150): + deadline = time.monotonic() + timeout + while True: + source = engine.inspect(name) + state = source.get('State', {}) + health = state.get('Health', {}).get('Status') + if source.get('Image') != image_id or not state.get('Running') or health == 'unhealthy': + return False + if health == 'healthy' or (health is None and _probe(source, name)): + return True + if time.monotonic() >= deadline: + return False + time.sleep(1) + + +def replace_container(engine, name, image, job_id, *, timeout=150, expected_id=None): + """Swap without destroying the previous runtime; prove recovery on failure.""" + backup = f'{name}.rollback-{job_id}' + detached = [] + started_stop = renamed = created = False + source = None + try: + source = engine.inspect(name) + if expected_id and source['Id'] != expected_id: + raise UpdateError('Container changed during update preparation.') + if not source.get('State', {}).get('Running'): + raise UpdateError('Service is stopped; use its installer to start it.') + candidate = engine.image(image) + if candidate['Id'] == source['Image']: + return {'success': True, 'state': 'unchanged', 'image_id': source['Image'], 'message': 'Already running the selected image.'} + request = build_update_request(source, engine.image(source['Image']), candidate) + if not image.startswith('sha256:'): + request['Config']['Labels'].update({CHANNEL_LABEL: image, IMAGE_LABEL: candidate['Id']}) + endpoints = request['NetworkingConfig']['EndpointsConfig'] + started_stop = True + engine.stop(name) + for network in endpoints: + engine.disconnect(name, network) + detached.append(network) + engine.rename(name, backup) + renamed = True + engine.create(name, request) + created = True + engine.start(name) + if not wait_ready(engine, name, candidate['Id'], timeout): + raise UpdateError('Replacement failed application health or image verification.') + return {'success': True, 'state': 'completed', 'image_id': candidate['Id'], + 'rollback_container': backup, 'message': 'Updated and health verified.'} + except Exception as error: + message = str(error) if isinstance(error, UpdateError) else 'Container update failed.' + if not started_stop: + return {'success': False, 'state': 'failed', 'message': message} + try: + if created: + engine.remove(name, ignore_missing=True) + if renamed: + engine.rename(backup, name) + for network in detached: + engine.connect(name, network, endpoints[network]) + engine.start(name) + if not wait_ready(engine, name, source['Image'], timeout): + raise UpdateError('Original service did not recover.') + return {'success': False, 'state': 'rolled_back', 'message': message + ' Original service restored and health verified.'} + except Exception: + return {'success': False, 'state': 'recovery_required', 'message': message + ' Automatic recovery was not verified; operator recovery required.', 'rollback_container': backup if renamed else name} + + +def _job_path(directory, job_id): + if not re.fullmatch('[0-9a-f]{32}', job_id): + raise UpdateError('Invalid update job identifier.') + return Path(directory) / f'{job_id}.json' + + +def _write_json(path, value): + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = path.with_suffix('.tmp-' + uuid.uuid4().hex) + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, 'w') as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + + +def write_job(directory, job): + _write_json(_job_path(directory, job['id']), job) + + +def read_job(directory, job_id): + try: + return json.loads(_job_path(directory, job_id).read_text()) + except FileNotFoundError: + raise UpdateError('Update job not found.') from None + + +def job_status(job_id, *, directory=JOBS_DIR, engine=None): + job = read_job(directory, job_id) + if job.get('state') in ACTIVE_STATES: + engine = engine or UpdateEngine() + try: + running = engine.inspect(job['helper'])['State']['Running'] + except MigrationError as error: + if not str(error).endswith('HTTP 404'): + return job + running = False + except Exception: + return job + if not running: + job = read_job(directory, job_id) + if job.get('state') in ACTIVE_STATES: + job.update(state='interrupted', success=False, + message='Update helper stopped before completion. Inspect services before retrying.') + write_job(directory, job) + return job + + +@contextmanager +def _lock(directory, name): + Path(directory).mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor = os.open(Path(directory) / name, os.O_RDWR | os.O_CREAT, 0o600) + with os.fdopen(descriptor, 'w') as stream: + fcntl.flock(stream, fcntl.LOCK_EX) + yield + + +def helper_request(source, directory, job_id): + """The helper uses the current immutable proxy image, outside its PID space.""" + parent = next((mount for mount in sorted(source.get('Mounts', []), key=lambda item: len(item['Destination']), reverse=True) + if str(directory).startswith(mount['Destination'].rstrip('/') + '/')), None) + if not parent or not parent.get('RW', True): + raise UpdateError('Updater needs a writable persistent /data/auth mount.') + mount = _mount_request(parent) + return { + 'Config': {'Image': source['Image'], 'Entrypoint': ['python3'], + 'Cmd': ['-m', 'cryptolabs_proxy.updates', 'worker', job_id, str(directory)], + 'Env': ['PYTHONPATH=/app/src'], 'Healthcheck': {'Test': ['NONE']}, + 'Labels': {'io.cryptolabs.update.worker': job_id, 'com.centurylinklabs.watchtower.enable': 'false'}}, + 'HostConfig': {'NetworkMode': 'host', 'RestartPolicy': {'Name': 'no'}, + 'Mounts': [mount, {'Type': 'bind', 'Source': '/var/run/docker.sock', 'Target': '/var/run/docker.sock'}], + 'LogConfig': {'Type': 'json-file', 'Config': {'max-size': '1m', 'max-file': '1'}}}, + 'NetworkingConfig': {}, + } + + +def submit_job(service, branch, action, *, directory=JOBS_DIR, engine=None): + if branch not in ('main', 'dev') or action not in ('pull', 'update'): + raise UpdateError('Invalid update channel or action.') + if service != 'all' and (service not in SERVICES or SERVICES[service].get('update_supported') is False): + raise UpdateError('Service is not managed by this updater.') + engine = engine or UpdateEngine() + with _lock(directory, 'dispatch.lock'): + active = Path(directory) / 'active.json' + if active.exists(): + previous = read_job(directory, json.loads(active.read_text())['id']) + if previous.get('state') in ACTIVE_STATES: + try: + running = engine.inspect(previous['helper'])['State']['Running'] + except MigrationError as error: + if not str(error).endswith('HTTP 404'): + # An inaccessible Engine is not proof that a worker stopped. + raise UpdateBusy(previous) from None + running = False + except Exception: + # An inaccessible Engine is not proof that a worker stopped. + raise UpdateBusy(previous) from None + if running: + raise UpdateBusy(previous) + previous.update(state='interrupted', success=False, + message='Update helper exited before recording completion. Inspect service status before retrying.') + write_job(directory, previous) + job_id = uuid.uuid4().hex + helper = 'cryptolabs-update-' + job_id + request = helper_request(engine.inspect('cryptolabs-proxy'), Path(directory), job_id) + job = {'id': job_id, 'helper': helper, 'service': service, 'branch': branch, + 'action': action, 'state': 'queued', 'results': {}, 'created_at': time.time()} + write_job(directory, job) + _write_json(active, {'id': job_id}) + try: + engine.create(helper, request) + engine.start(helper) + except Exception: + job.update(state='failed', success=False, message='Could not launch update helper.') + write_job(directory, job) + raise UpdateError(job['message']) from None + return job + + +def pull_target(target): + try: + result = subprocess.run(['docker', 'pull', target], capture_output=True, timeout=300) + return result.returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def run_job(directory, job_id, *, engine=None): + engine = engine or UpdateEngine() + with _lock(directory, 'operation.lock'): + job = read_job(directory, job_id) + if job.get('state') not in (None, 'queued'): + return job # A completed/interrupted operation is never replayed. + job.update(state='running', results={}) + write_job(directory, job) + names = list(SERVICES) if job['service'] == 'all' else [job['service']] + names.sort(key=lambda name: name == 'cryptolabs-proxy') + for name in names: + if SERVICES[name].get('update_supported') is False: + continue + try: + source = engine.inspect(name) + target = target_for(name, source, job['branch']) + if not source.get('State', {}).get('Running') or not target: + result = {'success': job['service'] == 'all', 'state': 'skipped', + 'message': 'Stopped or pinned service; configuration preserved.'} + elif not pull_target(target): + result = {'success': False, 'state': 'failed', 'message': 'Image pull failed; service unchanged.'} + elif job['action'] == 'pull': + image = engine.image(target) + result = {'success': True, 'state': 'checked', 'image_id': image['Id'], + 'update_available': image['Id'] != source['Image'], 'message': 'Image checked; running service unchanged.'} + else: + result = replace_container(engine, name, target, job_id, expected_id=source['Id']) + except Exception as error: + missing = isinstance(error, MigrationError) and str(error).endswith('HTTP 404') + if missing and job['service'] == 'all': + result = {'success': True, 'state': 'skipped', 'message': 'Service is not installed.'} + else: + result = {'success': False, 'state': 'failed', 'message': 'Could not inspect or prepare service; no update started.'} + job['results'][name] = result + write_job(directory, job) + if result.get('state') == 'recovery_required': + break + success = all(result['success'] for result in job['results'].values()) + job.update(state='completed' if success else 'failed', success=success, finished_at=time.time()) + write_job(directory, job) + return job + + +def main(argv=None): + """Supported JSON interface for host fleet CLI clients.""" + import argparse + parser = argparse.ArgumentParser(description='Fleet image updater') + commands = parser.add_subparsers(dest='command', required=True) + submit = commands.add_parser('submit') + submit.add_argument('--branch', choices=('main', 'dev'), default='main') + submit.add_argument('--service', default='all', choices=['all', *SERVICES]) + submit.add_argument('--action', choices=('update', 'pull'), default='update') + status = commands.add_parser('status') + status.add_argument('job_id') + worker = commands.add_parser('worker') + worker.add_argument('job_id') + worker.add_argument('directory', type=Path) + args = parser.parse_args(argv) + try: + if args.command == 'submit': + result = submit_job(args.service, args.branch, args.action) + elif args.command == 'status': + result = job_status(args.job_id) + else: + result = run_job(args.directory, args.job_id) + print(json.dumps(result)) + return 0 + except UpdateBusy as error: + print(json.dumps({'error': str(error), 'job': error.job})) + return 2 + except UpdateError as error: + print(json.dumps({'error': str(error)})) + return 1 + except Exception: + print(json.dumps({'error': 'Updater unavailable; no completion confirmed.'})) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_updates.py b/tests/test_updates.py new file mode 100644 index 0000000..b49cfb3 --- /dev/null +++ b/tests/test_updates.py @@ -0,0 +1,358 @@ +"""Updater failures must not destroy existing runtime state or conceal outcomes.""" +import importlib.util +import json +import subprocess +from copy import deepcopy +from pathlib import Path + +import pytest + +from cryptolabs_proxy import updates + + +def source(): + return { + 'Id': 'a' * 64, 'Name': '/prometheus', 'Image': 'sha256:old', + 'Config': { + 'Image': 'prom/prometheus:latest', 'Env': ['GIT_COMMIT=old', 'SECRET=keep', 'PATH=/old'], + 'Labels': {'org.opencontainers.image.revision': 'old', 'custom': 'keep'}, + 'Cmd': ['--web.route-prefix=/prometheus/', '--storage.tsdb.retention.time=90d'], + 'Entrypoint': ['/custom-entrypoint'], 'User': '123', + 'Healthcheck': {'Test': ['CMD', 'probe'], 'Interval': 10}, + }, + 'HostConfig': {'Binds': [], 'RestartPolicy': {'Name': 'on-failure', 'MaximumRetryCount': 3}, + 'PortBindings': {'999/udp': [{'HostIp': '127.0.0.1', 'HostPort': '999'}]}, + 'LogConfig': {'Type': 'json-file', 'Config': {'max-size': '10m'}}}, + 'Mounts': [{'Type': 'volume', 'Name': 'metrics', 'Destination': '/prometheus', 'RW': False}, + {'Type': 'bind', 'Source': '/site/nginx.conf', 'Destination': '/etc/nginx/nginx.conf', 'RW': False}], + 'NetworkSettings': {'Networks': {'cryptolabs': { + 'Aliases': ['prometheus', 'a' * 12], 'IPAMConfig': {'IPv4Address': '172.30.0.10'}, + 'IPAddress': '172.30.0.10', 'EndpointID': 'runtime', + }}}, + 'State': {'Running': True, 'Health': {'Status': 'healthy'}}, + } + + +def test_clone_preserves_runtime_but_refreshes_image_defaults(): + original = source() + old = {'Config': {'Env': ['GIT_COMMIT=old', 'PATH=/old'], 'Cmd': ['default'], + 'Entrypoint': ['/image-entrypoint'], 'Labels': {'org.opencontainers.image.revision': 'old'}}} + new = {'Id': 'sha256:new', 'Config': {'Env': ['GIT_COMMIT=new', 'PATH=/new'], + 'Labels': {'org.opencontainers.image.revision': 'new'}, 'Cmd': ['new-default']}} + request = updates.build_update_request(original, old, new) + assert original == source() + assert request['Config']['Image'] == 'sha256:new' + assert request['Config']['Env'] == ['GIT_COMMIT=new', 'PATH=/new', 'SECRET=keep'] + assert request['Config']['Labels'] == {'org.opencontainers.image.revision': 'new', 'custom': 'keep'} + for key in ('Cmd', 'Entrypoint', 'User', 'Healthcheck'): + assert request['Config'][key] == original['Config'][key] + assert request['HostConfig']['RestartPolicy'] == original['HostConfig']['RestartPolicy'] + assert request['HostConfig']['PortBindings'] == original['HostConfig']['PortBindings'] + assert request['HostConfig']['Mounts'][0]['Source'] == 'metrics' + assert request['HostConfig']['Mounts'][0]['ReadOnly'] is True + endpoint = request['NetworkingConfig']['EndpointsConfig']['cryptolabs'] + assert endpoint['IPAMConfig']['IPv4Address'] == '172.30.0.10' + assert endpoint['Aliases'] == ['prometheus'] + assert 'EndpointID' not in endpoint + + +def test_image_defaults_change_without_losing_custom_overrides(): + original = source() + original['Config']['Cmd'] = ['old-default'] + old = {'Config': {'Cmd': ['old-default']}} + new = {'Id': 'sha256:new', 'Config': {'Cmd': ['new-default']}} + assert updates.build_update_request(original, old, new)['Config']['Cmd'] == ['new-default'] + + +@pytest.mark.parametrize('reference', ['sha256:123', 'repo@sha256:123', 'prom/prometheus:v3.2', 'custom/prometheus:latest']) +def test_pinned_and_custom_images_are_never_silently_retargeted(reference): + current = source() + current['Config']['Image'] = reference + assert updates.target_for('prometheus', current, 'dev') is None + + +def test_channel_only_changes_first_party_images(): + assert updates.target_for('prometheus', source(), 'dev') == 'prom/prometheus:latest' + current = source() + current['Config']['Image'] = 'ghcr.io/cryptolabsza/dc-overview:latest' + assert updates.target_for('dc-overview', current, 'dev').endswith(':dev') + assert updates.target_for('vast-price-manager', current, 'main') is None + + +class Engine: + def __init__(self, failure=None): + self.current = source() + self.containers = {'prometheus': self.current} + self.calls = [] + self.failure = failure + + def inspect(self, name): + if self.failure == 'inspect': + raise RuntimeError('private detail') + if name not in self.containers: + raise updates.MigrationError('Docker Engine GET returned HTTP 404') + return deepcopy(self.containers[name]) + + def image(self, image): + return {'Id': 'sha256:old' if image == 'sha256:old' else 'sha256:new', 'Config': {}} + + def stop(self, name, **kwargs): + self.calls.append(('stop', name)) + self.containers[name]['State']['Running'] = False + + def disconnect(self, name, network): + self.calls.append(('disconnect', name, network)) + + def connect(self, name, network, endpoint): + self.calls.append(('connect', name, network)) + + def rename(self, name, new): + self.calls.append(('rename', name, new)) + self.containers[new] = self.containers.pop(name) + + def create(self, name, request): + self.calls.append(('create', name)) + if self.failure == 'create': + raise RuntimeError('private detail') + self.containers[name] = deepcopy(source()) + self.containers[name]['Config'] = request['Config'] + self.containers[name]['Image'] = request['Config']['Image'] + + def start(self, name): + self.calls.append(('start', name)) + current = self.containers[name] + current['State']['Running'] = True + if self.failure == 'health' and current['Image'] == 'sha256:new': + current['State']['Health']['Status'] = 'unhealthy' + + def remove(self, name, **kwargs): + self.calls.append(('remove', name)) + self.containers.pop(name, None) + + +@pytest.mark.parametrize('failure', ['create', 'health']) +def test_failed_update_restores_original_and_reports_failure(failure): + engine = Engine(failure) + result = updates.replace_container(engine, 'prometheus', 'sha256:new', 'job', timeout=0) + assert result['success'] is False + assert result['state'] == 'rolled_back' + assert engine.inspect('prometheus')['Image'] == 'sha256:old' + assert engine.inspect('prometheus')['State']['Running'] is True + assert 'private detail' not in json.dumps(result) + + +def test_inspection_failure_never_stops_original(): + engine = Engine('inspect') + result = updates.replace_container(engine, 'prometheus', 'sha256:new', 'job', timeout=0) + assert result['success'] is False + assert not engine.calls + + +def test_success_retains_old_container_and_verifies_new_identity(): + engine = Engine() + result = updates.replace_container(engine, 'prometheus', 'sha256:new', 'job', timeout=0) + assert result['state'] == 'completed' + assert result['image_id'] == 'sha256:new' + assert engine.inspect('prometheus.rollback-job')['Image'] == 'sha256:old' + assert not engine.inspect('prometheus.rollback-job')['State']['Running'] + + +def test_same_tag_image_identity_detects_update_and_unknown_is_not_current(): + engine = Engine() + status = updates.update_status('prometheus', 'main', engine=engine) + assert status['update_available'] is True + engine.image = lambda image: {'Id': 'sha256:old'} + assert updates.update_status('prometheus', 'main', engine=engine)['update_available'] is False + engine.image = lambda image: (_ for _ in ()).throw(RuntimeError()) + assert updates.update_status('prometheus', 'main', engine=engine)['update_available'] is None + + +def test_worker_does_not_touch_stopped_pinned_or_vpm(monkeypatch, tmp_path): + engine = Engine() + engine.current['State']['Running'] = False + monkeypatch.setattr(updates, 'SERVICES', {'prometheus': updates.SERVICES['prometheus'], + 'vast-price-manager': updates.SERVICES['vast-price-manager']}) + job = {'id': 'a' * 32, 'service': 'all', 'branch': 'main', 'action': 'update'} + updates.write_job(tmp_path, job) + result = updates.run_job(tmp_path, job['id'], engine=engine) + assert not engine.calls + assert result['results']['prometheus']['state'] == 'skipped' + assert 'vast-price-manager' not in result['results'] + + +def test_helper_runs_independently_with_persistent_state_and_no_copied_secrets(): + current = source() + current['Mounts'].append({'Type': 'volume', 'Name': 'fleet-data', 'Destination': '/data', 'RW': True}) + request = updates.helper_request(current, Path('/data/auth/update-jobs'), 'a' * 32) + assert request['Config']['Image'] == 'sha256:old' + assert request['Config']['Entrypoint'] == ['python3'] + assert request['Config']['Cmd'][:2] == ['-m', 'cryptolabs_proxy.updates'] + assert request['HostConfig']['NetworkMode'] == 'host' + assert request['Config']['Env'] == ['PYTHONPATH=/app/src'] + assert any(m.get('Source') == 'fleet-data' for m in request['HostConfig']['Mounts']) + assert request['Config']['Healthcheck'] == {'Test': ['NONE']} + + +def test_update_api_queues_job_instead_of_mutating_docker(monkeypatch): + path = Path(__file__).parents[1] / 'scripts' / 'health-api.py' + spec = importlib.util.spec_from_file_location('health_api_updates', path) + api = importlib.util.module_from_spec(spec) + spec.loader.exec_module(api) + import io + body = json.dumps({'service': 'prometheus', 'branch': 'main'}).encode() + handler = object.__new__(api.HealthHandler) + handler.path = '/api/update' + handler.headers = {'Content-Length': str(len(body))} + handler.rfile = io.BytesIO(body) + replies = [] + handler.send_json = lambda result, status=200: replies.append((result, status)) + monkeypatch.setattr(api, 'submit_update_job', lambda *args, **kwargs: {'id': 'job', 'state': 'queued'}) + handler.do_POST() + assert replies == [({'success': True, 'job': {'id': 'job', 'state': 'queued'}}, 202)] + + +def test_cli_submit_and_status_return_machine_readable_jobs(monkeypatch, capsys): + monkeypatch.setattr(updates, 'submit_job', lambda *args, **kwargs: {'id': 'a' * 32, 'state': 'queued'}) + assert updates.main(['submit', '--branch', 'dev']) == 0 + assert json.loads(capsys.readouterr().out)['state'] == 'queued' + monkeypatch.setattr(updates, 'job_status', lambda *args, **kwargs: {'id': 'a' * 32, 'state': 'failed', 'success': False}) + assert updates.main(['status', 'a' * 32]) == 0 + assert json.loads(capsys.readouterr().out)['success'] is False + + +def test_bulk_missing_service_is_skipped_and_partial_failure_is_reported(monkeypatch, tmp_path): + engine = Engine() + monkeypatch.setattr(updates, 'pull_target', lambda target: False) + monkeypatch.setattr(updates, 'SERVICES', {name: updates.SERVICES[name] for name in ['prometheus', 'runpod-exporter']}) + job = {'id': 'a' * 32, 'service': 'all', 'branch': 'main', 'action': 'update'} + updates.write_job(tmp_path, job) + result = updates.run_job(tmp_path, job['id'], engine=engine) + assert result['success'] is False + assert result['results']['prometheus']['success'] is False + assert result['results']['runpod-exporter']['state'] == 'skipped' + assert not engine.calls + + +def test_new_managed_image_is_not_misclassified_as_pinned(): + engine = Engine() + result = updates.replace_container(engine, 'prometheus', 'prom/prometheus:latest', 'job', timeout=0) + assert result['success'] is True + assert updates.target_for('prometheus', engine.inspect('prometheus'), 'main') == 'prom/prometheus:latest' + + +def test_no_healthcheck_requires_real_application_probe(monkeypatch): + engine = Engine() + engine.current['State'].pop('Health') + monkeypatch.setattr(updates, '_probe', lambda *args: False) + assert updates.wait_ready(engine, 'prometheus', 'sha256:old', timeout=0) is False + + +def test_duplicate_submission_does_not_launch_second_helper(monkeypatch, tmp_path): + engine = Engine() + job = {'id': 'a' * 32, 'state': 'running', 'helper': 'prometheus'} + updates.write_job(tmp_path, job) + (tmp_path / 'active.json').write_text(json.dumps({'id': job['id']})) + with pytest.raises(updates.UpdateBusy): + updates.submit_job('all', 'main', 'update', directory=tmp_path, engine=engine) + assert not engine.calls + + +def test_missing_helper_is_recorded_as_interrupted_not_left_active(tmp_path): + job = {'id': 'a' * 32, 'state': 'running', 'helper': 'missing-helper'} + updates.write_job(tmp_path, job) + result = updates.job_status(job['id'], directory=tmp_path, engine=Engine()) + assert result['state'] == 'interrupted' + assert result['success'] is False + assert 'stopped before completion' in result['message'] + assert updates.read_job(tmp_path, job['id'])['state'] == 'interrupted' + + +def test_submit_recovers_a_missing_previous_helper(tmp_path): + engine = Engine() + engine.current['Mounts'].append({ + 'Type': 'volume', 'Name': 'fleet-data', 'Destination': str(tmp_path.parent), 'RW': True, + }) + engine.containers['cryptolabs-proxy'] = engine.current + previous = {'id': 'a' * 32, 'state': 'running', 'helper': 'missing-helper'} + updates.write_job(tmp_path, previous) + (tmp_path / 'active.json').write_text(json.dumps({'id': previous['id']})) + + submitted = updates.submit_job('all', 'main', 'pull', directory=tmp_path, engine=engine) + + assert submitted['id'] != previous['id'] + assert updates.read_job(tmp_path, previous['id'])['state'] == 'interrupted' + assert ('create', submitted['helper']) in engine.calls + + +def test_submit_keeps_job_busy_when_docker_cannot_confirm_helper_state(tmp_path): + previous = {'id': 'a' * 32, 'state': 'running', 'helper': 'missing-helper'} + updates.write_job(tmp_path, previous) + (tmp_path / 'active.json').write_text(json.dumps({'id': previous['id']})) + + with pytest.raises(updates.UpdateBusy): + updates.submit_job('all', 'main', 'pull', directory=tmp_path, engine=Engine('inspect')) + + +def test_update_engine_uses_docker_29_compatible_api(monkeypatch): + calls = [] + + def request(self, method, path, payload=None, allowed=(200, 201, 204)): + calls.append((method, path, payload, allowed)) + return b'{}' + + monkeypatch.setattr(updates.DockerEngine, '_request', request) + updates.UpdateEngine()._request('GET', '/containers/example/json') + assert calls[0][1] == '/v1.45/containers/example/json' + + +def landing_script(): + html = (Path(__file__).parents[1] / 'landing-page/index.html').read_text() + return html.rsplit('