diff --git a/Dockerfile b/Dockerfile index a6f76d4d7..83cfac6ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ WORKDIR /app # Set production ENV NODE_ENV=production -RUN apt-get update && apt-get install -y curl unzip zip apache2-utils iproute2 rsync git-lfs && git lfs install && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y curl unzip zip apache2-utils iproute2 rsync git-lfs wireguard-tools iptables && git lfs install && rm -rf /var/lib/apt/lists/* # Nomad CLI — the deploy pipeline runs `nomad job run` to submit jobs to the # control plane's own Nomad. (Remote-server deploys use that server's own CLI.) diff --git a/apps/dokploy/__test__/nomad/nomad-builder.test.ts b/apps/dokploy/__test__/nomad/nomad-builder.test.ts index bd20e7033..5e90eccaa 100644 --- a/apps/dokploy/__test__/nomad/nomad-builder.test.ts +++ b/apps/dokploy/__test__/nomad/nomad-builder.test.ts @@ -66,7 +66,7 @@ describe("nomad builder — compose → HCL (live)", () => { // getBuildNomadCommand embeds the HCL as base64 in the deploy script. const match = cmd.match(/echo "([A-Za-z0-9+/=]+)" \| base64 -d/); expect(match).not.toBeNull(); - const hcl = Buffer.from(match![1], "base64").toString("utf8"); + const hcl = Buffer.from(match?.[1] ?? "", "base64").toString("utf8"); // Print it so the translation is visible when running the test. console.log("\n===== generated Nomad HCL =====\n" + hcl + "\n===============================\n"); diff --git a/apps/dokploy/components/dashboard/settings/servers/nomad-settings.tsx b/apps/dokploy/components/dashboard/settings/servers/nomad-settings.tsx index 391d5c856..eb5dfb6e6 100644 --- a/apps/dokploy/components/dashboard/settings/servers/nomad-settings.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/nomad-settings.tsx @@ -1,5 +1,5 @@ import { zodResolver } from "@hookform/resolvers/zod"; -import { Loader2, Terminal } from "lucide-react"; +import { Loader2, Network, Terminal } from "lucide-react"; import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; @@ -73,6 +73,34 @@ export const NomadSettings = ({ serverId }: Props) => { setIsBootstrapping(true); }; + const [isJoining, setIsJoining] = useState(false); + const [joinLogs, setJoinLogs] = useState(""); + + api.nomad.joinCluster.useSubscription( + { serverId }, + { + enabled: isJoining, + onData(log) { + if (log === "JOIN_DONE") { + setIsJoining(false); + toast.success("Server joined the Nomad cluster"); + refetch(); + return; + } + setJoinLogs((prev) => prev + log); + }, + onError(error) { + setIsJoining(false); + toast.error(error.message || "Cluster join failed"); + }, + }, + ); + + const startJoin = () => { + setJoinLogs(""); + setIsJoining(true); + }; + const form = useForm({ resolver: zodResolver(nomadSchema), values: { @@ -107,20 +135,35 @@ export const NomadSettings = ({ serverId }: Props) => { Configure Nomad cluster connection for deploying services. - +
+ + +
@@ -208,6 +251,11 @@ export const NomadSettings = ({ serverId }: Props) => { {bootstrapLogs || "Starting bootstrap…"} )} + {(isJoining || joinLogs) && ( + 
+						{joinLogs || "Joining cluster…"}
+
+ )} ); diff --git a/apps/dokploy/server/api/routers/nomad.ts b/apps/dokploy/server/api/routers/nomad.ts index f93d44a7a..e8c9e03ee 100644 --- a/apps/dokploy/server/api/routers/nomad.ts +++ b/apps/dokploy/server/api/routers/nomad.ts @@ -1,11 +1,27 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { findServerById, updateServerById } from "@nomploy/server"; import { getNomadBootstrapCommand } from "@nomploy/server/setup/nomad-bootstrap"; -import { execAsyncRemote } from "@nomploy/server/utils/process/execAsync"; +import { getClusterWorkerJoinCommand } from "@nomploy/server/setup/nomad-cluster"; +import { + execAsync, + execAsyncRemote, +} from "@nomploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { observable } from "@trpc/server/observable"; import { z } from "zod"; import { createTRPCRouter, withPermission } from "../trpc"; +// Cluster state written by the installer's hub setup and updated as nodes join. +const CLUSTER_FILE = "/etc/nomploy/cluster.json"; +interface ClusterState { + hubPublicKey: string; + gossipKey: string; + hubWgIp: string; + hubEndpoint: string; + overlayCidr?: string; + peers: { wgIp: string; publicKey: string; serverId: string; name: string }[]; +} + // Control-plane-local Nomad (used when no serverId is given). const DEFAULT_ADDRESS = process.env.NOMAD_ADDRESS || "http://127.0.0.1:4646"; const DEFAULT_TOKEN = process.env.NOMAD_TOKEN || ""; @@ -316,4 +332,90 @@ export const nomadRouter = createTRPCRouter({ }); }); }), + + // Join a server to the Nomad cluster over the WireGuard mesh: install + + // configure it as a Consul/Nomad client, then register its WireGuard peer on + // the hub (this control plane). Streams progress. + joinCluster: withPermission("server", "create") + .input(z.object({ serverId: z.string() })) + .subscription(async ({ input, ctx }) => { + const server = await findServerById(input.serverId); + if (server.organizationId !== ctx.session?.activeOrganizationId) { + throw new TRPCError({ code: "UNAUTHORIZED" }); + } + + return observable((emit) => { + (async () => { + try { + if (!existsSync(CLUSTER_FILE)) { + emit.next( + "❌ Cluster not initialized on the control plane (missing /etc/nomploy/cluster.json).\n", + ); + emit.complete(); + return; + } + const cluster: ClusterState = JSON.parse( + readFileSync(CLUSTER_FILE, "utf8"), + ); + cluster.peers = cluster.peers || []; + + // Allocate the next free overlay IP (hub keeps .1). + const prefix = cluster.hubWgIp.replace(/\.\d+$/, ""); + const used = new Set([ + cluster.hubWgIp, + ...cluster.peers.map((p) => p.wgIp), + ]); + let n = 2; + while (used.has(`${prefix}.${n}`)) n++; + const wgIp = `${prefix}.${n}`; + emit.next(`Assigning overlay IP ${wgIp} to "${server.name}"\n`); + + const script = getClusterWorkerJoinCommand({ + hubPublicKey: cluster.hubPublicKey, + hubEndpoint: cluster.hubEndpoint, + gossipKey: cluster.gossipKey, + workerWgIp: wgIp, + hubWgIp: cluster.hubWgIp, + overlayCidr: cluster.overlayCidr, + }); + + let pubkey = ""; + await execAsyncRemote(input.serverId, script, (log) => { + emit.next(log); + const cap = log.match(/WORKER_WG_PUBKEY=(\S+)/)?.[1]; + if (cap) pubkey = cap.trim(); + }); + if (!pubkey) { + emit.next("\n❌ Did not receive the worker's WireGuard key\n"); + emit.complete(); + return; + } + + emit.next(`\nRegistering WireGuard peer on the hub (${wgIp})\n`); + await execAsync( + `wg set wg0 peer ${pubkey} allowed-ips ${wgIp}/32 && wg-quick save wg0`, + ); + + cluster.peers.push({ + wgIp, + publicKey: pubkey, + serverId: input.serverId, + name: server.name, + }); + writeFileSync(CLUSTER_FILE, JSON.stringify(cluster, null, 2)); + await updateServerById(input.serverId, { + nomadAddress: `http://${wgIp}:4646`, + }); + + emit.next("JOIN_DONE"); + emit.complete(); + } catch (err: unknown) { + const message = + err instanceof Error ? err.message : "Cluster join failed"; + emit.next(`\n❌ ${message}\n`); + emit.complete(); + } + })(); + }); + }), }); diff --git a/install.sh b/install.sh index a6aebc079..c63766623 100755 --- a/install.sh +++ b/install.sh @@ -92,34 +92,67 @@ echo 1 | $SUDO tee /proc/sys/net/bridge/bridge-nf-call-iptables >/dev/null 2>&1 # ── Consul + Nomad config (single node: server + client) ───────────────────── $SUDO mkdir -p /etc/consul.d /opt/consul /etc/nomad.d /opt/nomad -# Single-node all-in-one: bind everything to 127.0.0.1. This avoids the -# multiple-private-IP ambiguity ({{ GetPrivateIP }} can pick the docker bridge) -# and keeps Nomad/Consul internal — only Traefik is exposed publicly. -$SUDO tee /etc/consul.d/consul.hcl >/dev/null <<'CONSULHCL' +# ── WireGuard hub (cluster overlay 10.10.0.0/24) ───────────────────────────── +# The control plane is the WireGuard hub; Consul/Nomad servers bind to the hub's +# overlay IP so worker nodes can join over an encrypted mesh. Consul/Nomad HTTP +# APIs still answer on 127.0.0.1 for the local app. Endpoint workers dial defaults +# to this host's public IP (open UDP 51820); override with NOMPLOY_WG_ENDPOINT. +$SUDO mkdir -p /etc/nomploy +if command -v apt-get >/dev/null 2>&1; then + $SUDO apt-get install -y wireguard wireguard-tools >/dev/null 2>&1 || true +else + $SUDO yum install -y wireguard-tools >/dev/null 2>&1 || true +fi +$SUDO mkdir -p /etc/wireguard && $SUDO chmod 700 /etc/wireguard +if [ ! -s /etc/wireguard/hub_priv ]; then + $SUDO sh -c 'wg genkey > /etc/wireguard/hub_priv && chmod 600 /etc/wireguard/hub_priv && wg pubkey < /etc/wireguard/hub_priv > /etc/wireguard/hub_pub' +fi +HUB_PUB="$($SUDO cat /etc/wireguard/hub_pub)" +if [ ! -f /etc/wireguard/wg0.conf ]; then + $SUDO tee /etc/wireguard/wg0.conf >/dev/null </dev/null 2>&1 || $SUDO wg-quick up wg0 || true +$SUDO systemctl enable wg-quick@wg0 >/dev/null 2>&1 || true + +# Shared gossip encryption key (Consul + Nomad). +[ -s /etc/nomploy/gossip.key ] || consul keygen | $SUDO tee /etc/nomploy/gossip.key >/dev/null +GOSSIP="$($SUDO cat /etc/nomploy/gossip.key)" +WG_ENDPOINT="${NOMPLOY_WG_ENDPOINT:-$(ip route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}'):51820}" + +$SUDO tee /etc/consul.d/consul.hcl >/dev/null </dev/null <<'NOMADHCL' +$SUDO tee /etc/nomad.d/nomad.hcl >/dev/null </dev/null </dev/null 2>&1 || true $SUDO docker run -d --name nomploy --restart unless-stopped \ --network host \ + --cap-add NET_ADMIN \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /etc/nomploy:/etc/nomploy \ + -v /etc/wireguard:/etc/wireguard \ -e NODE_ENV=production \ -e PORT="$NOMPLOY_PORT" \ -e DATABASE_URL="postgresql://nomploy:${POSTGRES_PASSWORD}@127.0.0.1:5432/nomploy" \ diff --git a/packages/server/src/setup/nomad-cluster.ts b/packages/server/src/setup/nomad-cluster.ts new file mode 100644 index 000000000..856484f9e --- /dev/null +++ b/packages/server/src/setup/nomad-cluster.ts @@ -0,0 +1,149 @@ +/** + * nomploy — multi-node cluster (WireGuard mesh). + * + * The control plane is a WireGuard "hub" (wg0 = 10.10.0.1/24, UDP 51820) running + * the Consul + Nomad servers, bound to their WireGuard IP. Worker nodes get a + * WireGuard IP on the same overlay, peer with the hub, and run Consul + Nomad as + * clients that join over the encrypted overlay. + * + * `getClusterWorkerJoinCommand` produces a single script (run over SSH on a + * fresh worker) that installs Docker + Consul + Nomad + CNI + WireGuard, brings + * up the overlay, prints its WireGuard public key (the caller then registers it + * as a peer on the hub with `wg set wg0 peer allowed-ips /32`), and + * starts Consul + Nomad as clients that retry-join the hub. Consul/Nomad keep + * retrying, so they connect as soon as the hub adds the peer. + * + * The worker's public key is emitted on its own line as: + * WORKER_WG_PUBKEY= + * so the orchestrator can parse it from the streamed output. + */ + +export interface ClusterWorkerJoinOptions { + /** WireGuard public key of the hub (control plane). */ + hubPublicKey: string; + /** How the worker reaches the hub's WireGuard endpoint, host:port. */ + hubEndpoint: string; + /** Shared Consul/Nomad gossip encryption key (base64, from `consul keygen`). */ + gossipKey: string; + /** WireGuard overlay IP assigned to this worker, e.g. "10.10.0.2". */ + workerWgIp: string; + /** Hub's WireGuard overlay IP (Consul/Nomad servers), default "10.10.0.1". */ + hubWgIp?: string; + /** WireGuard overlay CIDR, default "10.10.0.0/24". */ + overlayCidr?: string; + datacenter?: string; + cniVersion?: string; +} + +export const getClusterWorkerJoinCommand = ( + opts: ClusterWorkerJoinOptions, +): string => { + const hubWgIp = opts.hubWgIp || "10.10.0.1"; + const overlayCidr = opts.overlayCidr || "10.10.0.0/24"; + const datacenter = opts.datacenter || "dc1"; + const cniVersion = opts.cniVersion || "v1.5.1"; + + return ` +set -e +if [ "$EUID" -ne 0 ] && ! sudo -n true 2>/dev/null; then + echo "Error: needs root or passwordless sudo. ❌"; exit 1 +fi +SUDO=""; [ "$EUID" -ne 0 ] && SUDO="sudo" +export DEBIAN_FRONTEND=noninteractive +OS_TYPE=$(grep -w "ID" /etc/os-release | cut -d= -f2 | tr -d '"') +ARCH=$(uname -m); case "$ARCH" in x86_64) CNI_ARCH=amd64;; aarch64|arm64) CNI_ARCH=arm64;; *) echo "unsupported arch $ARCH"; exit 1;; esac + +echo "==> Installing Docker" +command -v docker >/dev/null 2>&1 || { curl -fsSL https://get.docker.com | $SUDO sh; $SUDO systemctl enable --now docker; } + +echo "==> Installing Consul + Nomad + WireGuard" +if ! command -v nomad >/dev/null 2>&1 || ! command -v consul >/dev/null 2>&1 || ! command -v wg >/dev/null 2>&1; then + case "$OS_TYPE" in + ubuntu|debian|raspbian|pop|linuxmint|zorin) + $SUDO apt-get update -y + $SUDO apt-get install -y curl gnupg lsb-release wireguard wireguard-tools + curl -fsSL https://apt.releases.hashicorp.com/gpg | $SUDO gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | $SUDO tee /etc/apt/sources.list.d/hashicorp.list + $SUDO apt-get update -y; $SUDO apt-get install -y nomad consul ;; + centos|rhel|rocky|almalinux|fedora|amzn|ol) + $SUDO yum install -y yum-utils wireguard-tools + $SUDO yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo + $SUDO yum -y install nomad consul ;; + *) echo "unsupported OS $OS_TYPE"; exit 1 ;; + esac +fi + +# CNI plugins +if [ ! -f /opt/cni/bin/bridge ]; then + $SUDO mkdir -p /opt/cni/bin + curl -fsSL "https://github.com/containernetworking/plugins/releases/download/${cniVersion}/cni-plugins-linux-\${CNI_ARCH}-${cniVersion}.tgz" | $SUDO tar -C /opt/cni/bin -xz +fi + +# Docker auth config (docker driver needs this file to exist, even for public pulls) +$SUDO mkdir -p /root/.docker +[ -s /root/.docker/config.json ] || echo '{"auths":{}}' | $SUDO tee /root/.docker/config.json >/dev/null + +# ── WireGuard: join the overlay, peer with the hub ───────────────────────── +$SUDO mkdir -p /etc/wireguard && $SUDO chmod 700 /etc/wireguard +$SUDO rm -f /etc/wireguard/w_priv /etc/wireguard/w_pub +$SUDO sh -c 'wg genkey > /etc/wireguard/w_priv && chmod 600 /etc/wireguard/w_priv && wg pubkey < /etc/wireguard/w_priv > /etc/wireguard/w_pub' +$SUDO tee /etc/wireguard/wg0.conf >/dev/null </dev/null 2>&1 || true +$SUDO wg-quick up wg0 +$SUDO systemctl enable wg-quick@wg0 >/dev/null 2>&1 || true +echo "WORKER_WG_PUBKEY=$($SUDO cat /etc/wireguard/w_pub)" + +# ── Consul client ────────────────────────────────────────────────────────── +$SUDO mkdir -p /etc/consul.d /opt/consul /etc/nomad.d /opt/nomad +$SUDO tee /etc/consul.d/consul.hcl >/dev/null </dev/null < Starting Consul + Nomad clients" +$SUDO systemctl enable consul nomad >/dev/null 2>&1 || true +$SUDO systemctl restart --no-block consul +sleep 3 +$SUDO systemctl restart --no-block nomad +echo "==> Worker join complete. It will register once the hub adds its WireGuard peer." +`; +};