diff --git a/bin/render_harness.sh b/bin/render_harness.sh new file mode 100755 index 00000000..c06db850 --- /dev/null +++ b/bin/render_harness.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# Deterministic-render diff harness. +# +# Renders the AWS terraform + ansible set into with every random input +# pinned, so two renders of the SAME revision are byte-identical and any diff +# between two revisions is a real output change. +# +# Two-revision recipe: +# +# git checkout && bash bin/render_harness.sh /tmp/base +# git checkout && bash bin/render_harness.sh /tmp/head +# diff -r /tmp/base /tmp/head # empty == no AWS output change +# +# The operator (or CI) supplies the two commits; this script renders one. +set -euo pipefail + +OUT_DIR="${1:?usage: render_harness.sh }" + +PINNED_PEM_APP_NAME="render-harness-pinned" +PINNED_DB_PASSWORD="RenderHarnessPinnedPassword" + +# Never rm -rf the caller's argument. An earlier version did, behind a guard that only +# rejected relative paths and directories containing .git/mix.exs — so `/`, `$HOME`, `/opt` +# and any other absolute path all sailed through it. Instead, refuse to touch anything that +# already exists and delete only the two subdirectories this script creates itself. +case "$OUT_DIR" in + /*) ;; + *) echo "render_harness.sh: must be an absolute path, got '$OUT_DIR'" >&2; exit 2 ;; +esac + +if [ -e "$OUT_DIR" ] && [ ! -d "$OUT_DIR/terraform" ] && [ ! -d "$OUT_DIR/ansible" ]; then + echo "render_harness.sh: '$OUT_DIR' already exists and is not a previous render dir." >&2 + echo " Refusing to touch it. Pass a fresh path." >&2 + exit 2 +fi + +mkdir -p "$OUT_DIR" +rm -rf "${OUT_DIR:?}/terraform" "${OUT_DIR:?}/ansible" + +mix terraform.build \ + --render-dir "$OUT_DIR/terraform" \ + --pem-app-name "$PINNED_PEM_APP_NAME" \ + --db-password "$PINNED_DB_PASSWORD" \ + --quiet < /dev/null + +mix ansible.build --render-dir "$OUT_DIR/ansible" --quiet < /dev/null diff --git a/deploys/ansible/aws_ec2.yaml b/deploys/ansible/aws_ec2.yaml deleted file mode 100644 index c821af51..00000000 --- a/deploys/ansible/aws_ec2.yaml +++ /dev/null @@ -1,34 +0,0 @@ -plugin: aws_ec2 - -regions: - - us-west-2 - -hostnames: - - tag:Name - -filters: - tag:Group: "Deploy Ex Backend" - -keyed_groups: - - key: tags['MonitoringKey'] - prefix: "monitoring" - - - key: tags['InstanceGroup'] - prefix: "group" - - - key: tags['DatabaseKey'] - prefix: "database" - - - key: tags['QaNode'] - prefix: "qa" - - -compose: - ansible_host: network_interfaces[0].ipv6_addresses[0].ipv6_address | default(public_ip_address, true) - letsencrypt_use_public_ip: (tags['UsePublicIpCert'] | default('false')) == 'true' - release_prefix: "'qa' if (tags['QaNode'] | default('false')) == 'true' else ''" - release_state_prefix: "'release-state/qa' if (tags['QaNode'] | default('false')) == 'true' else 'release-state'" - git_branch: tags['GitBranch'] | default('') - instance_tag: tags['InstanceTag'] | default('') - - diff --git a/deploys/ansible/roles/save_ami/tasks/main.yaml b/deploys/ansible/roles/save_ami/tasks/main.yaml deleted file mode 100644 index 41320a07..00000000 --- a/deploys/ansible/roles/save_ami/tasks/main.yaml +++ /dev/null @@ -1,89 +0,0 @@ -- name: save_ami - block: - - name: Get instance metadata - uri: - url: http://169.254.169.254/latest/meta-data/instance-id - return_content: yes - register: instance_id_result - - - name: Get AWS region - uri: - url: http://169.254.169.254/latest/meta-data/placement/region - return_content: yes - register: region_result - - - name: Set instance and region facts - set_fact: - instance_id: "{{ instance_id_result.content }}" - aws_region: "{{ region_result.content }}" - - - name: Clean cloud-init state so it re-runs on new instances - command: cloud-init clean --logs - - - name: Create AMI from current instance - command: > - aws ec2 create-image - --region {{ aws_region }} - --instance-id {{ instance_id }} - --name "{{ app_name }}-{{ env }}-{{ ansible_date_time.epoch }}" - --description "Auto-generated AMI for {{ app_name }} in {{ env }} at {{ ansible_date_time.iso8601 }}" - --tag-specifications 'ResourceType=image,Tags=[ - {Key=Name,Value={{ app_name }}-{{ env }}}, - {Key=App,Value={{ app_name }}}, - {Key=Environment,Value={{ env }}}, - {Key=CreatedAt,Value={{ ansible_date_time.iso8601 }}}, - {Key=ManagedBy,Value=DeployEx}, - {Key=Type,Value=AutoScaleReady} - ]' - --no-reboot - register: ami_creation_result - - - name: Parse AMI ID from result - set_fact: - new_ami_id: "{{ (ami_creation_result.stdout | from_json).ImageId }}" - - - name: Store AMI ID in SSM Parameter Store - command: > - aws ssm put-parameter - --region {{ aws_region }} - --name "/deploy_ex/{{ env }}/{{ app_name }}/latest_ami" - --value "{{ new_ami_id }}" - --type String - --overwrite - --description "Latest AMI for {{ app_name }} in {{ env }}" - - - name: Wait for AMI to be available (async, non-blocking) - command: > - aws ec2 wait image-available - --region {{ aws_region }} - --image-ids {{ new_ami_id }} - async: 1800 - poll: 0 - register: ami_wait_task - - - name: Log AMI creation success - debug: - msg: "Created AMI {{ new_ami_id }} for {{ app_name }}-{{ env }}. AMI will be available in ~5-10 minutes." - - - name: Cleanup old AMIs (keep last 3) - shell: | - aws ec2 describe-images \ - --region {{ aws_region }} \ - --owners self \ - --filters "Name=tag:App,Values={{ app_name }}" "Name=tag:Environment,Values={{ env }}" "Name=tag:ManagedBy,Values=DeployEx" \ - --query 'Images | sort_by(@, &CreationDate) | [:-3].[ImageId]' \ - --output text | while read ami_id; do - if [ -n "$ami_id" ]; then - echo "Deregistering old AMI: $ami_id" - aws ec2 deregister-image --region {{ aws_region }} --image-id $ami_id || true - fi - done - register: cleanup_result - ignore_errors: yes - - - name: Log cleanup result - debug: - msg: "{{ cleanup_result.stdout_lines }}" - when: cleanup_result.stdout_lines | length > 0 - - become: true diff --git a/guides/reference/terraform_variables.md b/guides/reference/terraform_variables.md index 2be13ca3..5a8569d2 100644 --- a/guides/reference/terraform_variables.md +++ b/guides/reference/terraform_variables.md @@ -88,12 +88,20 @@ my_app_project = { ### `load_balancer` +Listener and forwarded ports are **fixed at 80 (and 443 when `enable_https = true`) on both +providers** — there is no key that changes them. AWS declares `port` / `instance_port` in its +schema but never reads them (dead keys, kept only for backward compatibility); OCI's schema has +no such keys at all. + +`enable_https` defaults to `true` on both providers — the examples below set it `false` +deliberately (no TLS cert wired up yet). + +**AWS** + ```hcl load_balancer = { enable = true enable_https = false - port = 80 - instance_port = 4000 health_check = { path = "/health" @@ -111,9 +119,7 @@ load_balancer = { | Field | Default | Notes | |-------|---------|-------| | `enable` | `false` | Required when `instance_count > 1` or autoscaling | -| `enable_https` | `false` | Adds a 443 listener; needs an ACM cert (set up separately) | -| `port` | `80` | LB listener port (the URL clients hit) | -| `instance_port` | `4000` | Forwarded port on the instance | +| `enable_https` | `true` | Set `false` to skip the 443 listener; needs an ACM cert (set up separately) | | `health_check.path` | `/` | Endpoint hit by the LB | | `health_check.matcher` | `200-299,301` | HTTP status codes considered healthy | | `health_check.unhealthy_threshold` | `2` | Failed checks before unhealthy | @@ -121,6 +127,41 @@ load_balancer = { | `health_check.timeout` | `5` | Seconds per check | | `health_check.interval` | `20` | Seconds between checks | +**OCI** + +```hcl +load_balancer = { + enable = true + enable_https = false + reserved_ip_ocid = null + + health_check = { + path = "/health" + return_code = 200 + https_return_code = 200 + unhealthy_threshold = 3 + timeout = 3 + interval = 10 + } +} +``` + +| Field | Default | Notes | +|-------|---------|-------| +| `enable` | `false` | Gates on this alone — OCI has no autoscaling and no `instance_count > 1` requirement the way AWS does | +| `enable_https` | `true` | Set `false` to skip the 443 listener/backend set/NSG rule | +| `reserved_ip_ocid` | `null` | OCI-only. Pins the NLB's public IP to a pre-created reserved IP; unset leaves it ephemeral | +| `health_check.path` | `""` | Empty performs a TCP connect-only check; set performs an HTTP/HTTPS check | +| `health_check.return_code` / `https_return_code` | `200` | OCI-only — a single expected status code, not a range like AWS's `matcher` | +| `health_check.unhealthy_threshold` | `3` | Retries before a backend flips unhealthy (and before recovering it — OCI has one threshold, not two) | +| `health_check.timeout` | `3` | Seconds per check | +| `health_check.interval` | `10` | Seconds between checks | + +AWS keys ignored on OCI: `port`, `instance_port`, `health_check.protocol`, `health_check.matcher`, +`health_check.https_matcher`, `health_check.healthy_threshold` — see +`priv/terraform/providers/oci/README.md` for the full AWS-key-to-OCI mapping, the reason each is +ignored, and the per-app NSG security posture. + ### `autoscaling` ```hcl @@ -280,8 +321,6 @@ my_app = { instance_count = 2 # bump capacity first load_balancer = { enable = true - port = 80 - instance_port = 4000 health_check = { path = "/health" } } } @@ -308,7 +347,7 @@ my_app = { desired_capacity = 3 cpu_target_percent = 60 } - load_balancer = { enable = true, port = 80, instance_port = 4000 } + load_balancer = { enable = true } } ``` diff --git a/lib/deploy_ex/aws_bucket.ex b/lib/deploy_ex/aws_bucket.ex index d3576398..4e20204e 100644 --- a/lib/deploy_ex/aws_bucket.ex +++ b/lib/deploy_ex/aws_bucket.ex @@ -1,105 +1,44 @@ defmodule DeployEx.AwsBucket do - alias ExAws.S3 + @moduledoc """ + Region-first bucket helpers for the terraform state-bucket tasks. - @type bucket_res :: %{name: String.t, creation_date: String.t} + The S3 calls now live in `DeployEx.Cloud.S3ObjectStore`, which implements + `DeployEx.Cloud.ObjectStore`. This module stays because its call sites pass the region as the + FIRST argument, which the provider-neutral behaviour does not — it takes region in opts. + """ + + alias DeployEx.Cloud.S3ObjectStore + + @type bucket_res :: %{name: String.t(), creation_date: String.t()} @spec create_bucket(String.t()) :: ErrorMessage.t_res(any) @spec create_bucket(String.t(), String.t()) :: ErrorMessage.t_res(any) def create_bucket(region \\ DeployEx.Config.aws_region(), bucket_name) do - case ExAws.request(S3.put_bucket(bucket_name, region), region: region) do - {:ok, _} -> :ok - {:error, {:http_error, code, message}} -> - {:error, handle_error(code, message, %{bucket: bucket_name})} - end + S3ObjectStore.create_container(bucket_name, region: region) end @spec list_buckets() :: ErrorMessage.t_res(bucket_res) @spec list_buckets(String.t()) :: ErrorMessage.t_res(bucket_res) def list_buckets(region \\ DeployEx.Config.aws_region()) do - case ExAws.request(S3.list_buckets(), region: region) do - {:ok, %{body: %{buckets: buckets}}} -> {:ok, buckets} - {:error, {:http_error, code, message}} -> - {:error, handle_error(code, message, %{region: region})} - end + S3ObjectStore.list_containers(region: region) end def list_objects(region \\ DeployEx.Config.aws_region(), bucket_name) do - case ExAws.request(S3.list_objects(bucket_name), region: region) do - {:ok, _} = res -> res - {:error, {:http_error, code, message}} -> - {:error, handle_error(code, message, %{region: region, bucket: bucket_name})} - end + S3ObjectStore.list_objects(bucket_name, region: region) end - def delete_all_objects(region \\ DeployEx.Config.aws_region(), bucket_name, continuation_token \\ nil) do - list_opts = if continuation_token, do: [continuation_token: continuation_token], else: [] - - case ExAws.request(S3.list_objects_v2(bucket_name, list_opts), region: region) do - {:ok, %{body: %{contents: objects, is_truncated: is_truncated, next_continuation_token: next_token}}} when objects !== [] -> - object_keys = Enum.map(objects, & &1.key) - - case ExAws.request(S3.delete_multiple_objects(bucket_name, object_keys), region: region) do - {:ok, _} -> - if is_truncated do - delete_all_objects(region, bucket_name, next_token) - else - :ok - end - {:error, {:http_error, code, message}} -> - {:error, handle_error(code, message, %{region: region, bucket: bucket_name})} - end + @doc """ + Empties a bucket entirely. - {:ok, %{body: %{contents: [], is_truncated: is_truncated, next_continuation_token: next_token}}} -> - if is_truncated do - delete_all_objects(region, bucket_name, next_token) - else - :ok - end - - {:ok, %{body: %{contents: objects}}} when objects !== [] -> - object_keys = Enum.map(objects, & &1.key) - - case ExAws.request(S3.delete_multiple_objects(bucket_name, object_keys), region: region) do - {:ok, _} -> :ok - {:error, {:http_error, code, message}} -> - {:error, handle_error(code, message, %{region: region, bucket: bucket_name})} - end - - {:ok, _} -> :ok - {:error, {:http_error, code, message}} -> - {:error, handle_error(code, message, %{region: region, bucket: bucket_name})} - end + `all: true` is passed deliberately — the object store refuses an unscoped delete, and emptying + the bucket is exactly what this function is for. Its only caller drops the terraform state + bucket, whose name comes from config rather than an argument. + """ + def delete_all_objects(region \\ DeployEx.Config.aws_region(), bucket_name, continuation_token \\ nil) do + S3ObjectStore.delete_all_objects(bucket_name, [region: region, all: true], continuation_token) end def delete_bucket(region \\ DeployEx.Config.aws_region(), bucket_name) do - case ExAws.request(S3.delete_bucket(bucket_name), region: region) do - {:ok, _} -> :ok - {:error, {:http_error, code, message}} -> - {:error, handle_error(code, message, %{region: region, bucket: bucket_name})} - end - end - - defp handle_error(409, message, %{bucket: bucket_name}) do - ErrorMessage.conflict("bucket already exists", %{bucket: bucket_name, message: message}) - end - - defp handle_error(404, message, %{bucket: bucket_name}) do - ErrorMessage.not_found("bucket not found", %{bucket: bucket_name, message: message}) - end - - defp handle_error(code, message, %{region: region, bucket: bucket_name}) do - %ErrorMessage{ - code: ErrorMessage.http_code_reason_atom(code), - message: message, - details: %{region: region, bucket: bucket_name} - } - end - - defp handle_error(code, message, %{region: region}) do - %ErrorMessage{ - code: ErrorMessage.http_code_reason_atom(code), - message: message, - details: %{region: region} - } + S3ObjectStore.delete_container(bucket_name, region: region) end end diff --git a/lib/deploy_ex/aws_database.ex b/lib/deploy_ex/aws_database.ex index 9fce81de..0ea769b9 100644 --- a/lib/deploy_ex/aws_database.ex +++ b/lib/deploy_ex/aws_database.ex @@ -5,26 +5,36 @@ defmodule DeployEx.AwsDatabase do import SweetXml, only: [sigil_x: 2] - def fetch_aws_databases do - case ExAws.request(ExAws.RDS.describe_db_instances(), region: DeployEx.Config.aws_region()) do + @doc """ + Every RDS instance in the region, following pagination to completion. + + DescribeDBInstances caps a response (default 100) and signals more via `Marker`. A single + request therefore truncates silently on a large account — it returns `{:ok, partial}`, not an + error — same class of bug already fixed in `AwsAutoscaling.fetch_all_asgs/5`. + """ + def fetch_aws_databases(opts \\ []) do + fetch_databases_page(opts, []) + end + + defp fetch_databases_page(opts, acc) do + {request_fn, describe_opts} = Keyword.pop(opts, :request_fn, &ExAws.request/2) + + response = + describe_opts + |> ExAws.RDS.describe_db_instances() + |> request_fn.(region: DeployEx.Config.aws_region()) + + case response do {:ok, %{body: body}} -> - instances = body - |> SweetXml.xpath(~x"//DBInstances/DBInstance"l, - identifier: ~x"./DBInstanceIdentifier/text()"s, - endpoint: [ - ~x"./Endpoint", - host: ~x"./Address/text()"s, - port: ~x"./Port/text()"i - ], - username: ~x"./MasterUsername/text()"s, - database: ~x"./DBName/text()"s, - tags: [ - ~x"./TagList/Tag"l, - key: ~x"./Key/text()"s, - value: ~x"./Value/text()"s - ] - ) - {:ok, instances} + accumulated = acc ++ extract_instances(body) + + case extract_marker(body) do + marker when is_binary(marker) and marker !== "" -> + fetch_databases_page(Keyword.put(opts, :marker, marker), accumulated) + + _no_more_pages -> + {:ok, accumulated} + end {:error, {"AccessDenied", message}} -> {:error, ErrorMessage.unauthorized("AWS RDS access denied", %{message: message})} @@ -40,6 +50,26 @@ defmodule DeployEx.AwsDatabase do end end + defp extract_instances(body) do + SweetXml.xpath(body, ~x"//DBInstances/DBInstance"l, + identifier: ~x"./DBInstanceIdentifier/text()"s, + endpoint: [ + ~x"./Endpoint", + host: ~x"./Address/text()"s, + port: ~x"./Port/text()"i + ], + username: ~x"./MasterUsername/text()"s, + database: ~x"./DBName/text()"s, + tags: [ + ~x"./TagList/Tag"l, + key: ~x"./Key/text()"s, + value: ~x"./Value/text()"s + ] + ) + end + + defp extract_marker(body), do: SweetXml.xpath(body, ~x"//Marker/text()"s) + def fetch_aws_databases_by_identifier(identifier) do with {:ok, instances} <- fetch_aws_databases() do case Enum.find(instances, fn instance -> instance.identifier == identifier end) do @@ -52,7 +82,7 @@ defmodule DeployEx.AwsDatabase do def fetch_aws_databases_by_tag(key, value) do with {:ok, instances} <- fetch_aws_databases() do filtered_dbs = instances - |> Enum.to_list() |> IO.inspect() + |> Enum.to_list() |> Stream.filter(fn instance -> Enum.any?(instance.tags, fn %{key: ^key, value: ^value} -> true diff --git a/lib/deploy_ex/aws_dynamodb.ex b/lib/deploy_ex/aws_dynamodb.ex index f6a42e1b..f4080c6b 100644 --- a/lib/deploy_ex/aws_dynamodb.ex +++ b/lib/deploy_ex/aws_dynamodb.ex @@ -1,6 +1,10 @@ defmodule DeployEx.AwsDynamodb do alias ExAws.Dynamo + # ex_aws_dynamo's Dynamo.list_tables/0 wraps ListTables with no way to pass + # ExclusiveStartTableName, so the raw JSON operation is built by hand to page through it. + @namespace "DynamoDB_20120810" + @type table_res :: %{table_name: String.t(), table_status: String.t()} @spec create_table(String.t(), String.t(), String.t(), String.t(), Keyword.t()) :: ErrorMessage.t_res(any) @@ -17,16 +21,51 @@ defmodule DeployEx.AwsDynamodb do end end + @doc """ + Every DynamoDB table name in the region, following pagination to completion. + + ListTables caps a response (default 100) and signals more via `LastEvaluatedTableName`. A + single request therefore truncates silently on an account with many tables — it returns + `{:ok, partial}`, not an error. `Dynamo.list_tables/0` has no way to pass + `ExclusiveStartTableName`, so pages are requested via a hand-built JSON operation instead. + """ @spec list_tables() :: ErrorMessage.t_res([String.t()]) @spec list_tables(String.t()) :: ErrorMessage.t_res([String.t()]) - def list_tables(region \\ DeployEx.Config.aws_region()) do - case ExAws.request(Dynamo.list_tables(), region: region) do - {:ok, %{"TableNames" => table_names}} -> {:ok, table_names} + @spec list_tables(String.t(), Keyword.t()) :: ErrorMessage.t_res([String.t()]) + def list_tables(region \\ DeployEx.Config.aws_region(), opts \\ []) do + list_tables_page(region, opts, %{}, []) + end + + defp list_tables_page(region, opts, data, acc) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + + case request_fn.(list_tables_operation(data), region: region) do + {:ok, %{"TableNames" => table_names} = body} -> + accumulated = acc ++ table_names + + case body["LastEvaluatedTableName"] do + name when is_binary(name) and name !== "" -> + list_tables_page(region, opts, %{"ExclusiveStartTableName" => name}, accumulated) + + _no_more_pages -> + {:ok, accumulated} + end + {:error, {:http_error, code, message}} -> {:error, handle_error(code, message, %{region: region})} end end + defp list_tables_operation(data) do + ExAws.Operation.JSON.new(:dynamodb, %{ + data: data, + headers: [ + {"x-amz-target", "#{@namespace}.ListTables"}, + {"content-type", "application/x-amz-json-1.0"} + ] + }) + end + @spec describe_table(String.t()) :: ErrorMessage.t_res(table_res) @spec describe_table(String.t(), String.t()) :: ErrorMessage.t_res(table_res) def describe_table(region \\ DeployEx.Config.aws_region(), table_name) do diff --git a/lib/deploy_ex/aws_infrastructure.ex b/lib/deploy_ex/aws_infrastructure.ex index 6c63a938..ca5f8d11 100644 --- a/lib/deploy_ex/aws_infrastructure.ex +++ b/lib/deploy_ex/aws_infrastructure.ex @@ -4,18 +4,106 @@ defmodule DeployEx.AwsInfrastructure do This module follows the pattern established by `DeployEx.AwsSecurityGroup` which uses AWS APIs to find resources by naming conventions, avoiding terraform state dependency. + + Implements `DeployEx.Cloud.Infrastructure`. Those callbacks are neutral names over the + AWS-specific functions below — here a network is a VPC, an identity is an IAM instance + profile, and an image is an AMI. """ + @behaviour DeployEx.Cloud.Infrastructure + + @impl DeployEx.Cloud.Infrastructure + def find_network(opts \\ []), do: find_vpc_id(opts) + + @impl DeployEx.Cloud.Infrastructure + def find_subnet(opts \\ []) do + with {:ok, subnet_ids} <- find_subnet_ids(opts) do + case subnet_ids do + [subnet_id | _rest] -> {:ok, subnet_id} + [] -> {:error, ErrorMessage.not_found("no subnet found", %{opts: opts})} + end + end + end + + @impl DeployEx.Cloud.Infrastructure + def find_key_pair(project_name, opts \\ []) do + find_key_pair_name(Keyword.put(opts, :project_name, project_name)) + end + + @impl DeployEx.Cloud.Infrastructure + def find_image(opts \\ []), do: find_latest_ami(opts) + + @impl DeployEx.Cloud.Infrastructure + def find_instance_identity(opts \\ []), do: find_iam_instance_profile(opts) + def find_subnet_ids(opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() vpc_id = opts[:vpc_id] + request_fn = opts[:request_fn] || (&ExAws.request/2) if is_nil(vpc_id) do {:error, ErrorMessage.bad_request("vpc_id is required to find subnets")} else - ExAws.EC2.describe_subnets(filters: ["vpc-id": [vpc_id]]) - |> ExAws.request(region: region) - |> handle_subnets_response(vpc_id) + with {:ok, items} <- fetch_subnets(vpc_id, region, request_fn) do + case items do + [] -> {:error, ErrorMessage.not_found("no subnets found in VPC '#{vpc_id}'")} + items -> {:ok, items |> Enum.sort_by(& &1["availabilityZone"]) |> Enum.map(& &1["subnetId"])} + end + end + end + end + + # DescribeSubnets caps a response and signals more via nextToken. A single request silently + # truncates on a VPC with many subnets — same failure mode AwsMachine.fetch_instances/2 guards + # against for DescribeInstances. + defp fetch_subnets(vpc_id, region, request_fn, next_token \\ nil, acc \\ []) do + request_opts = maybe_put_next_token([filters: ["vpc-id": [vpc_id]]], next_token) + + response = + request_opts + |> ExAws.EC2.describe_subnets() + |> request_fn.(region: region) + + case response do + {:ok, %{body: body}} -> + case XmlToMap.naive_map(body) do + %{"DescribeSubnetsResponse" => envelope} -> + accumulated = acc ++ extract_items(envelope["subnetSet"]) + + case extract_next_token(envelope) do + nil -> {:ok, accumulated} + token -> fetch_subnets(vpc_id, region, request_fn, token, accumulated) + end + + structure -> + {:error, ErrorMessage.bad_request( + "couldn't parse subnets response from aws", + %{structure: structure} + )} + end + + {:error, {:http_error, status_code, %{body: body}}} -> + {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ + "error fetching subnets from aws", + %{error_body: body} + ])} + end + end + + defp maybe_put_next_token(request_opts, nil), do: request_opts + defp maybe_put_next_token(request_opts, next_token), do: Keyword.put(request_opts, :next_token, next_token) + + # AWS applies MaxResults to the underlying scan before any filters, so a small forced page + # (or a naturally small final page) can filter down to zero items while still carrying a + # nextToken — binding the whole envelope (not matching on the item set directly) means that + # token is never missed just because a page happened to filter to empty. + defp extract_items(nil), do: [] + defp extract_items(%{"item" => items}), do: List.wrap(items) + + defp extract_next_token(envelope) do + case Map.get(envelope, "nextToken") do + token when is_binary(token) and token !== "" -> token + _absent -> nil end end @@ -26,7 +114,9 @@ defmodule DeployEx.AwsInfrastructure do base_name = project_name |> String.replace("-#{environment}", "") |> String.replace("_#{environment}", "") key_pattern = ~r/^#{Regex.escape(base_name)}-.*key-pair/ - with {:ok, key_pairs} <- describe_key_pairs(region) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + + with {:ok, key_pairs} <- describe_key_pairs(region, request_fn) do matching = key_pairs |> Enum.filter(fn kp -> name = kp["keyName"] || "" @@ -45,9 +135,13 @@ defmodule DeployEx.AwsInfrastructure do end end - defp describe_key_pairs(region) do + # `:request_fn` is the same injection seam find_subnet_ids/1 and find_iam_instance_profile/1 + # already use. Without it this path can only be exercised against a live AWS account, which is + # how its tests ended up calling the real API and failing based on what happened to exist + # there. + defp describe_key_pairs(region, request_fn) do ExAws.EC2.describe_key_pairs() - |> ExAws.request(region: region) + |> request_fn.(region: region) |> handle_key_pairs_list_response() end @@ -56,8 +150,9 @@ defmodule DeployEx.AwsInfrastructure do nil -> environment = DeployEx.Config.env() default_name = "deploy-ex-ec2-instance-profile-#{environment}" + request_fn = opts[:request_fn] || (&ExAws.request/2) - with {:ok, profiles} <- list_instance_profiles() do + with {:ok, profiles} <- list_instance_profiles(request_fn) do if default_name in profiles do {:ok, default_name} else @@ -73,41 +168,57 @@ defmodule DeployEx.AwsInfrastructure do end end - defp list_instance_profiles do - %ExAws.Operation.Query{ - path: "/", - params: %{"Action" => "ListInstanceProfiles", "Version" => "2010-05-08"}, - service: :iam, - action: :list_instance_profiles - } - |> ExAws.request() - |> handle_instance_profiles_response() - end - - defp handle_instance_profiles_response({:ok, %{body: body}}) do - case XmlToMap.naive_map(body) do - %{"ListInstanceProfilesResponse" => %{"ListInstanceProfilesResult" => %{"InstanceProfiles" => %{"member" => profiles}}}} when is_list(profiles) -> - names = Enum.map(profiles, & &1["InstanceProfileName"]) - {:ok, names} - - %{"ListInstanceProfilesResponse" => %{"ListInstanceProfilesResult" => %{"InstanceProfiles" => %{"member" => profile}}}} -> - {:ok, [profile["InstanceProfileName"]]} - - %{"ListInstanceProfilesResponse" => %{"ListInstanceProfilesResult" => %{"InstanceProfiles" => nil}}} -> - {:ok, []} + # ListInstanceProfiles caps a response and signals more via IsTruncated + Marker. ExAws + # returns IsTruncated as the STRING "false", which is truthy in Elixir — branching on it + # directly would loop forever, so this compares against known values instead. + defp list_instance_profiles(request_fn, marker \\ nil, acc \\ []) do + base_params = %{"Action" => "ListInstanceProfiles", "Version" => "2010-05-08"} + params = if marker, do: Map.put(base_params, "Marker", marker), else: base_params + + response = + %ExAws.Operation.Query{ + path: "/", + params: params, + service: :iam, + action: :list_instance_profiles + } + |> request_fn.([]) + + case response do + {:ok, %{body: body}} -> + case XmlToMap.naive_map(body) do + %{"ListInstanceProfilesResponse" => %{"ListInstanceProfilesResult" => result}} -> + accumulated = acc ++ extract_instance_profile_names(result["InstanceProfiles"]) + + if truncated?(result["IsTruncated"]) and present_marker?(result["Marker"]) do + list_instance_profiles(request_fn, result["Marker"], accumulated) + else + {:ok, accumulated} + end + + structure -> + {:error, ErrorMessage.bad_request("couldn't parse instance profiles response", %{structure: structure})} + end - structure -> - {:error, ErrorMessage.bad_request("couldn't parse instance profiles response", %{structure: structure})} + {:error, {:http_error, status_code, %{body: body}}} -> + {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ + "error fetching IAM instance profiles", + %{error_body: body} + ])} end end - defp handle_instance_profiles_response({:error, {:http_error, status_code, %{body: body}}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ - "error fetching IAM instance profiles", - %{error_body: body} - ])} + defp extract_instance_profile_names(%{"member" => profiles}) when is_list(profiles) do + Enum.map(profiles, & &1["InstanceProfileName"]) end + defp extract_instance_profile_names(%{"member" => profile}), do: [profile["InstanceProfileName"]] + defp extract_instance_profile_names(nil), do: [] + + defp truncated?(value), do: value in [true, "true"] + + defp present_marker?(marker), do: is_binary(marker) and marker !== "" + def find_vpc_id(opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() resource_group = opts[:resource_group] || DeployEx.Config.aws_resource_group() @@ -121,42 +232,92 @@ defmodule DeployEx.AwsInfrastructure do app_name = opts[:app_name] region = opts[:region] || DeployEx.Config.aws_region() environment = opts[:environment] || DeployEx.Config.env() + request_fn = opts[:request_fn] || (&ExAws.request/2) - case app_name && find_app_ami(app_name, environment, region) do + case app_name && find_app_ami(app_name, environment, region, request_fn) do {:ok, ami_id} -> {:ok, ami_id} - _ -> find_base_ami(region) + _ -> find_base_ami(region, request_fn) end end - defp find_app_ami(app_name, environment, region) do - ExAws.EC2.describe_images( - owners: ["self"], - filters: [ - "tag:App": [app_name], - "tag:Environment": [to_string(environment)], - "tag:ManagedBy": ["DeployEx"], - state: ["available"] - ] + defp find_app_ami(app_name, environment, region, request_fn) do + fetch_latest_ami( + [ + owners: ["self"], + filters: [ + "tag:App": [app_name], + "tag:Environment": [to_string(environment)], + "tag:ManagedBy": ["DeployEx"], + state: ["available"] + ] + ], + region, + request_fn ) - |> ExAws.request(region: region) - |> handle_images_response() end - defp find_base_ami(region) do + defp find_base_ami(region, request_fn) do base_ami_name = DeployEx.Config.aws_base_ami_name() architecture = DeployEx.Config.aws_base_ami_architecture() owner = DeployEx.Config.aws_base_ami_owner() - ExAws.EC2.describe_images( - owners: [owner], - filters: [ - name: ["#{base_ami_name}-*"], - architecture: [architecture], - "virtualization-type": ["hvm"] - ] + fetch_latest_ami( + [ + owners: [owner], + filters: [ + name: ["#{base_ami_name}-*"], + architecture: [architecture], + "virtualization-type": ["hvm"] + ] + ], + region, + request_fn ) - |> ExAws.request(region: region) - |> handle_images_response() + end + + # DescribeImages caps its scan before tag/name filters are applied, so a small page can filter + # to zero matches while still returning a token, and sorting "latest" over just that page can + # pick a stale AMI believing it's current. Every page has to be in hand before creationDate + # sorting means anything. + defp fetch_latest_ami(request_opts, region, request_fn, next_token \\ nil, acc \\ []) do + opts = maybe_put_next_token(request_opts, next_token) + + response = + opts + |> ExAws.EC2.describe_images() + |> request_fn.(region: region) + + case response do + {:ok, %{body: body}} -> + case XmlToMap.naive_map(body) do + %{"DescribeImagesResponse" => envelope} -> + accumulated = acc ++ extract_items(envelope["imagesSet"]) + + case extract_next_token(envelope) do + nil -> latest_ami_result(accumulated) + token -> fetch_latest_ami(request_opts, region, request_fn, token, accumulated) + end + + structure -> + {:error, ErrorMessage.bad_request( + "couldn't parse images response from aws", + %{structure: structure} + )} + end + + {:error, {:http_error, status_code, %{body: body}}} -> + {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ + "error fetching AMIs from aws", + %{error_body: body} + ])} + end + end + + defp latest_ami_result([]), do: {:error, ErrorMessage.not_found("no debian-13 AMI found")} + + defp latest_ami_result(items) do + latest = items |> Enum.sort_by(& &1["creationDate"], :desc) |> List.first() + {:ok, latest["imageId"]} end def gather_infrastructure(opts \\ []) do @@ -190,13 +351,57 @@ defmodule DeployEx.AwsInfrastructure do def find_primary_subnet_id(security_group_id, opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() + request_fn = opts[:request_fn] || (&ExAws.request/2) - ExAws.EC2.describe_instances(filters: [ - "instance.group-id": [security_group_id], - "instance-state-name": ["running"] - ]) - |> ExAws.request(region: region) - |> handle_primary_subnet_response() + with {:ok, subnet_ids} <- fetch_primary_subnet_candidates(security_group_id, region, request_fn) do + most_common_subnet_id(subnet_ids) + end + end + + # DescribeInstances caps a response and signals more via nextToken. A truncated ballot can + # flip the winner in most_common_subnet_id/1 on a multi-AZ fleet — every page has to be + # counted before "most common" means anything. + defp fetch_primary_subnet_candidates(security_group_id, region, request_fn, next_token \\ nil, acc \\ []) do + request_opts = + maybe_put_next_token( + [filters: ["instance.group-id": [security_group_id], "instance-state-name": ["running"]]], + next_token + ) + + response = + request_opts + |> ExAws.EC2.describe_instances() + |> request_fn.(region: region) + + case response do + {:ok, %{body: body}} -> + case XmlToMap.naive_map(body) do + %{"DescribeInstancesResponse" => envelope} -> + subnet_ids = + envelope["reservationSet"] + |> extract_items() + |> Enum.flat_map(&extract_instance_subnet_ids/1) + + accumulated = acc ++ subnet_ids + + case extract_next_token(envelope) do + nil -> {:ok, accumulated} + token -> fetch_primary_subnet_candidates(security_group_id, region, request_fn, token, accumulated) + end + + structure -> + {:error, ErrorMessage.bad_request( + "couldn't parse instances response from aws", + %{structure: structure} + )} + end + + {:error, {:http_error, status_code, %{body: body}}} -> + {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ + "error fetching instances from aws", + %{error_body: body} + ])} + end end @doc false @@ -263,25 +468,6 @@ defmodule DeployEx.AwsInfrastructure do |> then(fn {subnet_id, _count} -> {:ok, subnet_id} end) end - defp handle_primary_subnet_response({:ok, %{body: body}}), do: parse_primary_subnet_response(body) - - defp handle_primary_subnet_response({:error, {:http_error, status_code, %{body: body}}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ - "error fetching instances from aws", - %{error_body: body} - ])} - end - - defp handle_subnets_response({:ok, %{body: body}}, resource_group), do: parse_subnets_response(body, resource_group) - - defp handle_subnets_response({:error, {:http_error, status_code, %{body: body}}}, _resource_group) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ - "error fetching subnets from aws", - %{error_body: body} - ])} - end - - defp handle_key_pairs_list_response({:ok, %{body: body}}) do case XmlToMap.naive_map(body) do %{"DescribeKeyPairsResponse" => %{"keySet" => %{"item" => items}}} when is_list(items) -> @@ -362,13 +548,4 @@ defmodule DeployEx.AwsInfrastructure do end end - defp handle_images_response({:ok, %{body: body}}), do: parse_images_response(body) - - defp handle_images_response({:error, {:http_error, status_code, %{body: body}}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ - "error fetching AMIs from aws", - %{error_body: body} - ])} - end - end diff --git a/lib/deploy_ex/aws_ip_whitelister.ex b/lib/deploy_ex/aws_ip_whitelister.ex index 70a066ae..12241681 100644 --- a/lib/deploy_ex/aws_ip_whitelister.ex +++ b/lib/deploy_ex/aws_ip_whitelister.ex @@ -1,59 +1,21 @@ defmodule DeployEx.AwsIpWhitelister do - alias ExAws.EC2 + @moduledoc """ + Whitelists a single IP for SSH on a security group. + + The EC2 calls now live in `DeployEx.AwsSecurityGroup`, which implements + `DeployEx.Cloud.Security`. This module stays as the IP-shaped front door its Mix task call + sites already use — it takes a bare address and widens it to a /32 CIDR. + """ + + alias DeployEx.AwsSecurityGroup def authorize(security_group_id, ip_address, opts \\ []) do - opts - |> Keyword.merge( - group_id: security_group_id, - cidr_ip: "#{ip_address}/32", - ip_protocol: "tcp", - from_port: 22, - to_port: 22 - ) - |> EC2.authorize_security_group_ingress - |> make_request(security_group_id, ip_address) + AwsSecurityGroup.authorize_ingress(security_group_id, to_cidr(ip_address), opts) end def deauthorize(security_group_id, ip_address, opts \\ []) do - opts - |> Keyword.merge( - group_id: security_group_id, - cidr_ip: "#{ip_address}/32", - ip_protocol: "tcp", - from_port: 22, - to_port: 22 - ) - |> EC2.revoke_security_group_ingress - |> make_request(security_group_id, ip_address) + AwsSecurityGroup.revoke_ingress(security_group_id, to_cidr(ip_address), opts) end - defp make_request(request, security_group_id, ip_address) do - case ExAws.request(request, region: DeployEx.Config.aws_region()) do - {:ok, %{body: _, status_code: 200}} -> :ok - - {:error, {:http_error, code, %{body: body}}} -> - message = body |> SweetXml.xpath(SweetXml.sigil_x"//Message/text()") |> to_string - - cond do - message =~ "already exists" -> - {:error, ErrorMessage.conflict( - message, - %{ip_address: ip_address, security_group_id: security_group_id - })} - - message =~ "does not exist" -> - {:error, ErrorMessage.not_found( - message, - %{ip_address: ip_address, security_group_id: security_group_id - })} - - true -> - {:error, %ErrorMessage{ - code: ErrorMessage.http_code_reason_atom(code), - message: message, - details: %{ip_address: ip_address, security_group_id: security_group_id} - }} - end - end - end + defp to_cidr(ip_address), do: "#{ip_address}/32" end diff --git a/lib/deploy_ex/aws_load_balancer.ex b/lib/deploy_ex/aws_load_balancer.ex index 3a5e600e..61d6ff9b 100644 --- a/lib/deploy_ex/aws_load_balancer.ex +++ b/lib/deploy_ex/aws_load_balancer.ex @@ -44,12 +44,50 @@ defmodule DeployEx.AwsLoadBalancer do |> handle_health_response() end + @doc """ + Every target group in the region, following pagination to completion. + + DescribeTargetGroups caps a response and signals more via NextMarker. A single request + silently truncates on an account with many target groups — the same failure mode + `S3ObjectStore.list_objects/2` guards against for S3's Marker. + """ def describe_target_groups(opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() + request_fn = opts[:request_fn] || (&ExAws.request/2) - ExAws.ElasticLoadBalancingV2.describe_target_groups() - |> ExAws.request(region: region) - |> handle_target_groups_response() + fetch_target_groups_page(region, request_fn, nil, []) + end + + defp fetch_target_groups_page(region, request_fn, marker, acc) do + request_opts = if marker, do: [marker: marker], else: [] + + response = + request_opts + |> ExAws.ElasticLoadBalancingV2.describe_target_groups() + |> request_fn.(region: region) + + case response do + {:ok, %{body: %{target_groups: target_groups} = body}} -> + accumulated = acc ++ parse_target_groups(target_groups) + + case next_marker(body) do + nil -> {:ok, accumulated} + next -> fetch_target_groups_page(region, request_fn, next, accumulated) + end + + {:error, {:http_error, status_code, %{body: body}}} -> + {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ + "error describing target groups", + %{error_body: body} + ])} + end + end + + defp next_marker(body) do + case Map.get(body, :next_marker) do + marker when is_binary(marker) and marker !== "" -> marker + _absent -> nil + end end def find_target_groups_by_app(app_name, opts \\ []) do @@ -141,8 +179,8 @@ defmodule DeployEx.AwsLoadBalancer do ])} end - defp handle_target_groups_response({:ok, %{body: %{target_groups: target_groups}}}) do - parsed = Enum.map(target_groups, fn tg -> + defp parse_target_groups(target_groups) do + Enum.map(target_groups, fn tg -> %{ arn: tg[:target_group_arn], name: tg[:target_group_name], @@ -154,15 +192,6 @@ defmodule DeployEx.AwsLoadBalancer do health_check_protocol: tg[:health_check_protocol] } end) - - {:ok, parsed} - end - - defp handle_target_groups_response({:error, {:http_error, status_code, %{body: body}}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ - "error describing target groups", - %{error_body: body} - ])} end defp parse_integer(nil), do: nil diff --git a/lib/deploy_ex/aws_machine.ex b/lib/deploy_ex/aws_machine.ex index 6370be13..016960a9 100644 --- a/lib/deploy_ex/aws_machine.ex +++ b/lib/deploy_ex/aws_machine.ex @@ -1,4 +1,112 @@ defmodule DeployEx.AwsMachine do + @moduledoc """ + EC2 instance discovery and lifecycle, and the AWS implementation of `DeployEx.Cloud.Machine`. + + The behaviour callbacks return `%DeployEx.Cloud.Instance{}`. The older functions below return + provider-shaped maps and keep doing so — seven Mix-task call sites depend on their shape, and + `mix deploy_ex.find_nodes --format json` publishes it as a user-visible contract. The caller + sweep moves those onto the callbacks; until then both live here side by side. + """ + + @behaviour DeployEx.Cloud.Machine + + @impl DeployEx.Cloud.Machine + def list_instances(tag_filters, opts \\ []) when is_list(tag_filters) do + with {:ok, instances} <- find_instances_by_tags(tag_filters, opts) do + {:ok, Enum.map(instances, &to_instance/1)} + end + end + + @impl DeployEx.Cloud.Machine + def find_app_instances(project_name, app_name, opts \\ []) do + with {:ok, instances} <- scoped_running_instances(opts) do + case Enum.filter(instances, &instance_in_app?(&1, app_name)) do + [] -> + {:error, + ErrorMessage.not_found("no instances found for #{app_name}", %{ + app_name: app_name, + project_name: project_name + })} + + matching -> + {:ok, Enum.map(matching, &to_instance/1)} + end + end + end + + @impl DeployEx.Cloud.Machine + def describe_instance(instance_id, opts \\ []) do + region = opts[:region] || DeployEx.Config.aws_region() + + with {:ok, [instance | _rest]} <- find_instances_by_id(region, [instance_id]) do + {:ok, to_instance(instance)} + end + end + + @impl DeployEx.Cloud.Machine + def start_instance(instance_id, opts \\ []) do + region = opts[:region] || DeployEx.Config.aws_region() + + with {:ok, _response} <- start(region, [instance_id]), do: :ok + end + + @impl DeployEx.Cloud.Machine + def stop_instance(instance_id, opts \\ []) do + region = opts[:region] || DeployEx.Config.aws_region() + + with {:ok, _response} <- stop(region, [instance_id]), do: :ok + end + + @impl DeployEx.Cloud.Machine + def fetch_tags(instance_id, opts \\ []) do + with {:ok, instance} <- describe_instance(instance_id, opts) do + {:ok, instance.tags} + end + end + + @doc """ + Preferred reachable address, IPv6 first. + + IPv6 wins because `find_instance_ips/3` has made that choice since before this behaviour + existed, and the deploy path depends on it. + """ + @impl DeployEx.Cloud.Machine + def instance_address(%DeployEx.Cloud.Instance{} = instance) do + case instance.ipv6 || instance.public_ip do + nil -> {:error, ErrorMessage.not_found("instance has no reachable address", %{id: instance.id})} + address -> {:ok, address} + end + end + + defp scoped_running_instances(opts) do + region = opts[:region] || DeployEx.Config.aws_region() + resource_group = opts[:resource_group] || DeployEx.Config.aws_resource_group() + + with {:ok, instances} <- fetch_instances_by_tag(region, "Group", resource_group) do + {:ok, + instances + |> Enum.filter(&running_with_instance_group?/1) + |> maybe_reject_qa_nodes(opts)} + end + end + + defp running_with_instance_group?(instance) do + instance["instanceState"]["name"] === "running" and + not is_nil(get_instance_tags(instance)["InstanceGroup"]) + end + + defp maybe_reject_qa_nodes(instances, opts) do + if opts[:exclude_qa_nodes] === true do + Enum.reject(instances, &(get_instance_tags(&1)["QaNode"] === "true")) + else + instances + end + end + + defp instance_in_app?(instance, app_name) do + instance |> get_instance_tags() |> Map.get("InstanceGroup", "") =~ app_name + end + def start(region \\ DeployEx.Config.aws_region(), instance_ids) do instance_ids |> ExAws.EC2.start_instances() @@ -144,12 +252,54 @@ defmodule DeployEx.AwsMachine do end end - def fetch_instances(region) do - ExAws.EC2.describe_instances() - |> ex_aws_request(region) - |> handle_describe_response + @doc """ + Every instance in the region, following pagination to completion. + + DescribeInstances caps a response and signals more via `nextToken`. A single request therefore + truncates silently on a large account — it returns `{:ok, partial}`, not an error — which would + make every caller here (tag filters, setup-state queries, `mix deploy_ex.find_nodes`) quietly + miss instances. `AwsAutoscaling.fetch_all_asgs/5` already paginates for the same reason. + """ + def fetch_instances(region, opts \\ []) do + fetch_instances_page(region, opts, nil, []) end + defp fetch_instances_page(region, opts, next_token, acc) do + # :request_fn is the injection seam for page-boundary tests. It is split out of opts before + # they become EC2 request params, since anything left in opts is sent to the API. + {request_fn, describe_opts} = Keyword.pop(opts, :request_fn, &ExAws.request/2) + + request_opts = + if next_token, do: Keyword.put(describe_opts, :next_token, next_token), else: describe_opts + + response = + request_opts + |> ExAws.EC2.describe_instances() + |> request_fn.(region: region || DeployEx.Config.aws_region()) + + with {:ok, instances} <- handle_describe_response(response) do + accumulated = acc ++ instances + + case describe_next_token(response) do + nil -> {:ok, accumulated} + token -> fetch_instances_page(region, opts, token, accumulated) + end + end + end + + defp describe_next_token({:ok, %{body: body}}) do + case XmlToMap.naive_map(body) do + %{"DescribeInstancesResponse" => %{"nextToken" => token}} + when is_binary(token) and token !== "" -> + token + + _no_more_pages -> + nil + end + end + + defp describe_next_token(_response), do: nil + defp ex_aws_request(request_struct, nil) do ExAws.request(request_struct) end @@ -351,6 +501,30 @@ defmodule DeployEx.AwsMachine do end end + @doc """ + Normalizes a raw AWS instance map into the provider-neutral struct. + + Distinct from `parse_instance_info/1` below, which produces the AWS-shaped map behind the + frozen `mix deploy_ex.find_nodes --format json` key set. That one is a display projection + whose keys are a user-visible contract; this one is the neutral behaviour type. + """ + def to_instance(instance) do + tags = get_instance_tags(instance) + + %DeployEx.Cloud.Instance{ + id: instance["instanceId"], + type: instance["instanceType"], + state: instance["instanceState"]["name"], + private_ip: instance["privateIpAddress"], + public_ip: instance["ipAddress"], + ipv6: instance["ipv6Address"], + launched_at: instance["launchTime"], + name: tags["Name"], + qa_node?: tags["QaNode"] === "true", + tags: tags + } + end + def parse_instance_info(instance) do tags = get_instance_tags(instance) diff --git a/lib/deploy_ex/aws_security_group.ex b/lib/deploy_ex/aws_security_group.ex index eaea4c4f..2f5499a0 100644 --- a/lib/deploy_ex/aws_security_group.ex +++ b/lib/deploy_ex/aws_security_group.ex @@ -1,4 +1,84 @@ defmodule DeployEx.AwsSecurityGroup do + @moduledoc """ + AWS implementation of `DeployEx.Cloud.Security`. + + Owns the SSH ingress rules `mix deploy_ex.ssh.authorize` manages. The ingress calls used to + live in `DeployEx.AwsIpWhitelister`; that module now delegates here so every EC2 call for + this capability sits behind one behaviour. + """ + + @behaviour DeployEx.Cloud.Security + + @ssh_port 22 + + @impl DeployEx.Cloud.Security + def find_group(opts \\ []), do: find_security_group_id(opts) + + @impl DeployEx.Cloud.Security + def authorize_ingress(security_group_id, cidr, opts \\ []) do + security_group_id + |> build_ingress_request(cidr, opts) + |> ExAws.EC2.authorize_security_group_ingress() + |> request_ingress_change(security_group_id, cidr, opts) + end + + @impl DeployEx.Cloud.Security + def revoke_ingress(security_group_id, cidr, opts \\ []) do + security_group_id + |> build_ingress_request(cidr, opts) + |> ExAws.EC2.revoke_security_group_ingress() + |> request_ingress_change(security_group_id, cidr, opts) + end + + @doc """ + Turns an AWS ingress error body into an `ErrorMessage`. + + Public because it is the only pure part of the ingress path — the request itself needs a live + account, this does not. + """ + def classify_ingress_error(status_code, body, details) do + message = body |> SweetXml.xpath(SweetXml.sigil_x("//Message/text()", [])) |> to_string() + + cond do + message =~ "already exists" -> {:error, ErrorMessage.conflict(message, details)} + message =~ "does not exist" -> {:error, ErrorMessage.not_found(message, details)} + true -> {:error, build_http_error(status_code, message, details)} + end + end + + defp build_ingress_request(security_group_id, cidr, opts) do + Keyword.merge(opts, + group_id: security_group_id, + cidr_ip: cidr, + ip_protocol: "tcp", + from_port: @ssh_port, + to_port: @ssh_port + ) + end + + defp request_ingress_change(request, security_group_id, cidr, opts) do + region = opts[:region] || DeployEx.Config.aws_region() + + case ExAws.request(request, region: region) do + {:ok, %{status_code: 200}} -> + :ok + + {:error, {:http_error, status_code, %{body: body}}} -> + classify_ingress_error(status_code, body, %{ + cidr: cidr, + security_group_id: security_group_id + }) + end + end + + defp build_http_error(status_code, message, details) do + %ErrorMessage{ + code: ErrorMessage.http_code_reason_atom(status_code), + message: message, + details: details + } + end + def find_security_group(opts \\ []) do security_group_id = opts[:security_group_id] || DeployEx.Config.aws_security_group_id() @@ -11,8 +91,9 @@ defmodule DeployEx.AwsSecurityGroup do defp find_security_group_by_id(security_group_id, opts) do region = opts[:region] || DeployEx.Config.aws_region() + request_fn = opts[:request_fn] || (&ExAws.request/2) - with {:ok, security_groups} <- describe_security_groups(region) do + with {:ok, security_groups} <- describe_security_groups(region, request_fn) do matching = Enum.find(security_groups, fn sg -> sg["groupId"] === security_group_id or sg["groupName"] === security_group_id or @@ -36,6 +117,7 @@ defmodule DeployEx.AwsSecurityGroup do region = opts[:region] || DeployEx.Config.aws_region() project_name = opts[:project_name] || DeployEx.Config.aws_project_name() environment = opts[:environment] || DeployEx.Config.env() + request_fn = opts[:request_fn] || (&ExAws.request/2) sg_prefix = if DeployEx.Config.aws_names_include_env?() do base_name = project_name @@ -47,7 +129,7 @@ defmodule DeployEx.AwsSecurityGroup do "#{project_name}-sg" end - with {:ok, security_groups} <- describe_security_groups(region) do + with {:ok, security_groups} <- describe_security_groups(region, request_fn) do matching = security_groups |> Enum.filter(fn sg -> name = sg["groupName"] || "" @@ -75,43 +157,54 @@ defmodule DeployEx.AwsSecurityGroup do end end - defp describe_security_groups(region) do - ExAws.EC2.describe_security_groups() - |> ex_aws_request(region) - |> handle_response() - end - - defp ex_aws_request(request_struct, nil) do - ExAws.request(request_struct) - end - - defp ex_aws_request(request_struct, region) do - ExAws.request(request_struct, region: region) - end - - defp handle_response({:error, {:http_error, status_code, %{body: body}}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ - "error fetching security groups from aws", - %{error_body: body} - ])} + # DescribeSecurityGroups caps a response and signals more via nextToken. A single request + # silently truncates on an account with many security groups — same failure mode + # AwsInfrastructure.fetch_subnets/5 guards against for DescribeSubnets. + defp describe_security_groups(region, request_fn, next_token \\ nil, acc \\ []) do + request_opts = maybe_put_next_token([], next_token) + + response = + request_opts + |> ExAws.EC2.describe_security_groups() + |> request_fn.(region: region) + + case response do + {:ok, %{body: body}} -> + case XmlToMap.naive_map(body) do + %{"DescribeSecurityGroupsResponse" => envelope} -> + accumulated = acc ++ extract_items(envelope["securityGroupInfo"]) + + case extract_next_token(envelope) do + nil -> {:ok, accumulated} + token -> describe_security_groups(region, request_fn, token, accumulated) + end + + structure -> + {:error, ErrorMessage.bad_request( + "couldn't parse security groups response from aws", + %{structure: structure} + )} + end + + {:error, {:http_error, status_code, %{body: body}}} -> + {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ + "error fetching security groups from aws", + %{error_body: body} + ])} + end end - defp handle_response({:ok, %{body: body}}) do - case XmlToMap.naive_map(body) do - %{"DescribeSecurityGroupsResponse" => %{"securityGroupInfo" => %{"item" => items}}} when is_list(items) -> - {:ok, items} - - %{"DescribeSecurityGroupsResponse" => %{"securityGroupInfo" => %{"item" => item}}} -> - {:ok, [item]} + defp maybe_put_next_token(request_opts, nil), do: request_opts + defp maybe_put_next_token(request_opts, next_token), do: Keyword.put(request_opts, :next_token, next_token) - %{"DescribeSecurityGroupsResponse" => %{"securityGroupInfo" => nil}} -> - {:ok, []} + defp extract_items(nil), do: [] + defp extract_items(%{"item" => items}) when is_list(items), do: items + defp extract_items(%{"item" => item}), do: [item] - structure -> - {:error, ErrorMessage.bad_request( - "couldn't parse security groups response from aws", - %{structure: structure} - )} + defp extract_next_token(envelope) do + case Map.get(envelope, "nextToken") do + token when is_binary(token) and token !== "" -> token + _absent -> nil end end diff --git a/lib/deploy_ex/cloud.ex b/lib/deploy_ex/cloud.ex new file mode 100644 index 00000000..c8aa5f37 --- /dev/null +++ b/lib/deploy_ex/cloud.ex @@ -0,0 +1,179 @@ +defmodule DeployEx.Cloud do + @moduledoc """ + Entry point for provider-aware behaviour lookup. + + Dispatch is DERIVED from provider descriptors rather than written here. This module holds + no capability or behaviour module names at all — adding a provider costs one registry + entry plus one descriptor module, and nothing in this file changes. + + A capability the active provider does not implement returns + `{:error, %ErrorMessage{code: :not_implemented}}` so tasks can surface an honest message + and exit rather than crashing on a nil module. + """ + + alias DeployEx.Config + + @providers %{ + aws: DeployEx.Cloud.Providers.Aws, + oci: DeployEx.Cloud.Providers.Oci + } + + @doc """ + Resolves the module implementing `capability` for the active provider. + + `:provider` in `opts` overrides the configured provider and accepts either a registered + atom key or a descriptor module. The module form is the injection seam used by tests, + which cannot call `Application.put_env/3`. + """ + @spec capability(atom(), keyword()) :: {:ok, module()} | {:error, ErrorMessage.t()} + def capability(capability, opts \\ []) do + with {:ok, descriptor} <- opts |> active_provider() |> fetch_descriptor() do + fetch_capability(descriptor, capability) + end + end + + @doc """ + Inventory strategy/template/filename for a provider's descriptor. + + Returns `{:error, %ErrorMessage{code: :not_implemented}}` both when the provider itself is + unregistered and when it has not filled its `inventory/0` slot yet — an unfilled slot means + a provider still short of that phase, not a bug in this lookup. The sole consumer today is + `Mix.Tasks.Ansible.{Build,Setup,Deploy,Ping}`, which all resolve the live inventory filename + through here so a provider switch can never leave one of them checking the other's file. + """ + @spec inventory(atom() | module()) :: + {:ok, DeployEx.Cloud.Provider.inventory()} | {:error, ErrorMessage.t()} + def inventory(provider) do + with {:ok, descriptor} <- fetch_descriptor(provider) do + case descriptor.inventory() do + nil -> + {:error, + ErrorMessage.not_implemented("#{inspect(descriptor)} has not implemented inventory/0", %{ + provider: descriptor + })} + + inventory -> + {:ok, inventory} + end + end + end + + @doc """ + Provider these opts resolve to: an explicit `:provider` override, else the configured one. + + Public so the resolution is testable on its own. Every dispatch path routes through it, so + a hardcoded provider anywhere else is a defect this function's tests will not hide. + """ + @spec active_provider(keyword()) :: atom() | module() + def active_provider(opts), do: opts[:provider] || Config.cloud_provider() + + @doc """ + Validates a provider's configuration namespace against its descriptor schema. + + The environment is an explicit argument so the check is a pure function. The convenience + arities below read the real application environment. + """ + @spec validate_config(atom() | module(), keyword()) :: :ok | {:error, ErrorMessage.t()} + def validate_config(provider, env) when is_list(env) do + if Keyword.keyword?(env) do + with {:ok, descriptor} <- fetch_descriptor(provider) do + validate_against_schema(descriptor, env, provider) + end + else + invalid_config_error(provider, env) + end + end + + def validate_config(provider, env), do: invalid_config_error(provider, env) + + @spec validate_config(atom() | keyword()) :: :ok | {:error, ErrorMessage.t()} + def validate_config(provider_or_opts \\ []) + + def validate_config(opts) when is_list(opts) do + provider = active_provider(opts) + + validate_config(provider, config_env(provider)) + end + + def validate_config(provider), do: validate_config(provider, config_env(provider)) + + @doc "Registered provider keys." + @spec providers() :: [atom()] + def providers, do: Map.keys(@providers) + + defp invalid_config_error(provider, env) do + {:error, + ErrorMessage.bad_request("#{inspect(provider)} config must be a keyword list", %{ + provider: provider, + config: env + })} + end + + defp fetch_descriptor(provider) when is_atom(provider) and not is_nil(provider) do + case Map.fetch(@providers, provider) do + {:ok, descriptor} -> {:ok, descriptor} + :error -> fetch_descriptor_module(provider) + end + end + + defp fetch_descriptor(provider) do + {:error, + ErrorMessage.not_implemented( + "cloud provider must be an atom, got #{inspect(provider)}", + %{provider: provider, known_providers: Map.keys(@providers)} + )} + end + + defp fetch_descriptor_module(module) do + if descriptor_module?(module) do + {:ok, module} + else + {:error, + ErrorMessage.not_implemented("cloud provider #{inspect(module)} is not implemented", %{ + provider: module, + known_providers: Map.keys(@providers) + })} + end + end + + defp descriptor_module?(module) do + Code.ensure_loaded?(module) and function_exported?(module, :capabilities, 0) and + function_exported?(module, :config_schema, 0) + end + + defp fetch_capability(descriptor, capability) do + case Map.fetch(descriptor.capabilities(), capability) do + {:ok, module} -> + {:ok, module} + + :error -> + {:error, + ErrorMessage.not_implemented( + "#{capability} is not implemented for #{inspect(descriptor)}", + %{capability: capability, provider: descriptor} + )} + end + end + + defp validate_against_schema(descriptor, env, provider) do + case NimbleOptions.validate(env, descriptor.config_schema()) do + {:ok, _validated} -> + :ok + + {:error, %NimbleOptions.ValidationError{} = error} -> + {:error, + ErrorMessage.bad_request("invalid #{inspect(provider)} config: #{Exception.message(error)}", %{ + provider: provider, + key: error.key + })} + end + end + + defp config_env(provider) do + if provider === :aws do + Application.get_all_env(:deploy_ex) + else + Application.get_env(:deploy_ex, provider) || [] + end + end +end diff --git a/lib/deploy_ex/cloud/infrastructure.ex b/lib/deploy_ex/cloud/infrastructure.ex new file mode 100644 index 00000000..1617f48c --- /dev/null +++ b/lib/deploy_ex/cloud/infrastructure.ex @@ -0,0 +1,21 @@ +defmodule DeployEx.Cloud.Infrastructure do + @moduledoc """ + Network, image and key discovery needed to launch an instance ad-hoc. + + Every callback answers "what already exists in this account/tenancy that a new instance + should attach to". Terraform owns CREATING these; this behaviour only reads them. + """ + + @callback find_network(keyword()) :: {:ok, String.t()} | {:error, ErrorMessage.t()} + + @callback find_subnet(keyword()) :: {:ok, String.t()} | {:error, ErrorMessage.t()} + + @callback find_key_pair(String.t(), keyword()) :: + {:ok, String.t()} | {:error, ErrorMessage.t()} + + @callback find_image(keyword()) :: {:ok, String.t()} | {:error, ErrorMessage.t()} + + @doc "Identity the instance assumes so it can read releases and write state markers." + @callback find_instance_identity(keyword()) :: + {:ok, String.t() | nil} | {:error, ErrorMessage.t()} +end diff --git a/lib/deploy_ex/cloud/instance.ex b/lib/deploy_ex/cloud/instance.ex new file mode 100644 index 00000000..e24cdb38 --- /dev/null +++ b/lib/deploy_ex/cloud/instance.ex @@ -0,0 +1,36 @@ +defmodule DeployEx.Cloud.Instance do + @moduledoc """ + Provider-neutral description of a single compute instance. + + Deliberately does NOT derive `Jason.Encoder`. The `--format json` output of + `mix deploy_ex.find_nodes` is a frozen user-visible contract whose key set differs from + these field names, so the mapping is written explicitly at the output site rather than + falling out of a derive. + """ + + defstruct [ + :id, + :ipv6, + :launched_at, + :name, + :private_ip, + :public_ip, + :qa_node?, + :state, + :tags, + :type + ] + + @type t :: %__MODULE__{ + id: String.t() | nil, + ipv6: String.t() | nil, + launched_at: DateTime.t() | String.t() | nil, + name: String.t() | nil, + private_ip: String.t() | nil, + public_ip: String.t() | nil, + qa_node?: boolean() | nil, + state: atom() | String.t() | nil, + tags: %{optional(String.t()) => String.t()} | nil, + type: String.t() | nil + } +end diff --git a/lib/deploy_ex/cloud/machine.ex b/lib/deploy_ex/cloud/machine.ex new file mode 100644 index 00000000..04a1d90a --- /dev/null +++ b/lib/deploy_ex/cloud/machine.ex @@ -0,0 +1,87 @@ +defmodule DeployEx.Cloud.Machine do + @moduledoc """ + Compute instance discovery and lifecycle. + + Tag filters are a LIST of `{key, matcher}` pairs, never a map. A map would collapse + repeated keys and turn today's AND semantics on `--tag Env=a --tag Env=b` into + "last one wins". + + A matcher is a scalar, a list of scalars, or a `Regex`. Only exact scalar and list + matchers may be pushed down into a provider-native query; a regex is evaluated + client-side against the decoded canonical tag map. Callers rely on the regex arm today + because autoscaling-group instances carry composite tag values. + """ + + alias DeployEx.Cloud.Instance + + @typedoc "A single tag value to match against" + @type scalar :: String.t() | boolean() | number() + + @typedoc "Exact scalar, any-of list, or a client-side evaluated pattern" + @type matcher :: scalar() | [scalar()] | Regex.t() + + @typedoc "Canonical tag filters, AND-ed together" + @type tag_filters :: [{String.t(), matcher()}] + + @doc """ + Raw tag-filter lookup returning normalized instances. + + Named `list_instances` rather than `find_instances_by_tags` on purpose: `AwsMachine` already + exports a function by the latter name that returns provider-shaped maps to seven Mix-task + call sites, and those keep their shape until the caller sweep rewires them. Two different + return types must not share one name. + """ + @callback list_instances(tag_filters(), keyword()) :: + {:ok, [Instance.t()]} | {:error, ErrorMessage.t()} + + @doc """ + Instances belonging to one app within one project. + + This is the caller-facing lookup behind `deploy_ex.ssh`, `restart_app` and the EBS tasks. + Three clauses are part of the CONTRACT, not incidental to the AWS implementation — an + implementation that drops any of them is non-conforming: + + 1. **Project scope is unconditional.** Results are restricted to the active project's + resource group. It is never caller-supplied and never optional; omitting it makes a + shared cloud account return another project's instances. + 2. **Only running instances** are returned. + 3. **Instances with no instance-group tag are excluded**, so half-provisioned machines + never surface as deploy targets. + + `find_instances_by_tags/2` applies none of these — it is the raw filter primitive. Do not + implement this callback by delegating to it without adding all three. + """ + @callback find_app_instances(String.t(), String.t(), keyword()) :: + {:ok, [Instance.t()]} | {:error, ErrorMessage.t()} + + @callback describe_instance(String.t(), keyword()) :: + {:ok, Instance.t()} | {:error, ErrorMessage.t()} + + @callback start_instance(String.t(), keyword()) :: :ok | {:error, ErrorMessage.t()} + + @callback stop_instance(String.t(), keyword()) :: :ok | {:error, ErrorMessage.t()} + + @callback terminate_instance(String.t(), keyword()) :: :ok | {:error, ErrorMessage.t()} + + @callback run_instance(map(), keyword()) :: + {:ok, Instance.t()} | {:error, ErrorMessage.t()} + + @doc "Preferred reachable address for an instance. IPv6 wins when present." + @callback instance_address(Instance.t()) :: {:ok, String.t()} | {:error, ErrorMessage.t()} + + @callback fetch_tags(String.t(), keyword()) :: + {:ok, %{optional(String.t()) => String.t()}} | {:error, ErrorMessage.t()} + + @callback put_tags(String.t(), %{optional(String.t()) => String.t()}, keyword()) :: + :ok | {:error, ErrorMessage.t()} + + @callback delete_tags(String.t(), [String.t()], keyword()) :: + :ok | {:error, ErrorMessage.t()} + + # Instance creation, termination and tag writes currently live inside the QA-node and + # load-test subsystems, which Phase 5 extracts. Optional keeps the contract honest: a + # provider conforms today without them, and the Phase-5 train makes them required when it + # moves those call sites behind this behaviour. Implementing them now would be unused code + # with no test that could fail. + @optional_callbacks run_instance: 2, terminate_instance: 2, put_tags: 3, delete_tags: 3 +end diff --git a/lib/deploy_ex/cloud/object_store.ex b/lib/deploy_ex/cloud/object_store.ex new file mode 100644 index 00000000..be23bf45 --- /dev/null +++ b/lib/deploy_ex/cloud/object_store.ex @@ -0,0 +1,40 @@ +defmodule DeployEx.Cloud.ObjectStore do + @moduledoc """ + Blob storage. Deliberately not an S3 interface. + + S3-compatible storage is the first implementation, not the contract. Azure Blob has no + S3-compatible API, so nothing here names buckets-as-S3, ETags, or multipart uploads. + + `put_object_tags/4` is OPTIONAL. Providers without native object tagging inherit a + portable default rather than being blocked. + """ + + @type key :: String.t() + @type container :: String.t() + + @callback get_object(container(), key(), keyword()) :: + {:ok, binary()} | {:error, ErrorMessage.t()} + + @callback put_object(container(), key(), binary(), keyword()) :: + :ok | {:error, ErrorMessage.t()} + + @callback delete_object(container(), key(), keyword()) :: :ok | {:error, ErrorMessage.t()} + + @doc "Lists keys under a prefix, following pagination to completion." + @callback list_objects(container(), keyword()) :: + {:ok, [key()]} | {:error, ErrorMessage.t()} + + @callback upload_file(container(), key(), Path.t(), keyword()) :: + :ok | {:error, ErrorMessage.t()} + + @callback put_object_tags(container(), key(), %{optional(String.t()) => String.t()}, keyword()) :: + :ok | {:error, ErrorMessage.t()} + + @callback create_container(container(), keyword()) :: :ok | {:error, ErrorMessage.t()} + + @callback delete_container(container(), keyword()) :: :ok | {:error, ErrorMessage.t()} + + @callback list_containers(keyword()) :: {:ok, [container()]} | {:error, ErrorMessage.t()} + + @optional_callbacks put_object_tags: 4 +end diff --git a/lib/deploy_ex/cloud/oci_cli.ex b/lib/deploy_ex/cloud/oci_cli.ex new file mode 100644 index 00000000..95694d41 --- /dev/null +++ b/lib/deploy_ex/cloud/oci_cli.ex @@ -0,0 +1,127 @@ +defmodule DeployEx.Cloud.OciCli do + @moduledoc """ + Runner for the `oci` CLI, which is how deploy_ex talks to Oracle Cloud. + + No usable OCI SDK exists for Elixir or Erlang: `ex_oci_sdk` covers only the Queue service, + and "OCI" in the Erlang ecosystem means Oracle Call Interface, a database driver for a + different product. ExAws cannot substitute either — its partition table is compile-time, so + an OCI region can never be registered, and signing with an AWS region against an OCI + endpoint returns 403 because OCI requires the OCI region inside the SigV4 credential scope. + + `OCI_CLI_AUTH` selects the credential source: `api_key` on a workstation or in CI, and + `instance_principal` on an OCI instance, which is the analogue of an EC2 instance profile. + Session-token auth is deliberately not the default because it needs a browser and expires. + """ + + alias DeployEx.{Config, Utils} + + @doc """ + Runs an `oci` subcommand and returns its raw stdout. + + `:run_fn` in `opts` replaces the shell call. It is the injection seam that makes output + parsing testable without a live tenancy, matching the `:request_fn` seam in + `DeployEx.Cloud.S3ObjectStore`. + """ + @spec run(String.t(), keyword()) :: {:ok, String.t()} | {:error, ErrorMessage.t()} + def run(subcommand, opts \\ []) do + command = build_command(subcommand, opts) + run_fn = opts[:run_fn] || (&Utils.run_command_with_return/2) + + case run_fn.(command, File.cwd!()) do + {:ok, output} -> {:ok, output} + {:error, error} -> {:error, classify_error(error, command)} + end + end + + @doc """ + Runs an `oci` subcommand and decodes its JSON payload. + + The CLI prints NOTHING — not `{"data": []}` — when a list matches no resources, so empty + output decodes to an empty map rather than a JSON error. Without this an empty compartment + is indistinguishable from a broken command. + """ + @spec run_json(String.t(), keyword()) :: {:ok, map()} | {:error, ErrorMessage.t()} + def run_json(subcommand, opts \\ []) do + with {:ok, output} <- run("#{subcommand} --output json", opts) do + decode_payload(output) + end + end + + @doc """ + Reads an OCI setting from opts first, then application config. + + Opts keys carry an `oci_` prefix (`:oci_region`, `:oci_profile`), matching the convention + `Mix.Tasks.Ansible.Build` already uses. The prefix is load-bearing, not cosmetic: opts + reaching here often came from an AWS-shaped caller carrying a bare `:region` of `us-west-2`, + and reading that as the OCI region would send every call to a region the tenancy is not in. + """ + @spec setting(keyword(), atom()) :: term() + def setting(opts, key), do: opts[:"oci_#{key}"] || Config.oci_setting(key) + + # SUPPRESS_LABEL_WARNING silences a stderr nag about unlabeled API keys. That nag would + # otherwise land in the merged stdout/stderr stream run_command_with_return/2 returns and + # break JSON decoding. + defp build_command(subcommand, opts) do + auth = setting(opts, :auth) || "api_key" + flags = build_flags(opts) + + String.trim("OCI_CLI_AUTH=#{auth} SUPPRESS_LABEL_WARNING=True oci #{subcommand} #{flags}") + end + + defp build_flags(opts) do + [{"--profile", setting(opts, :profile)}, {"--region", setting(opts, :region)}] + |> Enum.reject(fn {_flag, value} -> is_nil(value) end) + |> Enum.map_join(" ", fn {flag, value} -> "#{flag} #{value}" end) + end + + defp decode_payload(output) do + case String.trim(output) do + "" -> {:ok, %{}} + trimmed -> decode_json(trimmed, output) + end + end + + defp decode_json(trimmed, original) do + case Jason.decode(trimmed) do + {:ok, decoded} -> + {:ok, decoded} + + {:error, decode_error} -> + {:error, + ErrorMessage.internal_server_error( + "failed to decode oci CLI JSON output: #{Exception.message(decode_error)}", + %{output: original} + )} + end + end + + # A failed call prints `ServiceError:` followed by a JSON body carrying the real HTTP status. + # Mapping that status onto an ErrorMessage is what lets callers tell "bucket already exists" + # (409) and "no such object" (404) apart from a genuine failure, rather than treating every + # non-zero exit as opaque. + defp classify_error(%ErrorMessage{details: %{output: output}} = error, command) do + case parse_service_error(output) do + nil -> error + {status, message} -> build_service_error(status, message, command, output) + end + end + + defp classify_error(error, _command), do: error + + defp parse_service_error(output) do + with [_prefix, json] <- String.split(output, "ServiceError:", parts: 2), + {:ok, %{"status" => status} = body} <- Jason.decode(String.trim(json)) do + {status, body["message"] || "oci service error"} + else + _no_service_error -> nil + end + end + + defp build_service_error(status, message, command, output) do + %ErrorMessage{ + code: ErrorMessage.http_code_reason_atom(status), + message: message, + details: %{command: command, output: output, status: status} + } + end +end diff --git a/lib/deploy_ex/cloud/oci_object_store.ex b/lib/deploy_ex/cloud/oci_object_store.ex new file mode 100644 index 00000000..26873937 --- /dev/null +++ b/lib/deploy_ex/cloud/oci_object_store.ex @@ -0,0 +1,172 @@ +defmodule DeployEx.Cloud.OciObjectStore do + @moduledoc """ + OCI Object Storage implementation of `DeployEx.Cloud.ObjectStore`, driven by the `oci` CLI. + + Not built on OCI's S3-compatible endpoint. That endpoint needs long-lived Customer Secret + Keys and does NOT accept instance principals, so a node could not read its own release + bucket without a credential on disk. The native API accepts instance principals, which is + the OCI analogue of an EC2 instance profile — the same shape the AWS path already relies on. + + Every list call passes `--all`. That flag is OCI's equivalent of following S3 pagination to + completion, and `DeployEx.ReleaseUploader` reads release history from these listings: a + listing that silently stops after one page reports a fraction of the releases as if it were + all of them, with no error. + """ + + @behaviour DeployEx.Cloud.ObjectStore + + alias DeployEx.Cloud.OciCli + + @impl DeployEx.Cloud.ObjectStore + def get_object(container, key, opts \\ []) do + path = temp_path("get") + + with {:ok, _output} <- + OciCli.run("os object get#{namespace_flag(opts)} --bucket-name #{quote_arg(container)} --name #{quote_arg(key)} --file #{quote_arg(path)}", opts) do + read_and_discard(path) + end + end + + @impl DeployEx.Cloud.ObjectStore + def put_object(container, key, body, opts \\ []) do + path = temp_path("put") + + with :ok <- File.write(path, body) do + result = upload_file(container, key, path, opts) + + File.rm(path) + + result + end + end + + @impl DeployEx.Cloud.ObjectStore + def upload_file(container, key, path, opts \\ []) do + with {:ok, _output} <- + OciCli.run("os object put#{namespace_flag(opts)} --bucket-name #{quote_arg(container)} --name #{quote_arg(key)} --file #{quote_arg(path)} --force", opts) do + :ok + end + end + + @impl DeployEx.Cloud.ObjectStore + def delete_object(container, key, opts \\ []) do + with {:ok, _output} <- + OciCli.run("os object delete#{namespace_flag(opts)} --bucket-name #{quote_arg(container)} --name #{quote_arg(key)} --force", opts) do + :ok + end + end + + @impl DeployEx.Cloud.ObjectStore + def list_objects(container, opts \\ []) do + command = "os object list#{namespace_flag(opts)} --bucket-name #{quote_arg(container)} --all#{prefix_flag(opts)}" + + with {:ok, payload} <- OciCli.run_json(command, opts) do + {:ok, payload |> Map.get("data", []) |> Enum.map(&(&1["name"]))} + end + end + + @impl DeployEx.Cloud.ObjectStore + def create_container(container, opts \\ []) do + with {:ok, compartment_id} <- require_compartment_id(opts), + {:ok, _output} <- + OciCli.run("os bucket create#{namespace_flag(opts)} --compartment-id #{compartment_id} --name #{quote_arg(container)}", opts) do + :ok + end + end + + @impl DeployEx.Cloud.ObjectStore + def delete_container(container, opts \\ []) do + with {:ok, _output} <- OciCli.run("os bucket delete#{namespace_flag(opts)} --bucket-name #{quote_arg(container)} --force", opts) do + :ok + end + end + + @doc """ + Lists buckets in the compartment. + + Returns maps carrying `:name`, matching what `DeployEx.Cloud.S3ObjectStore` returns and what + `terraform.create_state_bucket`/`drop_state_bucket` read, rather than the bare strings the + behaviour's typespec names. The typespec is the odd one out here — changing it is a separate + correction from making OCI work. + """ + @impl DeployEx.Cloud.ObjectStore + def list_containers(opts \\ []) do + with {:ok, compartment_id} <- require_compartment_id(opts), + {:ok, payload} <- OciCli.run_json("os bucket list#{namespace_flag(opts)} --compartment-id #{compartment_id} --all", opts) do + {:ok, payload |> Map.get("data", []) |> Enum.map(&bucket_summary/1)} + end + end + + @doc """ + Object tagging, which OCI Object Storage does not offer. + + Its nearest equivalent is user metadata, settable only at put time — there is no + `oci os object update-metadata`, so tagging an already-uploaded object would mean + re-uploading it. Implemented as an honest error rather than left out: the sole caller is + `DeployEx.ReleaseUploader`'s `qa_release: true` path, and an unimplemented optional callback + would reach it as an UndefinedFunctionError instead of a message naming the limitation. + """ + @impl DeployEx.Cloud.ObjectStore + def put_object_tags(container, key, _tags, _opts \\ []) do + {:error, + ErrorMessage.not_implemented( + "oci object storage has no object tagging; tag data must be encoded in the key prefix", + %{container: container, key: key} + )} + end + + defp bucket_summary(bucket) do + %{name: bucket["name"], creation_date: bucket["time-created"]} + end + + defp require_compartment_id(opts) do + case OciCli.setting(opts, :compartment_id) do + nil -> + {:error, + ErrorMessage.bad_request( + "oci compartment_id is required for bucket operations " <> + "(config :deploy_ex, :oci, compartment_id: \"...\")" + )} + + compartment_id -> + {:ok, compartment_id} + end + end + + # Passed explicitly rather than left to the CLI's auto-resolution. Resolving the namespace + # internally requires a tenancy read, which a correctly-scoped CI credential does NOT have — + # MEASURED: a user holding only object permissions fails every `oci os` call with "Unable to + # retrieve namespace internally. Please provide the namespace using the option + # --['namespace-name']". Auto-resolution only appears to work when the credential is + # over-privileged, so relying on it quietly punishes least privilege. + defp namespace_flag(opts) do + case OciCli.setting(opts, :namespace) do + namespace when is_binary(namespace) and namespace !== "" -> " --namespace #{quote_arg(namespace)}" + _absent -> "" + end + end + + defp prefix_flag(opts) do + case opts[:prefix] do + prefix when is_binary(prefix) and prefix !== "" -> " --prefix #{quote_arg(prefix)}" + _absent -> "" + end + end + + defp read_and_discard(path) do + result = File.read(path) + + File.rm(path) + + case result do + {:ok, body} -> {:ok, body} + {:error, reason} -> {:error, ErrorMessage.internal_server_error("could not read downloaded object", %{reason: reason})} + end + end + + defp temp_path(kind) do + Path.join(System.tmp_dir!(), "deploy_ex_oci_#{kind}_#{System.unique_integer([:positive])}") + end + + defp quote_arg(value), do: "'#{String.replace(to_string(value), "'", "'\\''")}'" +end diff --git a/lib/deploy_ex/cloud/oci_security_group.ex b/lib/deploy_ex/cloud/oci_security_group.ex new file mode 100644 index 00000000..4f471925 --- /dev/null +++ b/lib/deploy_ex/cloud/oci_security_group.ex @@ -0,0 +1,188 @@ +defmodule DeployEx.Cloud.OciSecurityGroup do + @moduledoc """ + OCI implementation of `DeployEx.Cloud.Security`, backed by a Network Security Group (NSG). + + ## NSG, not security list + + OCI has two candidates for "the thing `mix deploy_ex.ssh.authorize` opens a hole in": Network + Security Groups, which attach to a VNIC, and security lists, which attach to a SUBNET. + `priv/terraform/providers/oci/network.tf` already creates a security list for the subnet's + baseline rules (open egress, an optional static SSH CIDR) — this module does not touch it. + + NSG was chosen for one decisive reason: the `oci network nsg rules` API is additive. + `add`/`remove` operate on individual rules by content or by ID; nothing in this module ever + reads the full rule set and writes it back. A security list has no such API — + `oci network security-list update --security-rules` REPLACES the entire list, so an ingress + toggle implemented against a security list has to read-merge-write the whole set on every + call, and a bug there silently deletes every rule terraform created. NSGs make that bug class + unreachable by construction. + + Blast radius, measured, is a wash in THIS codebase specifically — it is not the deciding + factor. `priv/terraform/providers/oci/modules/oci-instance` attaches the same `nsg_ids` list + to every instance's VNIC, the same way the security list already applies to every instance via + the one shared subnet. A real per-instance NSG assignment would narrow the grant below what + the security list allows; that is not how it is wired up today. If per-instance isolation is + wanted later, only the terraform wiring in `instance.tf.eex` needs to change — this module + already operates on whatever NSG id `find_group/1` resolves, one call site, not fanned out per + instance. + + ## Idempotency, measured + + MEASURED against a live tenancy: `oci network nsg rules add` is idempotent for identical + `(direction, protocol, source, port)` content — adding the same rule twice returns the SAME + rule id both times and the NSG ends up with one rule, not two, regardless of a differing + `description`. AWS's `authorize_ingress` instead gets a hard "already exists" error back from + EC2 that `AwsSecurityGroup.classify_ingress_error/3` turns into a conflict. + `authorize_ingress/3` here does NOT reimplement that check — doing so would fight OCI's native + idempotency for no benefit, and an idempotent authorize is what a retried + `mix deploy_ex.ssh.authorize` (or a re-run CI job) actually wants. + + `remove` has no content-addressed form — it deletes by rule ID, and removing an ID that no + longer exists is a 400. `revoke_ingress/3` reads the rule list first so it never calls + `remove` with a stale ID; a CIDR with no matching rule is a no-op `:ok`, not an error. + """ + + @behaviour DeployEx.Cloud.Security + + alias DeployEx.Cloud.OciCli + + @ssh_port 22 + @tcp_protocol "6" + + @impl DeployEx.Cloud.Security + def find_group(opts \\ []), do: find_nsg_id(opts) + + @impl DeployEx.Cloud.Security + def authorize_ingress(nsg_id, cidr, opts \\ []) do + command = add_rule_command(nsg_id, cidr) + + with {:ok, _output} <- OciCli.run(command, opts), do: :ok + end + + @impl DeployEx.Cloud.Security + def revoke_ingress(nsg_id, cidr, opts \\ []) do + with {:ok, rules} <- list_ingress_rules(nsg_id, opts) do + case Enum.find(rules, &matches_cidr?(&1, cidr)) do + nil -> :ok + rule -> remove_rule(nsg_id, rule["id"], opts) + end + end + end + + defp add_rule_command(nsg_id, cidr) do + rule_json = Jason.encode!([ingress_rule(cidr)]) + + "network nsg rules add --nsg-id #{quote_arg(nsg_id)} --security-rules #{quote_arg(rule_json)}" + end + + defp ingress_rule(cidr) do + %{ + direction: "INGRESS", + protocol: @tcp_protocol, + source: cidr, + sourceType: "CIDR_BLOCK", + isStateless: false, + tcpOptions: %{destinationPortRange: %{min: @ssh_port, max: @ssh_port}} + } + end + + defp remove_rule(nsg_id, rule_id, opts) do + ids_json = Jason.encode!([rule_id]) + + command = "network nsg rules remove --nsg-id #{quote_arg(nsg_id)} --security-rule-ids #{quote_arg(ids_json)}" + + with {:ok, _output} <- OciCli.run(command, opts), do: :ok + end + + defp list_ingress_rules(nsg_id, opts) do + command = "network nsg rules list --nsg-id #{quote_arg(nsg_id)} --direction INGRESS --all" + + with {:ok, payload} <- OciCli.run_json(command, opts) do + {:ok, Map.get(payload, "data", [])} + end + end + + defp matches_cidr?(rule, cidr) do + rule["protocol"] === @tcp_protocol and rule["source"] === cidr and + rule["source-type"] === "CIDR_BLOCK" and ssh_port?(rule["tcp-options"]) + end + + defp ssh_port?(%{"destination-port-range" => %{"min" => @ssh_port, "max" => @ssh_port}}), do: true + defp ssh_port?(_no_tcp_options), do: false + + defp find_nsg_id(opts) do + case opts[:security_group_id] do + nil -> find_nsg_by_prefix(opts) + nsg_id -> verify_nsg_exists(nsg_id, opts) + end + end + + # An explicit override is checked against a live `get` rather than trusted as-is, so a stale + # or mistyped id fails here with a clear not_found instead of confusing every ingress call + # that follows. + defp verify_nsg_exists(nsg_id, opts) do + with {:ok, _payload} <- OciCli.run_json("network nsg get --nsg-id #{quote_arg(nsg_id)}", opts) do + {:ok, nsg_id} + end + end + + defp find_nsg_by_prefix(opts) do + with {:ok, compartment_id} <- require_compartment_id(opts), + {:ok, nsgs} <- list_nsgs(compartment_id, opts) do + prefix = nsg_prefix(opts) + + case matching_nsg(nsgs, prefix) do + nil -> {:error, no_nsg_match_error(nsgs, prefix)} + nsg -> {:ok, nsg["id"]} + end + end + end + + defp no_nsg_match_error(nsgs, prefix) do + available = nsgs |> Enum.map(& &1["display-name"]) |> Enum.filter(& &1) + + ErrorMessage.not_found("no network security group found matching prefix #{prefix}", %{ + available: available + }) + end + + defp matching_nsg(nsgs, prefix) do + nsgs + |> Enum.filter(fn nsg -> matches_prefix?(nsg["display-name"] || "", prefix) end) + |> Enum.sort_by(& &1["display-name"], :desc) + |> List.first() + end + + defp matches_prefix?(name, prefix), do: name === prefix or String.starts_with?(name, prefix) + + defp list_nsgs(compartment_id, opts) do + command = "network nsg list --compartment-id #{quote_arg(compartment_id)} --all" + + with {:ok, payload} <- OciCli.run_json(command, opts) do + {:ok, Map.get(payload, "data", [])} + end + end + + defp nsg_prefix(opts) do + project_name = opts[:project_name] || DeployExHelpers.kebab_project_name() + environment = opts[:environment] || DeployEx.Config.env() + + "#{project_name}-#{environment}-nsg" + end + + defp require_compartment_id(opts) do + case OciCli.setting(opts, :compartment_id) do + nil -> {:error, missing_compartment_id_error()} + compartment_id -> {:ok, compartment_id} + end + end + + defp missing_compartment_id_error do + ErrorMessage.bad_request( + "oci compartment_id is required for security group operations " <> + "(config :deploy_ex, :oci, compartment_id: \"...\")" + ) + end + + defp quote_arg(value), do: "'#{String.replace(to_string(value), "'", "'\\''")}'" +end diff --git a/lib/deploy_ex/cloud/priv_file_set.ex b/lib/deploy_ex/cloud/priv_file_set.ex new file mode 100644 index 00000000..28830f31 --- /dev/null +++ b/lib/deploy_ex/cloud/priv_file_set.ex @@ -0,0 +1,85 @@ +defmodule DeployEx.Cloud.PrivFileSet do + @moduledoc """ + Selects which `priv/` templates belong to a provider. + + Every non-AWS provider lives under a `providers//` directory. AWS is the complement — + everything NOT under a `providers/` directory — expressed as one rule so adding a provider + never requires rewording it. + + Non-AWS files FLATTEN on the way out: `providers/oci/network.tf.eex` renders to `network.tf` + at the terraform root, because tofu only loads root-level `.tf` and runs in the configured + terraform folder. + + The `providers` predicate is a path-COMPONENT test, never a substring match — an AWS tree + legitimately contains a rendered `providers.tf` at its root, and a prefix match would + misclassify it as provider-scoped and drop it from the AWS set. + """ + + @providers_dir "providers" + + @doc """ + Source/destination pairs for `provider`, relative to the given priv subdirectory. + + Returns `{:ok, [{source_relative, dest_relative}]}`, or an error when the provider has no + files of its own — which is how an unimplemented provider surfaces rather than silently + seeding an empty tree. + """ + @spec files(atom(), Path.t()) :: {:ok, [{Path.t(), Path.t()}]} | {:error, ErrorMessage.t()} + def files(provider, priv_path) do + case provider_relative_paths(provider, priv_path) do + [] -> + {:error, + ErrorMessage.not_implemented("no #{provider} templates exist in #{priv_path}", %{ + provider: provider, + priv_path: priv_path + })} + + paths -> + {:ok, paths} + end + end + + @doc "True when `relative_path` belongs to `provider`'s file set." + @spec member?(atom(), Path.t()) :: boolean() + def member?(:aws, relative_path), do: not provider_scoped?(relative_path) + + def member?(provider, relative_path) do + case Path.split(relative_path) do + [@providers_dir, name | _rest] -> name === to_string(provider) + _not_scoped -> false + end + end + + @doc """ + Destination for a source path under `provider`. + + AWS keeps its layout. Everything else drops the `providers//` prefix so the file lands + where tofu and ansible actually look for it. + """ + @spec destination(atom(), Path.t()) :: Path.t() + def destination(:aws, relative_path), do: relative_path + + def destination(provider, relative_path) do + case Path.split(relative_path) do + [@providers_dir, name | rest] when rest !== [] -> + if name === to_string(provider), do: Path.join(rest), else: relative_path + + _not_scoped -> + relative_path + end + end + + defp provider_relative_paths(provider, priv_path) do + priv_path + |> Path.join("**") + |> Path.wildcard(match_dot: false) + |> Enum.reject(&File.dir?/1) + |> Enum.map(&Path.relative_to(&1, priv_path)) + |> Enum.filter(&member?(provider, &1)) + |> Enum.map(&{&1, destination(provider, &1)}) + end + + defp provider_scoped?(relative_path) do + @providers_dir in (relative_path |> Path.split() |> Enum.drop(-1)) + end +end diff --git a/lib/deploy_ex/cloud/provider.ex b/lib/deploy_ex/cloud/provider.ex new file mode 100644 index 00000000..6de0ea98 --- /dev/null +++ b/lib/deploy_ex/cloud/provider.ex @@ -0,0 +1,45 @@ +defmodule DeployEx.Cloud.Provider do + @moduledoc """ + Descriptor behaviour every cloud provider implements. + + A descriptor declares what a provider IS; `DeployEx.Cloud` derives dispatch from it, so + registering a provider costs one descriptor module plus one registry entry rather than + edits scattered through the dispatcher. + + Slots a provider has not implemented yet return `nil`, and `capabilities/0` simply omits + the key. Both surface as `{:error, %ErrorMessage{code: :not_implemented}}` at the call + site instead of a crash or a silent nil. + """ + + @typedoc "Capability name to the module implementing that capability behaviour" + @type capabilities :: %{optional(atom()) => module()} + + @typedoc "Where the inventory for this provider comes from and what it renders to" + @type inventory :: %{strategy: atom(), template: String.t(), filename: String.t()} + + @doc "Map of capability name to implementing module. Unimplemented capabilities are omitted." + @callback capabilities() :: capabilities() + + @doc "NimbleOptions schema validating this provider's config namespace at task start." + @callback config_schema() :: keyword() + + @doc "Terraform backend template identifier, or nil until the provider's phase fills it." + @callback backend_template() :: atom() | nil + + @doc "Strategy used to mark an instance as finished provisioning." + @callback completion_marker() :: atom() | nil + + @doc "Inventory strategy, source template path and rendered filename." + @callback inventory() :: inventory() | nil + + @doc "SSH user to use when the neutral `:ssh_user` config is unset." + @callback default_ssh_user() :: String.t() | nil + + @doc """ + Module adapting this provider's CLI to a shared runner, or nil when the provider uses an SDK. + + The shared runner does not exist yet — it arrives with the first CLI-backed provider. AWS + returns nil because ExAws covers it. + """ + @callback cli_adapter() :: module() | nil +end diff --git a/lib/deploy_ex/cloud/providers/aws.ex b/lib/deploy_ex/cloud/providers/aws.ex new file mode 100644 index 00000000..67950810 --- /dev/null +++ b/lib/deploy_ex/cloud/providers/aws.ex @@ -0,0 +1,47 @@ +defmodule DeployEx.Cloud.Providers.Aws do + @moduledoc """ + Descriptor for AWS, the default provider. + + `object_store` is intentionally absent from `capabilities/0` until `S3ObjectStore` + exists. Pointing the slot at a module that has not been written would be a dangling + reference that only fails at call time; omitting it yields an honest `:not_implemented`. + """ + + @behaviour DeployEx.Cloud.Provider + + @config_schema [*: [type: :any]] + + @impl DeployEx.Cloud.Provider + def capabilities do + %{ + machine: DeployEx.AwsMachine, + object_store: DeployEx.Cloud.S3ObjectStore, + infrastructure: DeployEx.AwsInfrastructure, + security: DeployEx.AwsSecurityGroup + } + end + + @impl DeployEx.Cloud.Provider + def config_schema, do: @config_schema + + @impl DeployEx.Cloud.Provider + def backend_template, do: :s3 + + @impl DeployEx.Cloud.Provider + def completion_marker, do: :ci_tag + + @impl DeployEx.Cloud.Provider + def inventory do + %{ + strategy: :aws_ec2_plugin, + template: "ansible/aws_ec2.yaml.eex", + filename: "aws_ec2.yaml" + } + end + + @impl DeployEx.Cloud.Provider + def default_ssh_user, do: "admin" + + @impl DeployEx.Cloud.Provider + def cli_adapter, do: nil +end diff --git a/lib/deploy_ex/cloud/providers/oci.ex b/lib/deploy_ex/cloud/providers/oci.ex new file mode 100644 index 00000000..2505c011 --- /dev/null +++ b/lib/deploy_ex/cloud/providers/oci.ex @@ -0,0 +1,78 @@ +defmodule DeployEx.Cloud.Providers.Oci do + @moduledoc """ + Descriptor skeleton for Oracle Cloud Infrastructure. + + Slots fill per phase. One invented ahead of its phase would be untested guesswork that reads + as working code, so an unfilled slot stays `nil` and surfaces as + `{:error, %ErrorMessage{code: :not_implemented}}` rather than a plausible default. + `object_store`, `inventory` and `security` are filled; compute and networking are not. + + The config schema is the exception: it is strict from the start so a typo'd key fails at + task start rather than mid-apply. Every key is optional — the schema catches mistakes, it + does not force configuration. + """ + + @behaviour DeployEx.Cloud.Provider + + @config_schema [ + region: [type: {:or, [:string, nil]}], + home_region: [type: {:or, [:string, nil]}], + profile: [type: {:or, [:string, nil]}], + auth: [type: {:or, [:string, nil]}], + compartment_id: [type: {:or, [:string, nil]}], + namespace: [type: {:or, [:string, nil]}], + availability_domain: [type: {:or, [:string, nil]}], + base_image: [type: {:or, [:keyword_list, :string, nil]}], + shape: [type: {:or, [:string, nil]}], + shape_ocpus: [type: {:or, [:pos_integer, nil]}], + shape_memory_gbs: [type: {:or, [:pos_integer, nil]}], + release_bucket: [type: {:or, [:string, nil]}], + release_state_bucket: [type: {:or, [:string, nil]}], + release_state_key: [type: {:or, [:string, nil]}], + vcn_cidr: [type: {:or, [:string, nil]}], + state_profile: [type: {:or, [:string, nil]}], + log_bucket: [type: {:or, [:string, nil]}], + log_region: [type: {:or, [:string, nil]}], + resource_group: [type: {:or, [:string, nil]}] + ] + + @impl DeployEx.Cloud.Provider + def capabilities do + %{ + object_store: DeployEx.Cloud.OciObjectStore, + security: DeployEx.Cloud.OciSecurityGroup + } + end + + @impl DeployEx.Cloud.Provider + def config_schema, do: @config_schema + + # State rides OCI's S3-compatibility endpoint — there is no native OCI backend in + # terraform, so the :s3 backend with a Customer Secret Key is the only remote option. + @impl DeployEx.Cloud.Provider + def backend_template, do: :s3 + + # Filled by Phase 2 (cloud-init completion marker, spike S5). + @impl DeployEx.Cloud.Provider + def completion_marker, do: nil + + # Static, not a plugin: no oci ansible collection dependency wanted, so + # Mix.Tasks.Ansible.Build queries the oci CLI directly and renders this template into a + # point-in-time snapshot. See priv/ansible/providers/oci/README.md for the generator. + @impl DeployEx.Cloud.Provider + def inventory do + %{strategy: :static_oci_cli, template: "ansible/providers/oci/oci.yaml.eex", filename: "oci.yaml"} + end + + # OCI's Ubuntu images have no `admin` user (AWS's default) — see + # priv/ansible/providers/oci/ansible.cfg.eex, which bakes this in directly rather than + # reading this slot, since ansible.cfg is a static file rendered once, not something a + # runtime caller looks up per request. Filled for parity with the AWS descriptor and any + # future caller that needs the ssh user without parsing a rendered ansible.cfg. + @impl DeployEx.Cloud.Provider + def default_ssh_user, do: "ubuntu" + + # Filled by Phase 3 (oci CLI adapter). + @impl DeployEx.Cloud.Provider + def cli_adapter, do: nil +end diff --git a/lib/deploy_ex/cloud/s3_object_store.ex b/lib/deploy_ex/cloud/s3_object_store.ex new file mode 100644 index 00000000..d60f87bc --- /dev/null +++ b/lib/deploy_ex/cloud/s3_object_store.ex @@ -0,0 +1,245 @@ +defmodule DeployEx.Cloud.S3ObjectStore do + @moduledoc """ + S3-backed implementation of `DeployEx.Cloud.ObjectStore`. + + This is implementation #1 of the object-store contract, not the contract itself. OCI and + Google both expose S3-compatible endpoints, so they can reuse this module by pointing ExAws + at their host; Azure Blob has no S3 API and needs its own implementation behind the same + behaviour. + + `region` in opts, defaulting to `DeployEx.Config.aws_region/0`, is the only AWS-shaped + detail here, and ExAws needs it on every request. + """ + + @behaviour DeployEx.Cloud.ObjectStore + + alias ExAws.S3 + + @impl DeployEx.Cloud.ObjectStore + def get_object(container, key, opts \\ []) do + container + |> S3.get_object(key) + |> run(opts, %{container: container, key: key}) + |> unwrap_body() + end + + @impl DeployEx.Cloud.ObjectStore + def put_object(container, key, body, opts \\ []) do + container + |> S3.put_object(key, body) + |> run(opts, %{container: container, key: key}) + |> discard_body() + end + + @impl DeployEx.Cloud.ObjectStore + def delete_object(container, key, opts \\ []) do + container + |> S3.delete_object(key) + |> run(opts, %{container: container, key: key}) + |> discard_body() + end + + @impl DeployEx.Cloud.ObjectStore + def list_objects(container, opts \\ []) do + {request_opts, request_config} = Keyword.split(opts, [:prefix, :marker, :max_keys]) + + list_objects_page(container, request_opts, request_config, []) + end + + # S3 caps a list response at 1000 keys and signals more with is_truncated, so a single + # request silently truncates. The release bucket holds thousands of objects, and a + # truncated listing reads as "these are all the releases" rather than as an error. + defp list_objects_page(container, request_opts, request_config, acc) do + response = + container + |> S3.list_objects(request_opts) + |> run(request_config, %{container: container}) + + case response do + {:ok, %{body: body}} -> + keys = body |> Map.get(:contents, []) |> Enum.map(& &1.key) + accumulated = acc ++ keys + + if truncated?(body) and not Enum.empty?(keys) do + next_opts = Keyword.put(request_opts, :marker, next_marker(body, keys)) + + list_objects_page(container, next_opts, request_config, accumulated) + else + {:ok, accumulated} + end + + {:error, _reason} = error -> + error + end + end + + defp truncated?(body), do: Map.get(body, :is_truncated) in [true, "true"] + + defp next_marker(body, keys) do + case Map.get(body, :next_marker) do + marker when is_binary(marker) and marker !== "" -> marker + _absent -> List.last(keys) + end + end + + @impl DeployEx.Cloud.ObjectStore + def upload_file(container, key, path, opts \\ []) do + path + |> S3.Upload.stream_file() + |> S3.upload(container, key) + |> run(opts, %{container: container, key: key, path: path}) + |> discard_body() + end + + @impl DeployEx.Cloud.ObjectStore + def put_object_tags(container, key, tags, opts \\ []) do + container + |> S3.put_object_tagging(key, tags) + |> run(opts, %{container: container, key: key}) + |> discard_body() + end + + @impl DeployEx.Cloud.ObjectStore + def create_container(container, opts \\ []) do + region = region(opts) + + container + |> S3.put_bucket(region) + |> run(opts, %{container: container}) + |> discard_body() + end + + @impl DeployEx.Cloud.ObjectStore + def delete_container(container, opts \\ []) do + container + |> S3.delete_bucket() + |> run(opts, %{container: container}) + |> discard_body() + end + + @impl DeployEx.Cloud.ObjectStore + def list_containers(opts \\ []) do + case run(S3.list_buckets(), opts, %{region: region(opts)}) do + {:ok, %{body: %{buckets: buckets}}} -> {:ok, buckets} + {:error, _} = error -> error + end + end + + @doc """ + Deletes objects in a container, following pagination to completion. + + Not a behaviour callback — it is built on the S3 bulk-delete API, which has no portable + equivalent. A provider without bulk delete would loop `delete_object/3`. + + **Scope is mandatory and explicit.** Pass either `prefix: "some/path"` to delete a subtree, or + `all: true` to empty the whole container. Calling it with neither raises rather than deleting + everything, because the earlier signature accepted `prefix:` and silently ignored it — a call + that read as narrowly scoped emptied an entire production bucket. + """ + def delete_all_objects(container, opts \\ [], continuation_token \\ nil) do + scope = delete_scope!(container, opts) + list_opts = scope ++ continuation_opts(continuation_token) + + case run(S3.list_objects_v2(container, list_opts), opts, %{container: container}) do + {:ok, %{body: body}} -> delete_listed_objects(container, body, opts) + {:error, _reason} = error -> error + end + end + + defp delete_scope!(container, opts) do + prefix = opts[:prefix] + + cond do + is_binary(prefix) and prefix !== "" -> [prefix: prefix] + opts[:all] === true -> [] + true -> raise ArgumentError, unscoped_delete_message(container) + end + end + + defp unscoped_delete_message(container) do + "refusing to delete every object in #{inspect(container)} without an explicit scope — " <> + "pass prefix: \"...\" to delete a subtree, or all: true to empty the container" + end + + defp continuation_opts(token) do + if present?(token), do: [continuation_token: token], else: [] + end + + @doc """ + Maps an S3 error status onto an `ErrorMessage`. + + Public because it is the only part of this module that needs no live account. + """ + def classify_error(409, message, details) do + {:error, ErrorMessage.conflict("container already exists", Map.put(details, :message, message))} + end + + def classify_error(404, message, details) do + {:error, ErrorMessage.not_found("object or container not found", Map.put(details, :message, message))} + end + + def classify_error(status_code, message, details) do + {:error, + %ErrorMessage{ + code: ErrorMessage.http_code_reason_atom(status_code), + message: to_string(message), + details: details + }} + end + + defp delete_listed_objects(container, body, opts) do + keys = body |> Map.get(:contents, []) |> Enum.map(& &1.key) + next_token = Map.get(body, :next_continuation_token) + + with :ok <- delete_batch(container, keys, opts) do + if truncated?(body) and present?(next_token) do + delete_all_objects(container, opts, next_token) + else + :ok + end + end + end + + # ExAws returns is_truncated as the STRING "false", which is truthy in Elixir — branching on it + # directly recurses forever, and the empty next_continuation_token then makes S3 reject the + # request. Compare against known values instead of relying on truthiness. + defp present?(value), do: is_binary(value) and value !== "" + + defp delete_batch(_container, [], _opts), do: :ok + + defp delete_batch(container, keys, opts) do + container + |> S3.delete_multiple_objects(keys) + |> run(opts, %{container: container}) + |> discard_body() + end + + # `:request_fn` is the injection seam that makes pagination testable. Without it every + # page-boundary case needs a live account, which is why the paginators originally shipped with + # source-grep pins instead of tests — and a source grep cannot fail when the loop is removed. + # Same dependency-injection style as ProjectContext.check_valid_project/1. + defp run(request, opts, details) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + + case request_fn.(request, region: region(opts)) do + {:ok, _} = success -> success + {:error, {:http_error, status_code, %{body: body}}} -> classify_error(status_code, body, details) + {:error, {:http_error, status_code, message}} -> classify_error(status_code, message, details) + {:error, reason} -> {:error, ErrorMessage.failed_dependency(describe_reason(reason), details)} + end + end + + # Credential lookup and socket failures come back as a bare term with no status to classify; + # without this clause they raise a CaseClauseError instead of surfacing as an error tuple. + defp describe_reason(reason) when is_binary(reason), do: reason + defp describe_reason(reason), do: inspect(reason) + + defp region(opts), do: opts[:region] || DeployEx.Config.aws_region() + + defp unwrap_body({:ok, %{body: body}}), do: {:ok, body} + defp unwrap_body({:error, _} = error), do: error + + defp discard_body({:ok, _}), do: :ok + defp discard_body({:error, _} = error), do: error + +end diff --git a/lib/deploy_ex/cloud/security.ex b/lib/deploy_ex/cloud/security.ex new file mode 100644 index 00000000..c60a5353 --- /dev/null +++ b/lib/deploy_ex/cloud/security.ex @@ -0,0 +1,18 @@ +defmodule DeployEx.Cloud.Security do + @moduledoc """ + Ingress rules for an instance or group of instances. + + Named for the CONCEPT rather than any provider's object. AWS has security groups, OCI + has network security groups, GCP has no per-instance construct at all and expresses the + same intent as VPC firewall rules selected by network tag. The callbacks take a group + identifier and a CIDR so all three can implement them. + """ + + @callback find_group(keyword()) :: {:ok, String.t()} | {:error, ErrorMessage.t()} + + @callback authorize_ingress(String.t(), String.t(), keyword()) :: + :ok | {:error, ErrorMessage.t()} + + @callback revoke_ingress(String.t(), String.t(), keyword()) :: + :ok | {:error, ErrorMessage.t()} +end diff --git a/lib/deploy_ex/config.ex b/lib/deploy_ex/config.ex index dceda743..5e654b13 100644 --- a/lib/deploy_ex/config.ex +++ b/lib/deploy_ex/config.ex @@ -3,6 +3,25 @@ defmodule DeployEx.Config do def iac_tool, do: Application.get_env(@app, :iac_tool) || "terraform" + def cloud_provider, do: Application.get_env(@app, :cloud_provider, :aws) + + @doc """ + Reads one key from the `:oci` config namespace. + + Namespaced rather than flat because `DeployEx.Cloud.Providers.Oci`'s `config_schema/0` + validates that namespace strictly — a typo'd key fails at task start instead of mid-apply. + The AWS keys stay flat and permissively validated so existing configs keep working. + """ + @spec oci_setting(atom()) :: term() + def oci_setting(key), do: @app |> Application.get_env(:oci, []) |> Keyword.get(key) + + # Follows the key layout already in use in the state bucket (oracle///…), + # so deploy_ex state sits alongside states written by other tooling without colliding. + def oci_release_state_key do + oci_setting(:release_state_key) || + "oracle/#{oci_setting(:region)}/#{DeployExHelpers.kebab_project_name()}-#{env()}/terraform.tfstate" + end + @default_env to_string(Mix.env()) def env, do: Application.get_env(@app, :env) || @default_env def aws_region, do: Application.get_env(@app, :aws_region) || "us-west-2" diff --git a/lib/deploy_ex/k6_runner.ex b/lib/deploy_ex/k6_runner.ex index 81c83344..e09efa63 100644 --- a/lib/deploy_ex/k6_runner.ex +++ b/lib/deploy_ex/k6_runner.ex @@ -111,19 +111,56 @@ defmodule DeployEx.K6Runner do end end + @doc """ + Every K6Runner-tagged instance in the region, following pagination to completion. + + DescribeInstances caps a response and signals more via `nextToken`. A single request + therefore truncates silently on a large fleet — it returns `{:ok, partial}`, not an error — + which would make `load_test.destroy_instance` miss instances and leave them running (and + billing). + """ def find_runners_from_ec2(opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() resource_group = opts[:resource_group] || DeployEx.Config.aws_resource_group() - ExAws.EC2.describe_instances(filters: [ + find_runners_from_ec2_page(region, resource_group, opts, nil, []) + end + + defp find_runners_from_ec2_page(region, resource_group, opts, next_token, acc) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + + filters = [ "tag:K6Runner": ["true"], "tag:Group": [resource_group], "instance-state-name": ["running", "pending", "stopping", "stopped"] - ]) - |> ExAws.request(region: region) - |> handle_describe_instances() + ] + + describe_opts = if next_token, do: [filters: filters, next_token: next_token], else: [filters: filters] + + response = + describe_opts + |> ExAws.EC2.describe_instances() + |> request_fn.(region: region) + + with {:ok, runners} <- handle_describe_instances(response) do + accumulated = acc ++ runners + + case describe_instances_next_token(response) do + nil -> {:ok, accumulated} + token -> find_runners_from_ec2_page(region, resource_group, opts, token, accumulated) + end + end end + defp describe_instances_next_token({:ok, %{body: body}}) do + case XmlToMap.naive_map(body) do + %{"DescribeInstancesResponse" => %{"nextToken" => token}} when is_binary(token) and token !== "" -> token + _no_more_pages -> nil + end + end + + defp describe_instances_next_token(_response), do: nil + def verify_instance_exists(nil), do: {:ok, nil} def verify_instance_exists(%__MODULE__{instance_id: instance_id} = runner) do @@ -178,33 +215,72 @@ defmodule DeployEx.K6Runner do end end + @doc """ + Every stored runner state under `#{@state_prefix}/`, following pagination to completion. + + S3 caps a ListObjects response at 1000 keys and signals more via `is_truncated`. A single + request therefore truncates silently — `{:ok, partial}`, not an error — which would make + every caller here (list/exec/destroy_instance/upload/create_instance) see a partial runner + list. + """ def fetch_all_runners(opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() bucket = opts[:bucket] || DeployEx.Config.aws_release_bucket() + request_fn = opts[:request_fn] || (&ExAws.request/2) - bucket - |> ExAws.S3.list_objects(prefix: "#{@state_prefix}/") - |> ExAws.request(region: region) - |> case do - {:ok, %{body: %{contents: contents}}} when is_list(contents) -> - runners = Enum.map(contents, fn content -> - case ExAws.S3.get_object(bucket, content.key) |> ExAws.request(region: region) do - {:ok, %{body: body}} -> from_json(body) - _ -> nil - end - end) - |> Enum.reject(&is_nil/1) + with {:ok, contents} <- list_runner_state_objects(bucket, region, opts) do + runners = Enum.map(contents, fn content -> + case ExAws.S3.get_object(bucket, content.key) |> request_fn.(region: region) do + {:ok, %{body: body}} -> from_json(body) + _ -> nil + end + end) + |> Enum.reject(&is_nil/1) + + {:ok, runners} + end + end + + defp list_runner_state_objects(bucket, region, opts) do + list_runner_state_objects_page(bucket, region, opts, nil, []) + end + + defp list_runner_state_objects_page(bucket, region, opts, marker, acc) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + list_opts = if marker, do: [prefix: "#{@state_prefix}/", marker: marker], else: [prefix: "#{@state_prefix}/"] - {:ok, runners} + response = + bucket + |> ExAws.S3.list_objects(list_opts) + |> request_fn.(region: region) + + case response do + {:ok, %{body: %{contents: contents} = body}} when is_list(contents) -> + accumulated = acc ++ contents + + if truncated_listing?(body) and not Enum.empty?(contents) do + list_runner_state_objects_page(bucket, region, opts, next_listing_marker(body, contents), accumulated) + else + {:ok, accumulated} + end {:ok, _} -> - {:ok, []} + {:ok, acc} {:error, error} -> {:error, ErrorMessage.failed_dependency("failed to list k6 runner states", %{error: error})} end end + defp truncated_listing?(body), do: Map.get(body, :is_truncated) in [true, "true"] + + defp next_listing_marker(body, contents) do + case Map.get(body, :next_marker) do + marker when is_binary(marker) and marker !== "" -> marker + _absent -> contents |> List.last() |> Map.fetch!(:key) + end + end + def delete_state(%__MODULE__{instance_id: instance_id}, opts) do delete_state(instance_id, opts) end diff --git a/lib/deploy_ex/priv_renderer.ex b/lib/deploy_ex/priv_renderer.ex index 70a5f7e0..f2de524a 100644 --- a/lib/deploy_ex/priv_renderer.ex +++ b/lib/deploy_ex/priv_renderer.ex @@ -38,40 +38,92 @@ defmodule DeployEx.PrivRenderer do # SECTION: Terraform Rendering + # Exports only the ACTIVE provider's file set, the same selection terraform.build uses. A + # whole-directory copy put every provider's templates into the user's ./deploys — an :oci + # project got the AWS tree plus an unrendered providers/oci/*.eex, and an :aws project got + # providers/oci/*.tf sitting in its terraform root where tofu would try to load them. defp render_terraform(temp_dir, opts) do priv_terraform = priv_source_path("terraform") target_dir = Path.join(temp_dir, "terraform") + provider = active_provider(opts) - with :ok <- copy_directory(priv_terraform, target_dir), - :ok <- remove_eex_files(target_dir), - :ok <- render_terraform_templates(priv_terraform, target_dir, opts) do - :ok + with {:ok, files} <- DeployEx.Cloud.PrivFileSet.files(provider, priv_terraform) do + File.mkdir_p!(target_dir) + + copy_provider_files(priv_terraform, target_dir, files) + + render_provider_templates(priv_terraform, target_dir, files, build_terraform_params(opts, provider)) end end - defp render_terraform_templates(priv_terraform, target_dir, opts) do - params = build_terraform_params(opts) + defp copy_provider_files(priv_terraform, target_dir, files) do + files + |> Enum.reject(fn {source, _dest} -> String.ends_with?(source, ".eex") end) + |> Enum.each(fn {source, dest} -> + target = Path.join(target_dir, dest) - priv_terraform - |> Path.join("*.eex") - |> Path.wildcard() - |> Enum.each(fn template_file -> - rendered = EEx.eval_file(template_file, assigns: params) + target |> Path.dirname() |> File.mkdir_p!() + File.cp!(Path.join(priv_terraform, source), target) + end) + end + + # Provider-scoped templates flatten on the way out — providers/oci/variables.tf.eex becomes + # variables.tf at the terraform root — because tofu reads one directory, matching how + # terraform.build writes them. + defp render_provider_templates(priv_terraform, target_dir, files, params) do + files + |> Enum.filter(fn {source, _dest} -> String.ends_with?(source, ".eex") end) + |> Enum.each(fn {source, dest} -> + rendered = EEx.eval_file(Path.join(priv_terraform, source), assigns: params) + target = Path.join(target_dir, String.replace_suffix(dest, ".eex", "")) + + target |> Path.dirname() |> File.mkdir_p!() + File.write!(target, rendered) + end) - output_name = - template_file - |> Path.basename() - |> String.replace_suffix(".eex", "") + :ok + end - output_path = Path.join(target_dir, output_name) + defp active_provider(opts), do: opts[:provider] || DeployEx.Config.cloud_provider() - File.write!(output_path, rendered) - end) + # Falls back to the shared template when a provider ships no variant, so a provider that + # only overrides some files does not need copies of the rest. + defp provider_template(priv_dir, provider, relative) do + provider_path = Path.join([priv_dir, "providers", to_string(provider), relative]) - :ok + if File.exists?(provider_path) do + provider_path + else + Path.join(priv_dir, relative) + end + end + + # AWS's inventory is a dynamic plugin config, so exporting it renders fully. OCI's is a + # point-in-time snapshot generated from the live compartment, which export cannot know — it + # is written EMPTY here on purpose, and `mix ansible.build` fills it in. Emitting a + # plausible-looking inventory with invented hosts would be worse than an obviously empty one. + defp render_inventory_template(priv_ansible, target_dir, provider, app_name) do + case DeployEx.Cloud.inventory(provider) do + {:ok, %{strategy: :aws_ec2_plugin, template: template, filename: filename}} -> + render_template( + Path.join(priv_ansible, Path.basename(template)), + Path.join(target_dir, filename), + %{app_name: app_name} + ) + + {:ok, %{template: template, filename: filename}} -> + render_template( + Path.join(priv_ansible, String.replace_prefix(template, "ansible/", "")), + Path.join(target_dir, filename), + %{hosts_section: "{}", children_section: "{}"} + ) + + {:error, _no_inventory} -> + :ok + end end - defp build_terraform_params(opts) do + defp build_terraform_params(opts, provider) do release_names = fetch_release_names() app_name = opts[:app_name] || DeployExHelpers.underscored_project_name() kebab_app_name = opts[:kebab_app_name] || DeployExHelpers.kebab_project_name() @@ -81,7 +133,7 @@ defmodule DeployEx.PrivRenderer do aws_log_bucket = opts[:aws_log_bucket] || DeployEx.Config.aws_log_bucket() terraform_app_releases_variables = release_names - |> Enum.map_join(",\n\n", &generate_terraform_release_variables/1) + |> Enum.map_join(",\n\n", &DeployEx.TerraformVariables.generate_terraform_release_variables(&1, provider)) random_bytes = 6 |> :crypto.strong_rand_bytes() |> Base.encode32(padding: false) @@ -102,7 +154,14 @@ defmodule DeployEx.PrivRenderer do terraform_backend: DeployEx.Config.terraform_backend(), - pem_app_name: "#{kebab_app_name}-#{random_bytes}", + oci_region: DeployEx.Config.oci_setting(:region), + oci_namespace: DeployEx.Config.oci_setting(:namespace), + oci_state_bucket: DeployEx.Config.oci_setting(:release_state_bucket), + oci_state_key: DeployEx.Config.oci_release_state_key(), + oci_state_profile: DeployEx.Config.oci_setting(:state_profile), + oci_vcn_cidr: DeployEx.Config.oci_setting(:vcn_cidr) || "10.20.0.0/16", + + pem_app_name: opts[:pem_app_name] || "#{kebab_app_name}-#{random_bytes}", app_name: app_name, kebab_app_name: kebab_app_name, @@ -115,16 +174,21 @@ defmodule DeployEx.PrivRenderer do terraform_app_releases_variables: terraform_app_releases_variables, terraform_release_variables: terraform_app_releases_variables, - terraform_redis_variables: terraform_redis_variables(opts), - terraform_sentry_variables: terraform_sentry_variables(opts), - terraform_grafana_variables: terraform_grafana_variables(opts), - terraform_loki_variables: terraform_loki_variables(opts), - terraform_prometheus_variables: terraform_prometheus_variables(opts) + terraform_redis_variables: DeployEx.TerraformVariables.terraform_redis_variables(opts, provider), + terraform_clickhouse_variables: DeployEx.TerraformVariables.terraform_clickhouse_variables(opts, provider), + terraform_rabbitmq_variables: DeployEx.TerraformVariables.terraform_rabbitmq_variables(opts, provider), + terraform_sentry_variables: DeployEx.TerraformVariables.terraform_sentry_variables(opts, provider), + terraform_grafana_variables: DeployEx.TerraformVariables.terraform_grafana_variables(opts, provider), + terraform_loki_variables: DeployEx.TerraformVariables.terraform_loki_variables(opts, provider), + terraform_prometheus_variables: DeployEx.TerraformVariables.terraform_prometheus_variables(opts, provider) } end # SECTION: Ansible Rendering + # The roles/setup/playbook templates are provider-neutral and ship to everyone, but the + # provider-EXCLUSIVE subtree is stripped back out: leaving it would put an oci user's + # ansible.cfg into an aws export and vice versa, and both are rendered fresh below. defp render_ansible(temp_dir, opts) do priv_ansible = priv_source_path("ansible") target_dir = Path.join(temp_dir, "ansible") @@ -132,32 +196,29 @@ defmodule DeployEx.PrivRenderer do with :ok <- copy_directory(priv_ansible, target_dir), :ok <- remove_eex_files_recursive(target_dir), :ok <- render_ansible_templates(priv_ansible, target_dir, opts) do + File.rm_rf!(Path.join(target_dir, "providers")) + :ok end end defp render_ansible_templates(priv_ansible, target_dir, opts) do app_name = opts[:app_name] || DeployExHelpers.underscored_project_name() + provider = active_provider(opts) - # ansible.cfg + # ansible.cfg — the provider variant when one exists, since the OCI config sets a different + # remote_user and points at a static inventory rather than the aws_ec2 plugin. ansible_cfg_vars = %{ pem_file_path: "../terraform/#{String.replace(app_name, "_", "-")}*pem" } render_template( - Path.join(priv_ansible, "ansible.cfg.eex"), + provider_template(priv_ansible, provider, "ansible.cfg.eex"), Path.join(target_dir, "ansible.cfg"), ansible_cfg_vars ) - # aws_ec2.yaml - hosts_vars = %{app_name: app_name} - - render_template( - Path.join(priv_ansible, "aws_ec2.yaml.eex"), - Path.join(target_dir, "aws_ec2.yaml"), - hosts_vars - ) + render_inventory_template(priv_ansible, target_dir, provider, app_name) # group_vars/all.yaml group_vars_vars = %{ @@ -182,9 +243,13 @@ defmodule DeployEx.PrivRenderer do File.mkdir_p!(Path.join(target_dir, "setup")) Enum.each(release_names, fn release_name -> + # cloud_provider is load-bearing in app_setup_playbook.yaml.eex — it selects awscli vs + # oci_cli and gates the AWS-only save_ami role. Omitting it raised + # "assign @cloud_provider not available" mid-render. playbook_vars = %{ no_logging: Keyword.get(opts, :no_logging, false), no_prometheus: Keyword.get(opts, :no_prometheus, false), + cloud_provider: provider, app_name: release_name, port: 80 } @@ -211,141 +276,6 @@ defmodule DeployEx.PrivRenderer do # SECTION: Terraform Variable Generators - defp generate_terraform_release_variables(release_name) do - String.trim_trailing(""" - #{release_name} = { - name = "#{DeployEx.Utils.upper_title_case(release_name)}" - tags = { - Vendor = "Self" - Type = "Self Made" - } - - # Autoscaling Configuration (optional) - # Uncomment and configure to enable AWS Auto Scaling Groups - # autoscaling = { - # enable = true - # min_size = 1 - # max_size = 5 - # desired_capacity = 2 - # cpu_target_percent = 60 - # } - } - """, "\n") - end - - defp terraform_redis_variables(opts) do - if Keyword.get(opts, :no_redis, false) do - "" - else - app_name = opts[:app_name] || DeployExHelpers.underscored_project_name() - title = DeployExHelpers.title_case_project_name() - - """ - #{app_name}_redis = { - name = "#{title} Redis" - private_ip = "10.0.1.60" - enable_ebs = true - - # This is a suggestion for instance - - instance_type = "r7g.medium" - - instance_ebs_secondary_size = 16 - - tags = { - Vendor = "Redis" - Type = "Database" - DatabaseKey = "#{app_name}_redis" - } - }, - """ - end - end - - defp terraform_sentry_variables(opts) do - if Keyword.get(opts, :no_sentry, false) do - "" - else - """ - sentry = { - name = "Sentry Monitoring" - tags = { - Vendor = "Sentry" - Type = "Monitoring" - } - }, - """ - end - end - - defp terraform_loki_variables(opts) do - if Keyword.get(opts, :no_logging, false) do - "" - else - """ - loki_aggregator = { - name = "Grafana Loki Logs" - instance_type = "t3.micro" - private_ip = "10.0.1.50" - - enable_ebs = true - instance_ebs_secondary_size = 8 - - tags = { - Vendor = "Grafana" - Type = "Monitoring" - MonitoringKey = "loki_logger" - } - }, - """ - end - end - - defp terraform_grafana_variables(opts) do - if Keyword.get(opts, :no_grafana, false) do - "" - else - """ - grafana_ui = { - name = "Grafana UI" - enable_ebs = true - enable_eip = true - instance_ebs_secondary_size = 8 - - tags = { - Vendor = "Grafana" - Type = "Monitoring" - MonitoringKey = "grafana_ui" - } - }, - """ - end - end - - defp terraform_prometheus_variables(opts) do - if Keyword.get(opts, :no_prometheus, false) do - "" - else - """ - prometheus_db = { - name = "Prometheus Metrics Database" - instance_type = "t3.micro" - enable_ebs = true - instance_ebs_secondary_size = 16 - private_ip = "10.0.1.40" - - tags = { - Vendor = "Grafana" - Type = "Monitoring" - MonitoringKey = "prometheus_db" - } - }, - """ - end - end - - # SECTION: Helpers - defp priv_source_path(subdirectory) do :deploy_ex |> :code.priv_dir() |> Path.join(subdirectory) end @@ -363,15 +293,6 @@ defmodule DeployEx.PrivRenderer do :ok end - defp remove_eex_files(directory) do - directory - |> Path.join("*.eex") - |> Path.wildcard() - |> Enum.each(&File.rm!/1) - - :ok - end - defp remove_eex_files_recursive(directory) do directory |> Path.join("**/*.eex") diff --git a/lib/deploy_ex/qa_node.ex b/lib/deploy_ex/qa_node.ex index 483efc46..b47afbdb 100644 --- a/lib/deploy_ex/qa_node.ex +++ b/lib/deploy_ex/qa_node.ex @@ -773,18 +773,22 @@ defmodule DeployEx.QaNode do end end + # ExAws.S3.list_objects caps a page at 1000 keys and signals more via is_truncated, so a single + # request would silently drop app-scoped QA state past the first page. Delegates pagination to + # DeployEx.Cloud.S3ObjectStore.list_objects/2, which already follows the marker to completion. def fetch_all_qa_states_for_app(app_name, opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() bucket = opts[:bucket] || DeployEx.Config.aws_release_bucket() prefix = "#{DeployEx.Config.qa_state_prefix()}/#{app_name}/" + request_fn = opts[:request_fn] || (&ExAws.request/2) - bucket - |> ExAws.S3.list_objects(prefix: prefix) - |> ExAws.request(region: region) - |> case do - {:ok, %{body: %{contents: contents}}} when is_list(contents) -> - states = Enum.map(contents, fn content -> - case ExAws.S3.get_object(bucket, content.key) |> ExAws.request(region: region) do + list_opts = opts |> Keyword.take([:request_fn]) |> Keyword.merge(prefix: prefix, region: region) + + case DeployEx.Cloud.S3ObjectStore.list_objects(bucket, list_opts) do + {:ok, keys} -> + states = keys + |> Enum.map(fn key -> + case ExAws.S3.get_object(bucket, key) |> request_fn.(region: region) do {:ok, %{body: body}} -> from_json(body) _ -> nil end @@ -793,11 +797,8 @@ defmodule DeployEx.QaNode do {:ok, states} - {:ok, _} -> - {:ok, []} - - {:error, error} -> - {:error, ErrorMessage.failed_dependency("failed to list qa states", %{error: error})} + {:error, _} = error -> + error end end @@ -821,15 +822,15 @@ defmodule DeployEx.QaNode do environment = opts[:environment] || DeployEx.Config.env() resource_group = opts[:resource_group] || DeployEx.Config.aws_resource_group() - ExAws.EC2.describe_instances(filters: [ + filters = [ "tag:QaNode": ["true"], "tag:Group": [resource_group], "tag:Environment": [environment], "tag:GitBranch": [branch], "instance-state-name": ["running", "pending", "stopping", "stopped"] - ]) - |> ExAws.request(region: region) - |> handle_describe_instances_for_qa_list() + ] + + fetch_qa_instances_from_ec2(filters, region, opts) end @doc """ @@ -867,14 +868,18 @@ defmodule DeployEx.QaNode do environment = opts[:environment] || DeployEx.Config.env() resource_group = opts[:resource_group] || DeployEx.Config.aws_resource_group() - ExAws.EC2.describe_instances(filters: [ + filters = [ "tag:QaNode": ["true"], "tag:Group": [resource_group], "tag:InstanceGroup": ["#{app_name}_#{environment}"], "instance-state-name": ["running", "pending", "stopping", "stopped"] - ]) - |> ExAws.request(region: region) - |> handle_describe_instances_for_qa() + ] + + case fetch_qa_instances_from_ec2(filters, region, opts) do + {:ok, [instance | _]} -> {:ok, instance} + {:ok, []} -> {:ok, nil} + {:error, _} = error -> error + end end def find_qa_nodes_from_ec2(app_name, opts \\ []) do @@ -882,66 +887,30 @@ defmodule DeployEx.QaNode do environment = opts[:environment] || DeployEx.Config.env() resource_group = opts[:resource_group] || DeployEx.Config.aws_resource_group() - ExAws.EC2.describe_instances(filters: [ + filters = [ "tag:QaNode": ["true"], "tag:Group": [resource_group], "tag:InstanceGroup": ["#{app_name}_#{environment}"], "instance-state-name": ["running", "pending", "stopping", "stopped"] - ]) - |> ExAws.request(region: region) - |> handle_describe_instances_for_qa_list() - end - - defp handle_describe_instances_for_qa({:ok, %{body: body}}) do - case XmlToMap.naive_map(body) do - %{"DescribeInstancesResponse" => %{"reservationSet" => %{"item" => reservations}}} -> - instances = extract_qa_instances(reservations) - case instances do - [instance | _] -> {:ok, instance} - [] -> {:ok, nil} - end - - %{"DescribeInstancesResponse" => %{"reservationSet" => nil}} -> - {:ok, nil} + ] - _ -> - {:ok, nil} - end + fetch_qa_instances_from_ec2(filters, region, opts) end - defp handle_describe_instances_for_qa_list({:ok, %{body: body}}) do - case XmlToMap.naive_map(body) do - %{"DescribeInstancesResponse" => %{"reservationSet" => %{"item" => reservations}}} -> - {:ok, extract_qa_instances(reservations)} - - %{"DescribeInstancesResponse" => %{"reservationSet" => nil}} -> - {:ok, []} + # DescribeInstances caps a page and signals more via nextToken, so a single request would + # silently miss QA nodes past the first page — e.g. a second node sharing a branch could stay + # invisible to select_by_branch/2's "multiple ->" conflict guard. Delegates pagination to + # DeployEx.AwsMachine.fetch_instances/2, which already follows nextToken to completion; the + # filters keep the request scoped server-side exactly as before. + defp fetch_qa_instances_from_ec2(filters, region, opts) do + ec2_opts = opts |> Keyword.take([:request_fn]) |> Keyword.merge(filters: filters) - _ -> - {:ok, []} + case DeployEx.AwsMachine.fetch_instances(region, ec2_opts) do + {:ok, instances} -> {:ok, Enum.map(instances, &build_qa_node_from_instance/1)} + {:error, _} = error -> error end end - defp handle_describe_instances_for_qa_list({:error, {:http_error, status, %{body: body}}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status), ["error fetching QA instances", %{body: body}])} - end - - defp extract_qa_instances(reservations) when is_list(reservations) do - Enum.flat_map(reservations, fn reservation -> - case reservation["instancesSet"]["item"] do - items when is_list(items) -> Enum.map(items, &build_qa_node_from_instance/1) - item when is_map(item) -> [build_qa_node_from_instance(item)] - _ -> [] - end - end) - end - - defp extract_qa_instances(reservation) when is_map(reservation) do - extract_qa_instances([reservation]) - end - - defp extract_qa_instances(_), do: [] - def build_qa_node_from_instance(instance) do tags = parse_instance_tags(instance["tagSet"]) @@ -998,27 +967,27 @@ defmodule DeployEx.QaNode do |> handle_delete_response() end + # Same truncation risk as fetch_all_qa_states_for_app/2 (S3 caps a page at 1000 keys): qa.cleanup + # and qa.destroy --all iterate every app_name returned here, so a truncated page here means they + # silently skip instances past key 1000 and report success. def list_all_qa_states(opts \\ []) do region = opts[:region] || DeployEx.Config.aws_region() bucket = opts[:bucket] || DeployEx.Config.aws_release_bucket() + prefix = DeployEx.Config.qa_state_prefix() - bucket - |> ExAws.S3.list_objects(prefix: DeployEx.Config.qa_state_prefix()) - |> ExAws.request(region: region) - |> case do - {:ok, %{body: %{contents: contents}}} when is_list(contents) -> - app_names = contents - |> Enum.map(&extract_app_name_from_key(&1.key)) + list_opts = opts |> Keyword.take([:request_fn]) |> Keyword.merge(prefix: prefix, region: region) + + case DeployEx.Cloud.S3ObjectStore.list_objects(bucket, list_opts) do + {:ok, keys} -> + app_names = keys + |> Enum.map(&extract_app_name_from_key/1) |> Enum.reject(&is_nil/1) |> Enum.uniq() {:ok, app_names} - {:ok, %{body: %{contents: _}}} -> - {:ok, []} - - {:error, error} -> - {:error, ErrorMessage.failed_dependency("failed to list qa states", %{error: error})} + {:error, _} = error -> + error end end diff --git a/lib/deploy_ex/release_tracker.ex b/lib/deploy_ex/release_tracker.ex index 73456c29..d2455fa2 100644 --- a/lib/deploy_ex/release_tracker.ex +++ b/lib/deploy_ex/release_tracker.ex @@ -1,4 +1,14 @@ defmodule DeployEx.ReleaseTracker do + @moduledoc """ + Tracks which release is current for an app, and the history of what came before it. + + Both live at fixed keys in the release container, read and written through the + provider-neutral `DeployEx.Cloud.S3ObjectStore`. Nothing here lists the container, so there + is no pagination surface to truncate. + """ + + alias DeployEx.Cloud.S3ObjectStore + @release_state_prefix "release-state" def current_release_key(app_name, opts \\ []) do @@ -10,41 +20,26 @@ defmodule DeployEx.ReleaseTracker do end def fetch_current_release(app_name, opts \\ []) do - region = opts[:region] || DeployEx.Config.aws_region() - bucket = opts[:bucket] || DeployEx.Config.aws_release_bucket() - - bucket - |> ExAws.S3.get_object(current_release_key(app_name, opts)) - |> ExAws.request(region: region) - |> handle_get_response() + app_name + |> current_release_key(opts) + |> fetch_release_state(opts) end def fetch_release_history(app_name, opts \\ []) do - region = opts[:region] || DeployEx.Config.aws_region() - bucket = opts[:bucket] || DeployEx.Config.aws_release_bucket() - - bucket - |> ExAws.S3.get_object(release_history_key(app_name, opts)) - |> ExAws.request(region: region) - |> handle_get_response() + app_name + |> release_history_key(opts) + |> fetch_release_state(opts) end def set_current_release(app_name, release_name, opts \\ []) do - region = opts[:region] || DeployEx.Config.aws_region() - bucket = opts[:bucket] || DeployEx.Config.aws_release_bucket() - with {:ok, _} <- append_to_release_history(app_name, release_name, opts) do - bucket - |> ExAws.S3.put_object(current_release_key(app_name, opts), "#{release_name}\n") - |> ExAws.request(region: region) - |> handle_put_response() + app_name + |> current_release_key(opts) + |> put_release_state("#{release_name}\n", opts) end end def append_to_release_history(app_name, release_name, opts \\ []) do - region = opts[:region] || DeployEx.Config.aws_region() - bucket = opts[:bucket] || DeployEx.Config.aws_release_bucket() - existing_history = case fetch_release_history(app_name, opts) do {:ok, history} -> history {:error, _} -> "" @@ -53,10 +48,9 @@ defmodule DeployEx.ReleaseTracker do new_history = "#{String.trim(existing_history)}\n#{release_name}\n" |> String.trim_leading("\n") - bucket - |> ExAws.S3.put_object(release_history_key(app_name, opts), new_history) - |> ExAws.request(region: region) - |> handle_put_response() + app_name + |> release_history_key(opts) + |> put_release_state(new_history, opts) end def list_release_history(app_name, limit \\ 25, opts \\ []) do @@ -70,30 +64,45 @@ defmodule DeployEx.ReleaseTracker do end end - defp handle_get_response({:ok, %{body: body}}), do: {:ok, String.trim(body)} - - defp handle_get_response({:error, {:http_error, 404, _}}) do - {:error, ErrorMessage.not_found("release state not found")} + defp fetch_release_state(key, opts) do + case S3ObjectStore.get_object(bucket(opts), key, region: region(opts)) do + {:ok, body} -> {:ok, String.trim(body)} + {:error, error} -> {:error, translate_error(error)} + end end - defp handle_get_response({:error, {:http_error, status, reason}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status), ["aws failure", %{reason: reason}])} + defp put_release_state(key, body, opts) do + case S3ObjectStore.put_object(bucket(opts), key, body, region: region(opts)) do + :ok -> {:ok, :uploaded} + {:error, error} -> {:error, translate_error(error)} + end end - defp handle_get_response({:error, error}) when is_binary(error) do - {:error, ErrorMessage.failed_dependency("aws failure: #{error}")} + defp translate_error(%ErrorMessage{code: :not_found}) do + ErrorMessage.not_found("release state not found") end - defp handle_put_response({:ok, _}), do: {:ok, :uploaded} - - defp handle_put_response({:error, {:http_error, status, reason}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status), ["aws failure", %{reason: reason}])} + # The cause stays in the message, not only in details: ErrorMessage.to_string/1 renders + # "code - message" and drops details, and ansible.deploy raises with exactly that string — + # so an operator would otherwise see a bare "aws failure" with no reason. + defp translate_error(%ErrorMessage{code: code, message: message, details: details}) do + %ErrorMessage{ + code: code, + message: "aws failure: #{message}", + details: reason_details(details, message) + } end - defp handle_put_response({:error, error}) when is_binary(error) do - {:error, ErrorMessage.failed_dependency("aws failure: #{error}")} + defp reason_details(details, message) when is_map(details) do + details |> Map.delete(:message) |> Map.put(:reason, message) end + defp reason_details(_details, message), do: %{reason: message} + + defp bucket(opts), do: opts[:bucket] || DeployEx.Config.aws_release_bucket() + + defp region(opts), do: opts[:region] || DeployEx.Config.aws_region() + defp release_state_prefix(opts) when is_map(opts) do release_state_prefix(Map.to_list(opts)) end diff --git a/lib/deploy_ex/release_uploader.ex b/lib/deploy_ex/release_uploader.ex index 8f7e9013..efc9a809 100644 --- a/lib/deploy_ex/release_uploader.ex +++ b/lib/deploy_ex/release_uploader.ex @@ -1,4 +1,6 @@ defmodule DeployEx.ReleaseUploader do + require Logger + alias DeployEx.ReleaseUploader.{State, AwsManager, UpdateValidator} @type opts :: [ @@ -150,6 +152,7 @@ defmodule DeployEx.ReleaseUploader do |> case do :ok -> :ok {:ok, _} -> :ok + {:error, %ErrorMessage{code: :not_implemented}} -> skip_unsupported_tagging(remote_file_path) {:error, _} = error -> error end @@ -158,6 +161,25 @@ defmodule DeployEx.ReleaseUploader do end end + # Object tagging is OPTIONAL. OCI Object Storage has none — its nearest equivalent is + # user metadata, settable only at put time — so a provider without it must not fail the + # upload. MEASURED: every release on a qa/ branch failed with "oci object storage has no + # object tagging" AFTER the object had already uploaded successfully. + # + # Nothing is lost. This tag is write-only inside deploy_ex — @qa_tag_key appears at its + # definition and at the write above, and nowhere else — while the thing that actually marks + # a QA release is the `qa/` key prefix, which drives remote-release lookup and is provider + # independent. The tag exists for external tooling (lifecycle rules and the like), so its + # absence is worth saying out loud rather than swallowing silently. + defp skip_unsupported_tagging(remote_file_path) do + Logger.info( + "#{__MODULE__}: provider has no object tagging, relying on the key prefix to mark the " <> + "QA release, path: #{inspect(remote_file_path)}" + ) + + :ok + end + defp release_prefix(opts) when is_map(opts) do release_prefix(Map.to_list(opts)) end diff --git a/lib/deploy_ex/release_uploader/aws_manager.ex b/lib/deploy_ex/release_uploader/aws_manager.ex index 648660c3..fa9a6855 100644 --- a/lib/deploy_ex/release_uploader/aws_manager.ex +++ b/lib/deploy_ex/release_uploader/aws_manager.ex @@ -1,72 +1,57 @@ defmodule DeployEx.ReleaseUploader.AwsManager do - def get_releases(region, bucket, prefix \\ nil) do - s3_opts = if prefix, do: [prefix: prefix], else: [] + @moduledoc """ + Release-bucket operations, resolved through the active provider's object store. + + Every function here was 100% S3 plumbing with no release-specific logic — `get_releases/3`, + `upload/4` and `tag_object/4` map exactly onto `list_objects/2`, `upload_file/4` and + `put_object_tags/4` on `DeployEx.Cloud.ObjectStore`. A per-provider release manager would + have been a second abstraction over the same three operations, so the provider seam lives in + the object store and this module only adapts the argument order. + + The name is historical: its callers pass region first, which the provider-neutral behaviour + does not, and renaming it would touch call sites unrelated to making a second provider work. + """ - fetch_paginated_keys(region, bucket, s3_opts, []) + alias DeployEx.Cloud + + @spec get_releases(String.t() | nil, String.t(), String.t() | nil) :: + {:ok, [String.t()]} | {:error, ErrorMessage.t()} + def get_releases(region, bucket, prefix \\ nil) do + with {:ok, store} <- Cloud.capability(:object_store) do + store.list_objects(bucket, store_opts(region, prefix: prefix)) + end rescue e -> {:error, - ErrorMessage.failed_dependency("failed to list S3 releases", %{ + ErrorMessage.failed_dependency("failed to list releases", %{ exception: inspect(e.__struct__), error: Exception.message(e), stacktrace: Exception.format_stacktrace(__STACKTRACE__) })} end - defp fetch_paginated_keys(region, bucket, s3_opts, acc) do - bucket - |> ExAws.S3.list_objects(s3_opts) - |> ExAws.request(region: region) - |> handle_list_response(region, bucket, s3_opts, acc) - end - - defp handle_list_response( - {:ok, %{body: %{contents: contents, is_truncated: "true"} = body}}, - region, - bucket, - s3_opts, - acc - ) do - keys = Enum.map(contents, & &1.key) - marker = next_marker(body, contents) - - fetch_paginated_keys(region, bucket, Keyword.put(s3_opts, :marker, marker), acc ++ keys) - end - - defp handle_list_response({:ok, %{body: %{contents: contents}}}, _region, _bucket, _s3_opts, acc) do - {:ok, acc ++ Enum.map(contents, & &1.key)} - end - - defp handle_list_response({:error, reason}, _region, _bucket, _s3_opts, _acc) do - {:error, ErrorMessage.failed_dependency("failed to list S3 releases", %{error: inspect(reason)})} - end - - defp next_marker(%{next_marker: marker}, _contents) when is_binary(marker) and marker !== "", do: marker - defp next_marker(_body, contents), do: contents |> List.last() |> Map.fetch!(:key) - + @spec upload(Path.t(), String.t() | nil, String.t(), String.t()) :: + :ok | {:error, ErrorMessage.t()} def upload(file_path, region, bucket, upload_path) do - file_path - |> ExAws.S3.Upload.stream_file - |> ExAws.S3.upload(bucket, upload_path) - |> ExAws.request(region: region) - |> handle_response + with {:ok, store} <- Cloud.capability(:object_store) do + store.upload_file(bucket, upload_path, file_path, store_opts(region)) + end end + @spec tag_object(String.t() | nil, String.t(), String.t(), map()) :: + :ok | {:error, ErrorMessage.t()} def tag_object(region, bucket, object_key, tags) do - bucket - |> ExAws.S3.put_object_tagging(object_key, tags) - |> ExAws.request(region: region) - |> handle_response - end - - defp handle_response({:ok, %{body: body}}), do: {:ok, body} - defp handle_response({:ok, :done}), do: :ok - - defp handle_response({:error, {:http_error, status, reason}}) do - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status), ["aws failure", %{reason: reason}])} + with {:ok, store} <- Cloud.capability(:object_store) do + store.put_object_tags(bucket, object_key, tags, store_opts(region)) end + end - defp handle_response({:error, error}) when is_binary(error) do - {:error, ErrorMessage.failed_dependency("aws failure: #{error}")} + defp store_opts(region, extra \\ []) do + extra + |> Enum.reject(fn {_key, value} -> is_nil(value) end) + |> Keyword.merge(region_opts(region)) end + + defp region_opts(nil), do: [] + defp region_opts(region), do: [region: region] end diff --git a/lib/deploy_ex/terraform_state.ex b/lib/deploy_ex/terraform_state.ex index 5ba658df..6cf8e19a 100644 --- a/lib/deploy_ex/terraform_state.ex +++ b/lib/deploy_ex/terraform_state.ex @@ -4,6 +4,8 @@ defmodule DeployEx.TerraformState do Supports both local state files and S3 backend. """ + alias DeployEx.Cloud.S3ObjectStore + @terraform_state_filename "terraform.tfstate" @terraform_state_key "terraform.tfstate" @@ -41,14 +43,14 @@ defmodule DeployEx.TerraformState do key = opts[:key] || @terraform_state_key region = opts[:region] || DeployEx.Config.aws_region() - case ExAws.S3.get_object(bucket, key) |> ExAws.request(region: region) do - {:ok, %{body: body}} -> + case S3ObjectStore.get_object(bucket, key, region: region) do + {:ok, body} -> Jason.decode(body) - {:error, {:http_error, 404, _}} -> + {:error, %ErrorMessage{code: :not_found}} -> {:error, "Terraform state not found in S3: s3://#{bucket}/#{key}"} - {:error, {:http_error, 403, _}} -> + {:error, %ErrorMessage{code: :forbidden}} -> {:error, "Access denied to S3 bucket: #{bucket}"} {:error, error} -> diff --git a/lib/deploy_ex/terraform_variables.ex b/lib/deploy_ex/terraform_variables.ex new file mode 100644 index 00000000..74d97d9f --- /dev/null +++ b/lib/deploy_ex/terraform_variables.ex @@ -0,0 +1,351 @@ +defmodule DeployEx.TerraformVariables do + @moduledoc """ + Generates the default `_project` map and the support-node entries that go into a + rendered variables.tf. + + Extracted because `Mix.Tasks.Terraform.Build` and `DeployEx.PrivRenderer` each carried + their own copy, and they drifted: build was made provider-aware while the renderer kept + emitting AWS-only keys. `mix deploy_ex.export_priv` on an OCI project therefore produced a + variables.tf whose sizing fields the OCI instance module never reads — values that look + authoritative and do nothing. One copy now, both callers. + """ + + def generate_terraform_release_variables(release_name, :oci) do + String.trim_trailing(""" + #{release_name} = { + name = "#{DeployEx.Utils.upper_title_case(release_name)}" + tags = { + Vendor = "Self" + Type = "Self Made" + } + + # Sizing is optional — unset keys fall back to the instance_shape / instance_ocpus / + # instance_memory_gbs variables at the top of this file. + # shape = "VM.Standard.E5.Flex" + # ocpus = 2 + # memory_gbs = 16 + # boot_volume_size_gbs = 100 + # instance_count = 2 + + # Load balancer is optional — uncomment to front this app with an OCI Network Load + # Balancer. Unlike AWS, OCI gates creation on `enable` alone (no instance_count / + # autoscaling gate — see providers/oci/README.md). There is no `port` / `instance_port` + # here: the listener is unconditionally 80 (and 443 when enable_https), matching what + # AWS already does under the hood. + # load_balancer = { + # enable = true + # enable_https = false + # reserved_ip_ocid = null + # + # health_check = { + # path = "/health" + # return_code = 200 + # https_return_code = 200 + # unhealthy_threshold = 3 + # timeout = 3 + # interval = 10 + # } + # } + } + """, "\n") + end + + def generate_terraform_release_variables(release_name, _provider) do + String.trim_trailing(""" + #{release_name} = { + name = "#{DeployEx.Utils.upper_title_case(release_name)}" + tags = { + Vendor = "Self" + Type = "Self Made" + } + + # Autoscaling Configuration (optional) + # Uncomment and configure to enable AWS Auto Scaling Groups + # autoscaling = { + # enable = true + # min_size = 1 + # max_size = 5 + # desired_capacity = 2 + # cpu_target_percent = 60 + # } + } + """, "\n") + end + + # Support-node defaults are written per provider rather than shared, because the two + # instance modules read disjoint key sets: AWS takes instance_type/ebs/eip, OCI takes + # shape/ocpus/memory_gbs/boot_volume_size_gbs. Emitting the AWS keys into an OCI tree + # produced a variables.tf whose values were silently ignored — `instance_type = "t3.micro"` + # sat there looking authoritative while the module read `shape` and never saw it. The AWS + # clauses below are byte-for-byte what they always were; the render is pinned to that. + # + # NOTE: the OCI variants drop `private_ip`. The oci-instance module does not take one, and + # the fixed 10.0.1.x addresses the monitoring roles point at (grafana_loki_url, + # grafana_prometheus_url in group_vars) therefore do not resolve on OCI. Monitoring on OCI + # needs its own address plan — see the OCI monitoring gap, still open. + def terraform_redis_variables(opts, :oci) do + if opts[:no_redis] do + "" + else + """ + #{DeployExHelpers.underscored_project_name()}_redis = { + name = "#{DeployExHelpers.title_case_project_name()} Redis" + + shape = "VM.Standard.E5.Flex" + ocpus = 2 + memory_gbs = 8 + + boot_volume_size_gbs = 64 + + tags = { + Vendor = "Redis" + Type = "Database" + DatabaseKey = "#{DeployExHelpers.underscored_project_name()}_redis" + } + }, + """ + end + end + + def terraform_redis_variables(opts, _provider) do + if opts[:no_redis] do + "" + else + """ + #{DeployExHelpers.underscored_project_name()}_redis = { + name = "#{DeployExHelpers.title_case_project_name()} Redis" + private_ip = "10.0.1.60" + enable_ebs = true + + # This is a suggestion for instance + + instance_type = "r7g.medium" + + instance_ebs_secondary_size = 16 + + tags = { + Vendor = "Redis" + Type = "Database" + DatabaseKey = "#{DeployExHelpers.underscored_project_name()}_redis" + } + }, + """ + end + end + + # Opt-in (--clickhouse), unlike redis: ClickHouse is not part of the default stack on either + # provider — an AWS user hand-writes this entry — so an OCI render must not grow a node + # nobody asked for. The DatabaseKey tag is what routes it into the database_*_clickhouse + # inventory group the setup playbook targets. + def terraform_clickhouse_variables(opts, :oci) do + if opts[:clickhouse] do + """ + #{DeployExHelpers.underscored_project_name()}_clickhouse = { + name = "#{DeployExHelpers.title_case_project_name()} Clickhouse" + + shape = "VM.Standard.E6.Flex" + ocpus = 2 + memory_gbs = 16 + + boot_volume_size_gbs = 200 + + tags = { + Vendor = "ClickHouse" + Type = "Database" + DatabaseKey = "#{DeployExHelpers.underscored_project_name()}_clickhouse" + } + }, + """ + else + "" + end + end + + def terraform_clickhouse_variables(_opts, _provider), do: "" + + # Opt-in (--rabbitmq), same reasoning as clickhouse. Single node: the app declares quorum + # queues, which are correct on one node (Raft majority of 1); more nodes buy HA and a + # 2-node layout is the one to avoid. The DatabaseKey tag routes it into the + # database_*_rabbitmq inventory group the setup playbook targets. + def terraform_rabbitmq_variables(opts, :oci) do + if opts[:rabbitmq] do + """ + #{DeployExHelpers.underscored_project_name()}_rabbitmq = { + name = "#{DeployExHelpers.title_case_project_name()} Rabbitmq" + + shape = "VM.Standard.E6.Flex" + ocpus = 2 + memory_gbs = 16 + + boot_volume_size_gbs = 100 + + tags = { + Vendor = "RabbitMQ" + Type = "Database" + DatabaseKey = "#{DeployExHelpers.underscored_project_name()}_rabbitmq" + } + }, + """ + else + "" + end + end + + def terraform_rabbitmq_variables(_opts, _provider), do: "" + + # Sentry carries no sizing keys on either provider, so one clause serves both. + def terraform_sentry_variables(opts, _provider) do + if opts[:no_sentry] do + "" + else + """ + sentry = { + name = "Sentry Monitoring" + tags = { + Vendor = "Sentry" + Type = "Monitoring" + } + }, + """ + end + end + + def terraform_loki_variables(opts, :oci) do + if opts[:no_logging] do + "" + else + """ + loki_aggregator = { + name = "Grafana Loki Logs" + + shape = "VM.Standard.E5.Flex" + ocpus = 1 + memory_gbs = 4 + + boot_volume_size_gbs = 64 + + tags = { + Vendor = "Grafana" + Type = "Monitoring" + MonitoringKey = "loki_logger" + } + }, + """ + end + end + + def terraform_loki_variables(opts, _provider) do + if opts[:no_logging] do + "" + else + """ + loki_aggregator = { + name = "Grafana Loki Logs" + instance_type = "t3.micro" + private_ip = "10.0.1.50" + + enable_ebs = true + instance_ebs_secondary_size = 8 + + tags = { + Vendor = "Grafana" + Type = "Monitoring" + MonitoringKey = "loki_logger" + } + }, + """ + end + end + + def terraform_grafana_variables(opts, :oci) do + if opts[:no_grafana] do + "" + else + """ + grafana_ui = { + name = "Grafana UI" + + shape = "VM.Standard.E5.Flex" + ocpus = 1 + memory_gbs = 4 + + boot_volume_size_gbs = 64 + assign_public_ip = true + + tags = { + Vendor = "Grafana" + Type = "Monitoring" + MonitoringKey = "grafana_ui" + } + }, + """ + end + end + + def terraform_grafana_variables(opts, _provider) do + if opts[:no_grafana] do + "" + else + """ + grafana_ui = { + name = "Grafana UI" + enable_ebs = true + enable_eip = true + instance_ebs_secondary_size = 8 + + tags = { + Vendor = "Grafana" + Type = "Monitoring" + MonitoringKey = "grafana_ui" + } + }, + """ + end + end + + def terraform_prometheus_variables(opts, :oci) do + if opts[:no_prometheus] do + "" + else + """ + prometheus_db = { + name = "Prometheus Metrics Database" + + shape = "VM.Standard.E5.Flex" + ocpus = 1 + memory_gbs = 4 + + boot_volume_size_gbs = 64 + + tags = { + Vendor = "Grafana" + Type = "Monitoring" + MonitoringKey = "prometheus_db" + } + }, + """ + end + end + + def terraform_prometheus_variables(opts, _provider) do + if opts[:no_prometheus] do + "" + else + """ + prometheus_db = { + name = "Prometheus Metrics Database" + instance_type = "t3.micro" + enable_ebs = true + instance_ebs_secondary_size = 16 + private_ip = "10.0.1.40" + + tags = { + Vendor = "Grafana" + Type = "Monitoring" + MonitoringKey = "prometheus_db" + } + }, + """ + end + end +end diff --git a/lib/deploy_ex/tui/deploy_progress.ex b/lib/deploy_ex/tui/deploy_progress.ex index 8cdb6a75..2807796c 100644 --- a/lib/deploy_ex/tui/deploy_progress.ex +++ b/lib/deploy_ex/tui/deploy_progress.ex @@ -33,10 +33,28 @@ defmodule DeployEx.TUI.DeployProgress do app_playbooks |> Task.async_stream(fn playbook -> run_fn.(playbook, fn line -> IO.puts(line) end) - end, max_concurrency: max_concurrency, timeout: timeout) + end, max_concurrency: max_concurrency, timeout: timeout, on_timeout: :kill_task) + |> unwrap_task_results() |> DeployEx.Utils.reduce_status_tuples() end + # Task.async_stream reports task COMPLETION, not the task's own result: a run_fn returning + # {:error, _} arrives here as {:ok, {:error, _}}, which reduce_status_tuples/1 matches with + # its `{:ok, record}` SUCCESS clause and accumulates as a successful record. The aggregate + # then reads :ok, ansible.setup/ansible.deploy skip their Mix.raise, and the task exits 0 — + # a failed play reported as a clean deploy, which in CI is a green pipeline that deployed + # nothing. reduce_status_tuples/1 is shared with callers that pass plain status tuples, so + # the envelope is stripped here rather than by loosening the reducer. + defp unwrap_task_results(stream) do + Stream.map(stream, fn + {:ok, result} -> + result + + {:exit, reason} -> + {:error, ErrorMessage.internal_server_error("playbook task exited", %{reason: inspect(reason)})} + end) + end + defp run_tui(app_playbooks, run_fn, opts) do max_concurrency = Keyword.get(opts, :max_concurrency, 4) timeout = Keyword.get(opts, :timeout, :timer.minutes(30)) @@ -81,7 +99,8 @@ defmodule DeployEx.TUI.DeployProgress do send(coordinator, {:deploy_finished, app_name, result}) result - end, max_concurrency: max_concurrency, timeout: timeout) + end, max_concurrency: max_concurrency, timeout: timeout, on_timeout: :kill_task) + |> unwrap_task_results() |> DeployEx.Utils.reduce_status_tuples() end) diff --git a/lib/deploy_ex/tui/wizard/command_registry.ex b/lib/deploy_ex/tui/wizard/command_registry.ex index 2af72a05..18a0c20b 100644 --- a/lib/deploy_ex/tui/wizard/command_registry.ex +++ b/lib/deploy_ex/tui/wizard/command_registry.ex @@ -122,6 +122,7 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do input(:aws_region, "AWS region", :string, description: "Override AWS region"), input(:aws_release_bucket, "AWS release bucket", :string), input(:parallel, "Max concurrency", :integer), + input(:upload_timeout, "Per-release upload timeout (ms)", :integer), input(:qa, "QA upload", :boolean, description: "Upload to the QA prefix") ] }, @@ -448,6 +449,9 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do module: Mix.Tasks.DeployEx.Autoscale.RefreshStatus, category: "Autoscaling", inputs: [ + input(:wait, "Wait for completion", :boolean), + input(:timeout, "Timeout in seconds", :integer), + input(:poll_interval, "Poll interval in seconds", :integer), input(:app_name, "App name", :select, required: true, positional: true, @@ -519,6 +523,7 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do module: Mix.Tasks.DeployEx.Qa.Deploy, category: "QA", inputs: [ + input(:only_local_release, "Only deploy apps built locally", :boolean), input(:app_name, "App name", :select, required: true, positional: true, @@ -600,6 +605,12 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do module: Mix.Tasks.Ansible.Build, category: "Ansible", inputs: [ + input(:provider, "Cloud provider (aws or oci)", :string), + input(:oci_compartment_id, "OCI compartment OCID", :string), + input(:oci_profile, "OCI CLI profile", :string), + input(:oci_region, "OCI region", :string), + input(:oci_namespace, "OCI object storage namespace", :string), + input(:oci_release_bucket, "OCI release bucket", :string), input(:new_only, "New only", :boolean, description: "Only render new files; skip existing"), input(:force, "Force overwrite", :boolean), input(:host_only, "Host only", :boolean), @@ -620,6 +631,7 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do module: Mix.Tasks.Ansible.Deploy, category: "Ansible", inputs: [ + input(:provider, "Cloud provider (aws or oci)", :string), input(:directory, "Ansible directory", :string), input(:quiet, "Quiet", :boolean), input(:only, "Only app(s)", :string, description: "Comma-separated app names to deploy"), @@ -664,6 +676,10 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do module: Mix.Tasks.Ansible.Setup, category: "Ansible", inputs: [ + input(:aws_region, "AWS region", :string), + input(:instance_id, "Instance ID", :string), + input(:git_branch, "Git branch of the QA nodes to target", :string), + input(:provider, "Cloud provider (aws or oci)", :string), input(:directory, "Ansible directory", :string), input(:only, "Only app(s)", :string), input(:except, "Except app(s)", :string), @@ -691,6 +707,7 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do module: Mix.Tasks.Terraform.Build, category: "Terraform", inputs: [ + input(:provider, "Cloud provider (aws or oci)", :string), input(:directory, "Terraform directory", :string), input(:force, "Force overwrite", :boolean), input(:quiet, "Quiet", :boolean), @@ -703,6 +720,8 @@ defmodule DeployEx.TUI.Wizard.CommandRegistry do input(:no_sentry, "Disable Sentry", :boolean), input(:no_grafana, "Disable Grafana", :boolean), input(:no_redis, "Disable Redis", :boolean), + input(:clickhouse, "Add a ClickHouse node (OCI)", :boolean), + input(:rabbitmq, "Add a RabbitMQ node (OCI)", :boolean), input(:no_prometheus, "Disable Prometheus", :boolean) ] }, diff --git a/lib/mix/deploy_ex_helpers.ex b/lib/mix/deploy_ex_helpers.ex index 42ef0a07..5d797269 100644 --- a/lib/mix/deploy_ex_helpers.ex +++ b/lib/mix/deploy_ex_helpers.ex @@ -29,7 +29,11 @@ defmodule DeployExHelpers do def kebab_project_name, do: String.replace(underscored_project_name(), "_", "-") def title_case_project_name, do: DeployEx.Utils.upper_title_case(underscored_project_name()) - def check_valid_project, do: DeployEx.ProjectContext.check_valid_project() + def check_valid_project do + with :ok <- DeployEx.ProjectContext.check_valid_project() do + DeployEx.Cloud.validate_config() + end + end def priv_folder(priv_subdirectory) do priv_path = :deploy_ex |> :code.priv_dir() |> Path.join(priv_subdirectory) @@ -65,12 +69,28 @@ defmodule DeployExHelpers do if !opts[:quiet] do Mix.shell().info(opts[:message]) end + else + warn_write_skipped(file_path, opts) end else Mix.Generator.create_file(file_path, contents, opts) end end + # Mix.Generator.overwrite?/2 only prompts when the new contents DIFFER from what is on disk, + # and it returns false at EOF. So under `< /dev/null`, or anywhere without a tty, the files + # that most need updating are precisely the ones left stale — and the task still exits 0. + # MEASURED: a rebuilt ansible tree kept its previous inventory and setup playbook this way, + # which in CI is a deploy against the wrong host list reported as a success. + defp warn_write_skipped(file_path, opts) do + if !opts[:quiet] do + Mix.shell().error( + "* SKIPPED #{file_path} — on-disk contents differ and the overwrite was declined; " <> + "pass --force to update it" + ) + end + end + def check_file_exists!(file_path) do if !File.exists?(file_path) do raise to_string(IO.ANSI.format([ diff --git a/lib/mix/tasks/ansible.build.ex b/lib/mix/tasks/ansible.build.ex index 118cce72..b87e79de 100644 --- a/lib/mix/tasks/ansible.build.ex +++ b/lib/mix/tasks/ansible.build.ex @@ -6,6 +6,7 @@ defmodule Mix.Tasks.Ansible.Build do @ansible_default_path Config.ansible_folder_path() @terraform_default_path Config.terraform_folder_path() @aws_credentials_regex ~r/aws_access_key_id = (?[A-Z0-9]+)\naws_secret_access_key = (?[a-z-A-Z0-9\/\+]+)\n/ + @render_dir_pem_file_path "../terraform/RENDER_DIR_PLACEHOLDER.pem" @shortdoc "Builds ansible files into your repository" @moduledoc """ @@ -15,12 +16,24 @@ defmodule Mix.Tasks.Ansible.Build do ## Options - `directory` - Ansible directory path (default: #{@ansible_default_path}) - `terraform_directory` - Terraform directory path (default: #{@terraform_default_path}) + - `provider` - Cloud provider file set to render (default: `DeployEx.Config.cloud_provider/0`) + - `render_dir` - Render every ansible file into this directory instead of the live + tree, using a placeholder pem path rather than globbing for a real one. Used by + the render diff harness to compare output across revisions. - `force` - Force overwrite existing files - `quiet` - Suppress output messages - `host_only` - Only generate host configuration files - `new_only` - Only generate files for new applications - - `auto_pull_aws` - Automatically pull AWS credentials from ~/.aws/credentials + - `auto_pull_aws` - Automatically pull AWS credentials from ~/.aws/credentials (aws only) - `aws_release_bucket` - AWS S3 bucket for releases + - `oci_compartment_id` - OCI compartment to list instances from when building the + static inventory (default: `config :deploy_ex, :oci, compartment_id: ...`) + - `oci_profile` - OCI CLI profile (default: `config :deploy_ex, :oci, profile: ...`) + - `oci_region` - OCI region (default: `config :deploy_ex, :oci, region: ...`) + - `oci_namespace` - OCI Object Storage namespace, required for the oci provider + (default: `config :deploy_ex, :oci, namespace: ...`) + - `oci_release_bucket` - OCI Object Storage bucket for releases + (default: `config :deploy_ex, :oci, release_bucket: ...`) - `no_logging` - Disable logging configuration (Alloy + Loki) - `no_loki` - Deprecated alias for `no_logging` - `no_sentry` - Disable Sentry error tracking configuration @@ -32,11 +45,18 @@ defmodule Mix.Tasks.Ansible.Build do Application.ensure_all_started(:hackney) Application.ensure_all_started(:telemetry) - opts = args - |> parse_args + parsed_opts = parse_args(args) + + # --provider overrides the configured cloud_provider. Resolved once so the seed, the + # inventory render and ansible.cfg can never disagree about which file set they use — + # mirrors terraform.build.ex's resolve_provider/1. + provider = resolve_provider(parsed_opts) + + opts = parsed_opts + |> put_render_dir_paths(provider) |> Keyword.put_new(:directory, @ansible_default_path) |> Keyword.put_new(:terraform_directory, @terraform_default_path) - |> Keyword.put_new(:hosts_file, "./deploys/ansible/aws_ec2.yaml") + |> Keyword.put_new(:hosts_file, "./deploys/ansible/#{inventory_filename(provider)}") |> Keyword.put_new(:config_file, "./deploys/ansible/ansible.cfg") |> Keyword.put_new(:group_vars_file, "./deploys/ansible/group_vars/all.yaml") |> Keyword.put_new(:aws_logging_bucket, Config.aws_log_bucket()) @@ -48,13 +68,15 @@ defmodule Mix.Tasks.Ansible.Build do opts = Keyword.put(opts, :no_logging, no_logging) with :ok <- DeployExHelpers.check_valid_project(), - :ok <- ensure_ansible_directory_exists(opts[:directory], opts), - :ok <- sync_ansible_roles(opts[:directory], opts), - :ok <- create_ansible_hosts_file(opts), - :ok <- create_ansible_config_file(opts), - :ok <- create_ansible_group_vars_file(opts), + :ok <- validate_provider_opts(provider, opts), + :ok <- ensure_ansible_directory_exists(opts[:directory], provider, opts), + :ok <- sync_ansible_roles(opts[:directory], provider, opts), + :ok <- seed_new_setup_playbooks(opts[:directory], opts), + :ok <- create_ansible_hosts_file(provider, opts), + :ok <- create_ansible_config_file(provider, opts), + :ok <- create_ansible_group_vars_file(provider, opts), {:ok, app_names} <- DeployExHelpers.fetch_mix_release_names(), - :ok <- create_ansible_playbooks(app_names, opts) do + :ok <- create_ansible_playbooks(app_names, provider, opts) do :ok else {:error, [h | tail]} -> @@ -65,6 +87,86 @@ defmodule Mix.Tasks.Ansible.Build do end end + # Runs BEFORE ensure_ansible_directory_exists/3 on purpose: fetch_oci_instances/1 raised the + # same "compartment_id is required" error, but only once create_ansible_hosts_file/2 got + # around to calling it — after the tree was already seeded, leaving a half-built directory + # full of unrendered .eex behind a raise. Validating first means a bad invocation never + # touches the filesystem at all. + defp validate_provider_opts(provider, opts) do + with :ok <- validate_auto_pull_aws(provider, opts) do + validate_provider_config(provider, opts) + end + end + + # A flag the provider does not support is a usage error, so it is reported before any + # environment requirement — otherwise `--provider oci --auto-pull-aws` complains about + # missing OCI config and never mentions that the flag itself is the problem. This check also + # lived inside the directory-seeding branch, so it only fired when the tree did not already + # exist: re-running against an existing ./deploys ignored the flag silently. + defp validate_auto_pull_aws(:aws, _opts), do: :ok + + defp validate_auto_pull_aws(provider, opts) do + if opts[:auto_pull_aws] do + {:error, + ErrorMessage.bad_request("--auto-pull-aws only supports the aws provider, got #{inspect(provider)}")} + else + :ok + end + end + + defp validate_provider_config(:oci, opts) do + with {:ok, _compartment_id} <- require_oci_compartment_id(opts), + {:ok, _namespace} <- require_oci_namespace(opts) do + :ok + end + end + + defp validate_provider_config(_provider, _opts), do: :ok + + # Matches the flag against the registered providers rather than calling to_existing_atom/1 — + # see terraform.build.ex's identical resolve_provider/1 for the rationale. + defp resolve_provider(opts) do + case opts[:provider] do + nil -> + DeployEx.Config.cloud_provider() + + name -> + known = DeployEx.Cloud.providers() + + case Enum.find(known, &(to_string(&1) === name)) do + nil -> Mix.raise("unknown provider #{inspect(name)}, expected one of #{inspect(known)}") + provider -> provider + end + end + end + + # Single source of truth for "what is this provider's inventory called" — the descriptor's + # inventory/0 slot, also read by Mix.Tasks.Ansible.{Setup,Deploy,Ping}. Keeping ansible.build + # on its own hardcoded filename here (instead of the same lookup) would let this task and + # the ones that consume its output silently drift apart on a provider swap. + defp inventory_filename(provider) do + case DeployEx.Cloud.inventory(provider) do + {:ok, %{filename: filename}} -> filename + {:error, error} -> Mix.raise(to_string(error)) + end + end + + defp inventory_template_filename(provider), do: "#{inventory_filename(provider)}.eex" + + defp put_render_dir_paths(opts, provider) do + case opts[:render_dir] do + nil -> + opts + + render_dir -> + opts + |> Keyword.put(:directory, render_dir) + |> Keyword.put(:hosts_file, Path.join(render_dir, inventory_filename(provider))) + |> Keyword.put(:config_file, Path.join(render_dir, "ansible.cfg")) + |> Keyword.put(:group_vars_file, Path.join(render_dir, "group_vars/all.yaml")) + end + end + defp parse_args(args) do {opts, _} = OptionParser.parse!(args, aliases: [f: :force, q: :quit, d: :directory, a: :auto_pull_aws, h: :host_only, n: :new_only], @@ -74,9 +176,16 @@ defmodule Mix.Tasks.Ansible.Build do host_only: :boolean, quiet: :boolean, directory: :string, + render_dir: :string, + provider: :string, terraform_directory: :string, auto_pull_aws: :boolean, aws_release_bucket: :string, + oci_compartment_id: :string, + oci_profile: :string, + oci_region: :string, + oci_namespace: :string, + oci_release_bucket: :string, no_logging: :boolean, no_loki: :boolean, no_sentry: :boolean, @@ -88,7 +197,15 @@ defmodule Mix.Tasks.Ansible.Build do opts end - defp ensure_ansible_directory_exists(directory, opts) do + # Setup playbooks and the playbook/group_vars templates are shared across providers, so + # they're seeded verbatim for everyone via the same whole-tree copy as before. The + # `providers/` subtree is provider-EXCLUSIVE content (ansible.cfg.eex, the hosts template, + # and now the oci-only role variants under providers/oci/roles) that's about to be rendered + # fresh by create_ansible_config_file/2, create_ansible_hosts_file/2, and + # sync_ansible_roles/3 regardless — copying it raw here would leak an oci user's + # ansible.cfg.eex (or role files) into an aws tree (and vice versa) and then sit there + # unused, so it's stripped right back out. + defp ensure_ansible_directory_exists(directory, provider, opts) do if File.exists?(directory) do :ok else @@ -100,9 +217,27 @@ defmodule Mix.Tasks.Ansible.Build do |> DeployExHelpers.priv_folder() |> File.cp_r!(directory) + providers_dir = Path.join(directory, "providers") + + if File.dir?(providers_dir) do + File.rm_rf!(providers_dir) + end + + # aws_ec2.yaml.eex is the one root-level template with no same-path counterpart for + # other providers (ansible.cfg.eex is shared-by-name and always gets overwritten below + # regardless of which provider rendered it) — so it's the one leftover a non-aws build + # would otherwise leak, unused, into the tree. + if provider !== :aws do + aws_hosts_template = Path.join(directory, inventory_template_filename(:aws)) + + if File.exists?(aws_hosts_template) do + File.rm!(aws_hosts_template) + end + end + File.rm!(Path.join(directory, "group_vars/all.yaml.eex")) - create_ansible_group_vars_file(opts) + create_ansible_group_vars_file(provider, opts) if opts[:auto_pull_aws] do pull_aws_credentials_into_awscli_variables(directory, opts) @@ -162,74 +297,159 @@ defmodule Mix.Tasks.Ansible.Build do end end - defp create_ansible_group_vars_file(opts) do + defp create_ansible_group_vars_file(provider, opts) do if opts[:host_only] do :ok else - variables = %{ - is_logging_enabled: !opts[:no_logging], - is_prometheus_enabled: !opts[:no_prometheus], - loki_logger_s3_region: opts[:aws_logging_bucket], - loki_logger_s3_bucket_name: opts[:aws_logging_region] - } - - DeployExHelpers.write_template( - DeployExHelpers.priv_folder("ansible/group_vars/all.yaml.eex"), - opts[:group_vars_file], - variables, - opts - ) - - if File.exists?("#{opts[:group_vars_file]}.eex") do - File.rm!("#{opts[:group_vars_file]}.eex") + with {:ok, template_path} <- find_provider_template(provider, "group_vars/all.yaml.eex") do + DeployExHelpers.write_template( + template_path, + opts[:group_vars_file], + group_vars_template_variables(provider, opts), + opts + ) + + if File.exists?("#{opts[:group_vars_file]}.eex") do + File.rm!("#{opts[:group_vars_file]}.eex") + end + + :ok end - - :ok end end - defp create_ansible_config_file(opts) do + defp group_vars_template_variables(:oci, opts) do + %{ + is_logging_enabled: !opts[:no_logging], + is_prometheus_enabled: !opts[:no_prometheus], + oci_namespace: oci_setting(opts, :namespace), + oci_release_bucket: oci_release_bucket(opts), + oci_vcn_cidr: oci_setting(opts, :vcn_cidr) || "10.20.0.0/16" + } + end + + defp group_vars_template_variables(_provider, opts) do + %{ + is_logging_enabled: !opts[:no_logging], + is_prometheus_enabled: !opts[:no_prometheus], + loki_logger_s3_region: opts[:aws_logging_bucket], + loki_logger_s3_bucket_name: opts[:aws_logging_region] + } + end + + defp create_ansible_config_file(provider, opts) do if opts[:host_only] do :ok else app_name = String.replace(DeployExHelpers.underscored_project_name(), "_", "-") variables = %{ - pem_file_path: pem_file_path(app_name, opts[:directory]) + pem_file_path: config_pem_file_path(app_name, opts) } - DeployExHelpers.write_template( - DeployExHelpers.priv_folder("ansible/ansible.cfg.eex"), - opts[:config_file], - variables, - opts - ) + with {:ok, template_path} <- find_provider_template(provider, "ansible.cfg.eex") do + DeployExHelpers.write_template(template_path, opts[:config_file], variables, opts) + + if File.exists?("#{opts[:config_file]}.eex") do + File.rm!("#{opts[:config_file]}.eex") + end + + :ok + end + end + end + + defp create_ansible_hosts_file(provider, opts) do + with {:ok, template_path} <- find_provider_template(provider, inventory_template_filename(provider)), + {:ok, variables} <- hosts_template_variables(provider, opts) do + DeployExHelpers.write_template(template_path, opts[:hosts_file], variables, opts) - if File.exists?("#{opts[:config_file]}.eex") do - File.rm!("#{opts[:config_file]}.eex") + if File.exists?("#{opts[:hosts_file]}.eex") do + File.rm!("#{opts[:hosts_file]}.eex") end + remove_other_provider_inventories(provider, opts) + :ok end end - defp create_ansible_hosts_file(opts) do - variables = %{ - app_name: DeployExHelpers.underscored_project_name() - } + @doc false + # Re-running ansible.build with a DIFFERENT --provider against an already-built directory + # only overwrites ansible.cfg and adds the new provider's inventory file — it never touched + # the previous provider's leftover inventory file, which would then sit there stale while + # ansible.cfg's `inventory =` line (correctly) points elsewhere. ansible.setup/deploy/ping's + # preflight existence check would find that stale file and pass, checking the WRONG + # provider's inventory while ansible-playbook itself reads the freshly-rendered one from + # cfg — the "cfg says X, something else says Y" trap this exists to close. Deleting every + # OTHER known provider's inventory file on each build keeps exactly one inventory file + # present at a time, so a provider switch is either fully clean or (missing template/config) + # loudly broken, never silently mixed. Public (not private) so this is unit-testable without + # a live oci CLI: the impure fetch and this cleanup are separate steps on purpose. + def remove_other_provider_inventories(provider, opts) do + DeployEx.Cloud.providers() + |> Enum.reject(&(&1 === provider)) + |> Enum.each(&remove_stale_inventory(&1, opts[:directory])) + end - DeployExHelpers.write_template( - DeployExHelpers.priv_folder("ansible/aws_ec2.yaml.eex"), - opts[:hosts_file], - variables, - opts - ) + defp remove_stale_inventory(other_provider, directory) do + case DeployEx.Cloud.inventory(other_provider) do + {:ok, %{filename: filename}} -> + stale_path = Path.join(directory, filename) - if File.exists?("#{opts[:hosts_file]}.eex") do - File.rm!("#{opts[:hosts_file]}.eex") + if File.exists?(stale_path) do + File.rm!(stale_path) + end + + {:error, _not_implemented} -> + :ok end + end - :ok + # Resolves which priv template a provider uses for a given rendered filename via + # DeployEx.Cloud.PrivFileSet, the same seam terraform.build.ex uses for its whole file + # set. Here it's used per-file rather than tree-wide because most of priv/ansible (roles, + # setup, playbook templates) is shared across providers and only ansible.cfg/the hosts + # template actually vary. + defp find_provider_template(provider, dest_filename) do + priv_path = DeployExHelpers.priv_folder("ansible") + + with {:ok, files} <- DeployEx.Cloud.PrivFileSet.files(provider, priv_path) do + case Enum.find(files, fn {_source, dest} -> dest === dest_filename end) do + {source, _dest} -> + {:ok, Path.join(priv_path, source)} + + nil -> + {:error, + ErrorMessage.not_found("no #{dest_filename} template for #{provider} under #{priv_path}", %{ + provider: provider, + filename: dest_filename + })} + end + end + end + + defp hosts_template_variables(:aws, _opts) do + {:ok, %{app_name: DeployExHelpers.underscored_project_name()}} + end + + defp hosts_template_variables(_provider, opts) do + with {:ok, instances} <- fetch_oci_instances(opts) do + hosts = oci_inventory_hosts(instances) + + {:ok, %{ + hosts_section: render_oci_hosts_section(hosts), + children_section: render_oci_children_section(hosts) + }} + end + end + + defp config_pem_file_path(app_name, opts) do + if opts[:render_dir] do + @render_dir_pem_file_path + else + pem_file_path(app_name, opts[:directory]) + end end defp pem_file_path(app_name, directory) do @@ -256,7 +476,7 @@ defmodule Mix.Tasks.Ansible.Build do "#{host_name}_#{:io_lib.format("~3..0B", [index])}" end - defp create_ansible_playbooks(app_names, opts) do + defp create_ansible_playbooks(app_names, provider, opts) do if opts[:host_only] do :ok else @@ -272,9 +492,15 @@ defmodule Mix.Tasks.Ansible.Build do end if opts[:new_only] do - deploy_new_playbooks(app_names, project_playbooks_path, project_setup_playbooks_path, opts) + deploy_new_playbooks( + app_names, + provider, + project_playbooks_path, + project_setup_playbooks_path, + opts + ) else - deploy_all_playbooks(app_names, opts) + deploy_all_playbooks(app_names, provider, opts) end remove_usless_copied_template_folder(opts) @@ -283,20 +509,20 @@ defmodule Mix.Tasks.Ansible.Build do end end - defp deploy_all_playbooks(app_names, opts) do + defp deploy_all_playbooks(app_names, provider, opts) do Enum.each(app_names, fn app_name -> - build_host_setup_playbook(app_name, opts) + build_host_setup_playbook(app_name, provider, opts) build_host_playbook(app_name, opts) end) end - defp deploy_new_playbooks(app_names, project_playbooks_path, project_setup_playbooks_path, opts) do + defp deploy_new_playbooks(app_names, provider, project_playbooks_path, project_setup_playbooks_path, opts) do project_deploy_files = File.ls!(project_playbooks_path) project_setup_files = File.ls!(project_setup_playbooks_path) Enum.each(app_names, fn app_name -> if not Enum.any?(project_setup_files, &(&1 =~ app_name)) do - build_host_setup_playbook(app_name, opts) + build_host_setup_playbook(app_name, provider, opts) end if not Enum.any?(project_deploy_files, &(&1 =~ app_name)) do @@ -324,7 +550,7 @@ defmodule Mix.Tasks.Ansible.Build do ) end - defp build_host_setup_playbook(app_name, opts) do + defp build_host_setup_playbook(app_name, provider, opts) do setup_playbook_path = DeployExHelpers.priv_folder("ansible/app_setup_playbook.yaml.eex") setup_host_playbook = Path.join(opts[:directory], "setup/#{app_name}.yaml") @@ -332,7 +558,8 @@ defmodule Mix.Tasks.Ansible.Build do no_logging: opts[:no_logging], no_prometheus: opts[:no_prometheus], app_name: app_name, - port: 80 + port: 80, + cloud_provider: provider } DeployExHelpers.write_template( @@ -343,7 +570,7 @@ defmodule Mix.Tasks.Ansible.Build do ) end - defp sync_ansible_roles(directory, opts) do + defp sync_ansible_roles(directory, provider, opts) do priv_roles = DeployExHelpers.priv_folder("ansible/roles") target_roles = Path.join(directory, "roles") @@ -353,6 +580,50 @@ defmodule Mix.Tasks.Ansible.Build do end File.cp_r!(priv_roles, target_roles) + sync_provider_role_overlay(provider, target_roles) + end + + :ok + end + + # Setup playbooks were seeded only when the ansible directory was first created, so a role + # added to deploy_ex later (rabbitmq_server) synced its ROLE into an existing tree but never + # its setup/.yaml — `mix ansible.setup --only rabbitmq` then matched no file and + # exited 0 having done nothing (measured). Only NEW playbooks are seeded: existing ones are + # user-owned (operators hand-add roles to them) and are never overwritten here. + defp seed_new_setup_playbooks(directory, opts) do + priv_setup = DeployExHelpers.priv_folder("ansible/setup") + target_setup = Path.join(directory, "setup") + + if File.dir?(priv_setup) and File.dir?(target_setup) do + priv_setup + |> Path.join("*.yaml") + |> Path.wildcard() + |> Enum.reject(&File.exists?(Path.join(target_setup, Path.basename(&1)))) + |> Enum.each(fn source -> + target = Path.join(target_setup, Path.basename(source)) + File.cp!(source, target) + + unless opts[:quiet] do + Mix.shell().info([:green, "* seeding new setup playbook ", :reset, target]) + end + end) + end + + :ok + end + + # AWS is the shared role tree as-is — there is no providers/aws/roles directory, so this + # is a no-op for it. A provider with its own role variants (e.g. OCI's deploy_node + # tasks/main.yaml and files/*.sh, which use the oci CLI instead of aws s3) ships them + # under providers//roles/ mirroring the shared roles/ layout; copying that tree on + # top after the shared copy above overlays/replaces just those files, leaving every other + # role byte-identical to the shared set. + defp sync_provider_role_overlay(provider, target_roles) do + provider_roles = DeployExHelpers.priv_folder("ansible/providers/#{provider}/roles") + + if File.dir?(provider_roles) do + File.cp_r!(provider_roles, target_roles) end :ok @@ -370,4 +641,241 @@ defmodule Mix.Tasks.Ansible.Build do File.rm!(setup_template_file) end end + + # SECTION OCI STATIC INVENTORY + # + # AWS resolves hosts live at ansible-run-time through the aws_ec2 plugin. OCI has no + # inventory plugin we're willing to add as a collection dependency, so this queries the + # `oci` CLI directly (no DeployEx.Cloud.Machine capability exists for OCI yet) and renders + # a point-in-time snapshot — regenerated on every `mix ansible.build` run, same as the rest + # of the hosts file. + # + # The four-part contract this mirrors from aws_ec2.yaml.eex: group names keyed off + # MonitoringKey/InstanceGroup/DatabaseKey/QaNode tags, the same seven tag-derived hostvars + # plus ansible_host, hostname = "-", and the project-scope filter + # on the Group tag. NOTE: deploy_ex has no SharedUtils dependency (it isn't part of the + # umbrella apps that vendor it), so the tag-filtering below is plain Enum, not + # SharedUtils.Enum.reject_empty_values/1. + + @oci_keyed_group_tags [ + {"MonitoringKey", "monitoring"}, + {"InstanceGroup", "group"}, + {"DatabaseKey", "database"}, + {"QaNode", "qa"} + ] + + defp fetch_oci_instances(opts) do + with {:ok, compartment_id} <- require_oci_compartment_id(opts), + {:ok, instances} <- oci_list_instances(compartment_id, opts) do + instances + |> Enum.filter(&oci_project_scoped?/1) + |> oci_hydrate_instances(opts, []) + end + end + + defp require_oci_compartment_id(opts) do + case oci_setting(opts, :compartment_id) do + nil -> + {:error, + ErrorMessage.bad_request( + "oci compartment_id is required to build the static inventory " <> + "(config :deploy_ex, :oci, compartment_id: \"...\", or --oci-compartment-id)" + )} + + compartment_id -> + {:ok, compartment_id} + end + end + + # Required like compartment_id: unlike the release bucket (a name deploy_ex can default), + # the Object Storage namespace is a tenancy-assigned identifier with no sensible guess, so + # a missing value fails validate_provider_opts/2 up front rather than surfacing later as a + # broken group_vars render or a confusing oci CLI error on the node. + defp require_oci_namespace(opts) do + case oci_setting(opts, :namespace) do + nil -> + {:error, + ErrorMessage.bad_request( + "oci namespace is required for the oci provider " <> + "(config :deploy_ex, :oci, namespace: \"...\", or --oci-namespace)" + )} + + namespace -> + {:ok, namespace} + end + end + + defp oci_setting(opts, key), do: opts[:"oci_#{key}"] || oci_config_setting(key) + + defp oci_config_setting(key), do: :deploy_ex |> Application.get_env(:oci, []) |> Keyword.get(key) + + defp oci_release_bucket(opts) do + oci_setting(opts, :release_bucket) || + "#{DeployExHelpers.kebab_project_name()}-elixir-deploys-#{Config.env()}" + end + + defp oci_project_scoped?(instance) do + expected_group = "#{DeployEx.Utils.upper_title_case(DeployExHelpers.underscored_project_name())} Backend" + + get_in(instance, ["freeform-tags", "Group"]) === expected_group + end + + defp oci_list_instances(compartment_id, opts) do + command = + oci_command(opts, "compute instance list --compartment-id #{compartment_id} --lifecycle-state RUNNING --output json") + + with {:ok, output} <- DeployEx.Utils.run_command_with_return(command, File.cwd!()), + {:ok, decoded} <- oci_decode_json(output) do + {:ok, decoded["data"] || []} + end + end + + defp oci_hydrate_instances([], _opts, acc), do: {:ok, Enum.reverse(acc)} + + defp oci_hydrate_instances([instance | rest], opts, acc) do + case oci_hydrate_instance(instance, opts) do + {:ok, hydrated} -> oci_hydrate_instances(rest, opts, [hydrated | acc]) + {:error, _} = error -> error + end + end + + defp oci_hydrate_instance(instance, opts) do + command = oci_command(opts, "compute instance list-vnics --instance-id #{instance["id"]} --output json") + + with {:ok, output} <- DeployEx.Utils.run_command_with_return(command, File.cwd!()), + {:ok, decoded} <- oci_decode_json(output) do + case oci_primary_vnic(decoded["data"] || []) do + nil -> {:error, ErrorMessage.not_found("no vnic found for oci instance #{instance["id"]}")} + vnic -> {:ok, oci_instance_from_vnic(instance, vnic)} + end + end + end + + defp oci_primary_vnic(vnics), do: Enum.find(vnics, & &1["is-primary"]) || List.first(vnics) + + defp oci_instance_from_vnic(instance, vnic) do + tags = instance["freeform-tags"] || %{} + + %{ + id: instance["id"], + name: tags["Name"] || instance["display-name"], + tags: tags, + public_ip: vnic["public-ip"], + private_ip: vnic["private-ip"], + ipv6: vnic |> Map.get("ipv6-addresses") |> List.wrap() |> List.first() + } + end + + # OCI_CLI_AUTH=api_key is the non-interactive auth mode deploy_ex automation needs + # (session-token auth requires a browser). SUPPRESS_LABEL_WARNING avoids a stderr nag + # about unlabeled API keys, which would otherwise land in the merged stdout/stderr stream + # DeployEx.Utils.run_command_with_return/3 returns and break JSON decoding. + defp oci_command(opts, subcommand) do + flags = + [oci_flag("--profile", oci_setting(opts, :profile)), oci_flag("--region", oci_setting(opts, :region))] + |> Enum.filter(& &1) + |> Enum.join(" ") + + String.trim("OCI_CLI_AUTH=api_key SUPPRESS_LABEL_WARNING=True oci #{subcommand} #{flags}") + end + + defp oci_flag(_flag, nil), do: nil + defp oci_flag(flag, value), do: "#{flag} #{value}" + + # The oci CLI prints NOTHING — not `{"data": []}` — when a list matches no resources, so an + # empty compartment has to decode to an empty result rather than a JSON error. Without this a + # project with no instances yet cannot build an inventory at all: it fails with "unexpected + # end of input" instead of producing an empty one, which is the normal first-run state. + defp oci_decode_json(output) do + case String.trim(output) do + "" -> {:ok, %{"data" => []}} + trimmed -> decode_oci_payload(trimmed, output) + end + end + + defp decode_oci_payload(trimmed, original) do + case Jason.decode(trimmed) do + {:ok, decoded} -> + {:ok, decoded} + + {:error, decode_error} -> + {:error, + ErrorMessage.internal_server_error( + "failed to decode oci CLI JSON output: #{Exception.message(decode_error)}", + %{output: original} + )} + end + end + + @doc false + # Pure transform from hydrated OCI instances to inventory host entries — kept separate + # from fetch_oci_instances/1 so the group/hostvar composition contract is unit-testable + # without a live oci CLI or network access. + def oci_inventory_hosts(instances) do + Enum.map(instances, &oci_inventory_host/1) + end + + defp oci_inventory_host(%{id: id, name: name, tags: tags} = instance) do + qa_node? = Map.get(tags, "QaNode") === "true" + + %{ + hostname: "#{id}-#{name}", + groups: oci_keyed_groups(tags), + vars: %{ + ansible_host: instance[:ipv6] || instance[:public_ip] || instance[:private_ip], + release_prefix: if(qa_node?, do: "qa", else: ""), + release_state_prefix: if(qa_node?, do: "release-state/qa", else: "release-state"), + git_branch: Map.get(tags, "GitBranch", ""), + qa_node: qa_node?, + qa_node_suffix: if(qa_node?, do: "_qa", else: ""), + instance_tag: Map.get(tags, "InstanceTag", ""), + letsencrypt_use_public_ip: Map.get(tags, "UsePublicIpCert") === "true" + } + } + end + + defp oci_keyed_groups(tags) do + @oci_keyed_group_tags + |> Enum.map(fn {tag_key, prefix} -> {prefix, Map.get(tags, tag_key)} end) + |> Enum.reject(fn {_prefix, value} -> value in [nil, ""] end) + |> Enum.map(fn {prefix, value} -> "#{prefix}_#{value}" end) + end + + @doc false + def render_oci_hosts_section(hosts) do + if Enum.empty?(hosts) do + " hosts: {}" + else + " hosts:\n" <> Enum.map_join(hosts, "\n", &render_oci_host_entry/1) + end + end + + defp render_oci_host_entry(%{hostname: hostname, vars: vars}) do + var_lines = Enum.map_join(vars, "\n", fn {key, value} -> " #{key}: #{oci_yaml_scalar(value)}" end) + + " #{hostname}:\n#{var_lines}" + end + + @doc false + def render_oci_children_section(hosts) do + groups = + hosts + |> Enum.flat_map(fn %{hostname: hostname, groups: groups} -> Enum.map(groups, &{&1, hostname}) end) + |> Enum.group_by(fn {group, _hostname} -> group end, fn {_group, hostname} -> hostname end) + + if Enum.empty?(groups) do + " children: {}" + else + " children:\n" <> Enum.map_join(groups, "\n", &render_oci_group_entry/1) + end + end + + defp render_oci_group_entry({group, hostnames}) do + host_lines = Enum.map_join(hostnames, "\n", &" #{&1}: {}") + + " #{group}:\n hosts:\n#{host_lines}" + end + + defp oci_yaml_scalar(value) when is_boolean(value), do: to_string(value) + defp oci_yaml_scalar(value), do: inspect(to_string(value)) end diff --git a/lib/mix/tasks/ansible.deploy.ex b/lib/mix/tasks/ansible.deploy.ex index d08483d2..47636ac8 100644 --- a/lib/mix/tasks/ansible.deploy.ex +++ b/lib/mix/tasks/ansible.deploy.ex @@ -50,6 +50,10 @@ defmodule Mix.Tasks.Ansible.Deploy do ## Options - `directory` - Directory containing ansible playbooks (default: #{@ansible_default_path}) + - `provider` - Cloud provider whose inventory the directory was built with + (default: `DeployEx.Config.cloud_provider/0`). Only affects which inventory + filename is checked for before deploying — mismatch raises rather than + silently deploying against the wrong (or a stale) inventory. - `only` - Only deploy specified apps (can be used multiple times) - `except` - Skip deploying specified apps (can be used multiple times) - `copy-json-env-file` - Copy environment file and load into host environments @@ -94,7 +98,9 @@ defmodule Mix.Tasks.Ansible.Deploy do arg -> [arg] end) - DeployExHelpers.check_file_exists!(Path.join(opts[:directory], "aws_ec2.yaml")) + provider = resolve_provider(opts) + + DeployExHelpers.check_file_exists!(Path.join(opts[:directory], inventory_filename(provider))) if opts[:target_sha] || opts[:qa] === true || opts[:select_sha] === true do Application.ensure_all_started(:hackney) @@ -158,6 +164,7 @@ defmodule Mix.Tasks.Ansible.Deploy do aliases: [f: :force, q: :quit, d: :directory, l: :only_local_release, t: :target_sha], switches: [ directory: :string, + provider: :string, quiet: :boolean, only: :keep, except: :keep, @@ -185,6 +192,30 @@ defmodule Mix.Tasks.Ansible.Deploy do Mix.raise("--release-prefix must be 'prod' or 'qa', got: #{inspect(prefix)}") end + # Matches the flag against the registered providers rather than calling to_existing_atom/1 — + # see terraform.build.ex's identical resolve_provider/1 for the rationale. + defp resolve_provider(opts) do + case opts[:provider] do + nil -> + DeployEx.Config.cloud_provider() + + name -> + known = DeployEx.Cloud.providers() + + case Enum.find(known, &(to_string(&1) === name)) do + nil -> Mix.raise("unknown provider #{inspect(name)}, expected one of #{inspect(known)}") + provider -> provider + end + end + end + + defp inventory_filename(provider) do + case DeployEx.Cloud.inventory(provider) do + {:ok, %{filename: filename}} -> filename + {:error, error} -> Mix.raise(to_string(error)) + end + end + def build_ansible_playbook_command(host_playbook, opts) do ["ansible-playbook", host_playbook] |> add_copy_env_file_flag(opts) diff --git a/lib/mix/tasks/ansible.ping.ex b/lib/mix/tasks/ansible.ping.ex index 64f3fc46..76fe2a20 100644 --- a/lib/mix/tasks/ansible.ping.ex +++ b/lib/mix/tasks/ansible.ping.ex @@ -8,9 +8,12 @@ defmodule Mix.Tasks.Ansible.Ping do ## Example ```bash mix ansible.ping + mix ansible.ping --provider oci ``` ## Options + - `provider` - Cloud provider whose inventory to ping (default: `DeployEx.Config.cloud_provider/0`) + Any additional arguments passed will be forwarded directly to the ansible command. Common options include: - `-v` - Increase verbosity @@ -19,15 +22,47 @@ defmodule Mix.Tasks.Ansible.Ping do def run(args) do ansible_args = DeployEx.Ansible.parse_args(args) + provider = args |> parse_args() |> resolve_provider() with :ok <- DeployExHelpers.check_valid_project(), :ok <- DeployEx.ToolInstaller.ensure_installed(:ansible) do - DeployExHelpers.check_file_exists!("./deploys/ansible/aws_ec2.yaml") + inventory_filename = inventory_filename(provider) + + DeployExHelpers.check_file_exists!("./deploys/ansible/#{inventory_filename}") DeployEx.Utils.run_command( - "ansible -i aws_ec2.yaml #{ansible_args} all -m ping", + "ansible -i #{inventory_filename} #{ansible_args} all -m ping", "./deploys/ansible" ) end end + + defp parse_args(args) do + {opts, _extra_args} = OptionParser.parse!(args, switches: [provider: :string]) + opts + end + + # Matches the flag against the registered providers rather than calling to_existing_atom/1 — + # see terraform.build.ex's identical resolve_provider/1 for the rationale. + defp resolve_provider(opts) do + case opts[:provider] do + nil -> + DeployEx.Config.cloud_provider() + + name -> + known = DeployEx.Cloud.providers() + + case Enum.find(known, &(to_string(&1) === name)) do + nil -> Mix.raise("unknown provider #{inspect(name)}, expected one of #{inspect(known)}") + provider -> provider + end + end + end + + defp inventory_filename(provider) do + case DeployEx.Cloud.inventory(provider) do + {:ok, %{filename: filename}} -> filename + {:error, error} -> Mix.raise(to_string(error)) + end + end end diff --git a/lib/mix/tasks/ansible.setup.ex b/lib/mix/tasks/ansible.setup.ex index 425a5be0..9ca9f838 100644 --- a/lib/mix/tasks/ansible.setup.ex +++ b/lib/mix/tasks/ansible.setup.ex @@ -75,6 +75,12 @@ defmodule Mix.Tasks.Ansible.Setup do ## Options - `directory` - Directory containing ansible playbooks (default: ./deploys/ansible) + - `provider` - Cloud provider whose inventory the directory was built with + (default: `DeployEx.Config.cloud_provider/0`). Only affects which inventory + filename is checked for before setup runs — mismatch raises rather than + silently running against the wrong (or a stale) inventory. `--instance-id` + and `--git-branch` targeting stay AWS-only regardless of this flag (they + resolve through `DeployEx.AwsMachine`/`DeployEx.QaNode` directly). - `parallel` - Maximum number of concurrent setup operations (default: 4) - `only` - Only setup specified apps (can be used multiple times) - `except` - Skip setup for specified apps (can be used multiple times) @@ -117,7 +123,9 @@ defmodule Mix.Tasks.Ansible.Setup do targets = resolve_targets(instance_ids, git_branch, opts) ansible_args = ansible_args ++ build_limit_args(targets.patterns) - DeployExHelpers.check_file_exists!(Path.join(opts[:directory], "aws_ec2.yaml")) + provider = resolve_provider(opts) + + DeployExHelpers.check_file_exists!(Path.join(opts[:directory], inventory_filename(provider))) DeployEx.TUI.setup_no_tui(opts) @@ -320,6 +328,7 @@ defmodule Mix.Tasks.Ansible.Setup do aliases: [f: :force, q: :quit, d: :directory, i: :instance_id, b: :git_branch], switches: [ directory: :string, + provider: :string, only: :keep, except: :keep, force: :boolean, @@ -335,4 +344,28 @@ defmodule Mix.Tasks.Ansible.Setup do opts end + + # Matches the flag against the registered providers rather than calling to_existing_atom/1 — + # see terraform.build.ex's identical resolve_provider/1 for the rationale. + defp resolve_provider(opts) do + case opts[:provider] do + nil -> + DeployEx.Config.cloud_provider() + + name -> + known = DeployEx.Cloud.providers() + + case Enum.find(known, &(to_string(&1) === name)) do + nil -> Mix.raise("unknown provider #{inspect(name)}, expected one of #{inspect(known)}") + provider -> provider + end + end + end + + defp inventory_filename(provider) do + case DeployEx.Cloud.inventory(provider) do + {:ok, %{filename: filename}} -> filename + {:error, error} -> Mix.raise(to_string(error)) + end + end end diff --git a/lib/mix/tasks/deploy_ex.release.ex b/lib/mix/tasks/deploy_ex.release.ex index b1fd8585..9f0486cc 100644 --- a/lib/mix/tasks/deploy_ex.release.ex +++ b/lib/mix/tasks/deploy_ex.release.ex @@ -218,8 +218,27 @@ defmodule Mix.Tasks.DeployEx.Release do |> DeployEx.Utils.reduce_status_tuples end + # A release name is not required to match an umbrella app directory — a release bundles + # applications and is commonly named for its ROLE (server, ingestion, migrate) rather than + # for any one app. app_path/1 looks the name up in apps_paths() and returns nil for those, + # which reached Path.join/2 and killed the whole build with "no function clause matching in + # IO.chardata_to_string/1" — MEASURED against an umbrella whose six releases are all + # role-named. + # + # With no single app directory there is nothing to search for assets in, so the asset steps + # are skipped. A release whose name DOES match an app still resolves and builds as before. defp run_app_type_pre_release(:phoenix, candidate) do - app_path = DeployEx.ProjectContext.app_path(candidate.app_name) + case DeployEx.ProjectContext.app_path(candidate.app_name) do + nil -> :ok + app_path -> run_phoenix_pre_release(app_path, candidate) + end + end + + defp run_app_type_pre_release(:normal, _candidate) do + nil + end + + defp run_phoenix_pre_release(app_path, candidate) do package_json_path = Path.join(app_path, "assets/package.json") has_package_lock? = File.exists?(package_json_path) @@ -244,13 +263,17 @@ defmodule Mix.Tasks.DeployEx.Release do :ok end - defp run_app_type_pre_release(:normal, _candidate) do - nil + defp run_phoenix_asset_pipeline(app_name) do + case DeployEx.ProjectContext.app_path(app_name) do + nil -> :ok + app_path -> run_phoenix_asset_pipeline(app_name, app_path) + end end - defp run_phoenix_asset_pipeline(app_name) do + # Same nil case as run_app_type_pre_release/2: a role-named release has no app directory, so + # Path.join/2 below would crash on nil. + defp run_phoenix_asset_pipeline(app_name, app_path) do app_name_atom = String.to_atom(app_name) - app_path = DeployEx.ProjectContext.app_path(app_name) with {:ok, js_files} <- app_path |> Path.join("./assets/js") |> File.ls do if Enum.any?(js_files) do diff --git a/lib/mix/tasks/deploy_ex.ssh.authorize.ex b/lib/mix/tasks/deploy_ex.ssh.authorize.ex index 24afd466..428198d3 100644 --- a/lib/mix/tasks/deploy_ex.ssh.authorize.ex +++ b/lib/mix/tasks/deploy_ex.ssh.authorize.ex @@ -3,7 +3,8 @@ defmodule Mix.Tasks.DeployEx.Ssh.Authorize do @shortdoc "Add or remove ssh authorization to the internal network for specific IPs" @moduledoc """ - Manages SSH authorization by adding or removing IP addresses from the AWS security group whitelist. + Manages SSH authorization by adding or removing IP addresses from the active cloud provider's + security group / network security group whitelist. This task allows you to: 1. Add your current IP address to the security group whitelist @@ -33,8 +34,8 @@ defmodule Mix.Tasks.DeployEx.Ssh.Authorize do - `quiet` (`-q`) - Suppress output messages - `remove` (`-r`) - Remove authorization instead of adding it - `ip` - Specific IP address to whitelist (defaults to current device's IP) - - `region` - AWS region (defaults to configured region) - - `security_group_id` - AWS security group ID to use (bypasses auto-detection) + - `region` - AWS region (defaults to configured region; ignored on OCI, which reads its own `:oci` config) + - `security_group_id` - Security group ID (AWS) or network security group OCID (OCI) to use (bypasses auto-detection) """ def run(args) do @@ -43,8 +44,9 @@ defmodule Mix.Tasks.DeployEx.Ssh.Authorize do opts = parse_args(args) with :ok <- DeployExHelpers.check_valid_project(), - {:ok, security_group_id} <- DeployEx.AwsSecurityGroup.find_security_group_id(region: opts[:region], security_group_id: opts[:security_group_id]), - :ok <- add_or_remove_whitelist(opts, security_group_id) do + {:ok, security} <- DeployEx.Cloud.capability(:security), + {:ok, security_group_id} <- security.find_group(region: opts[:region], security_group_id: opts[:security_group_id]), + :ok <- add_or_remove_whitelist(security, opts, security_group_id) do :ok else {:error, e} -> Mix.raise(to_string(e)) @@ -67,30 +69,32 @@ defmodule Mix.Tasks.DeployEx.Ssh.Authorize do opts end - defp add_or_remove_whitelist(opts, security_group_id) do + defp add_or_remove_whitelist(security, opts, security_group_id) do if opts[:remove] do - revoke_whitelist(opts, security_group_id) + revoke_whitelist(security, opts, security_group_id) else - whitelist(opts, security_group_id) + whitelist(security, opts, security_group_id) end end - defp revoke_whitelist(opts, security_group_id) do + defp revoke_whitelist(security, opts, security_group_id) do with {:ok, current_ip} <- get_arg_id_or_current_ip(opts) do Mix.shell().info(IO.ANSI.format([:yellow, "Deauthorizing current device #{current_ip} from security group #{security_group_id}", :reset])) - DeployEx.AwsIpWhitelister.deauthorize(security_group_id, current_ip) + security.revoke_ingress(security_group_id, to_cidr(current_ip), opts) end end - defp whitelist(opts, security_group_id) do + defp whitelist(security, opts, security_group_id) do with {:ok, current_ip} <- get_arg_id_or_current_ip(opts) do Mix.shell().info(IO.ANSI.format([:yellow, "Authorizing current device #{current_ip} in security group #{security_group_id}", :reset])) - DeployEx.AwsIpWhitelister.authorize(security_group_id, current_ip) + security.authorize_ingress(security_group_id, to_cidr(current_ip), opts) end end + defp to_cidr(ip_address), do: "#{ip_address}/32" + defp get_arg_id_or_current_ip(opts) do if opts[:ip] do {:ok, opts[:ip]} diff --git a/lib/mix/tasks/deploy_ex.upload.ex b/lib/mix/tasks/deploy_ex.upload.ex index 9e987090..2e7c948b 100644 --- a/lib/mix/tasks/deploy_ex.upload.ex +++ b/lib/mix/tasks/deploy_ex.upload.ex @@ -22,6 +22,7 @@ defmodule Mix.Tasks.DeployEx.Upload do - `aws-region` - Region for aws (default: `#{Config.aws_region()}`) - `aws-bucket` - Region for aws (default: `#{Config.aws_release_bucket()}`) - `qa` - Marks the release as a QA release + - `upload-timeout` - Per-release upload ceiling in ms (default: 600000) """ def run(args) do @@ -34,6 +35,7 @@ defmodule Mix.Tasks.DeployEx.Upload do |> Keyword.put_new(:aws_release_bucket, Config.aws_release_bucket()) |> Keyword.put_new(:aws_region, Config.aws_region()) |> Keyword.put_new(:parallel, @max_upload_concurrency) + |> Keyword.put_new(:upload_timeout, :timer.minutes(10)) |> then(&Keyword.put(&1, :qa_release, qa_release?(&1))) |> Keyword.put_new(:branch, git_branch_name()) @@ -68,6 +70,7 @@ defmodule Mix.Tasks.DeployEx.Upload do aws_region: :string, aws_release_bucket: :string, parallel: :integer, + upload_timeout: :integer, qa: :boolean ] ) @@ -129,11 +132,17 @@ defmodule Mix.Tasks.DeployEx.Upload do ])) end + # A 60s per-release ceiling made seven parallel ~30MB uploads race a GitHub runner's + # bandwidth to OCI: six landed and the seventh's timeout took the whole stream down with + # `Task.Supervised.stream(60000) ** (EXIT) time out` (measured on a real prod build), so + # the run failed AFTER most artifacts were already in the bucket — a partial release set + # with a red build. Timeout is now per release, generous, and overridable. defp upload_releases(release_candidates, opts) do release_candidates |> Task.async_stream(&upload_release(&1, opts), max_concurrency: opts[:parallel], - timeout: :timer.seconds(60) + timeout: opts[:upload_timeout], + on_timeout: :kill_task ) |> DeployEx.Utils.reduce_task_status_tuples end diff --git a/lib/mix/tasks/terraform.apply.ex b/lib/mix/tasks/terraform.apply.ex index 86bf6a7a..dbdd4513 100644 --- a/lib/mix/tasks/terraform.apply.ex +++ b/lib/mix/tasks/terraform.apply.ex @@ -3,9 +3,9 @@ defmodule Mix.Tasks.Terraform.Apply do @terraform_default_path DeployEx.Config.terraform_folder_path() - @shortdoc "Applies terraform changes to provision AWS infrastructure" + @shortdoc "Applies terraform changes to provision infrastructure" @moduledoc """ - Applies terraform changes to provision or update AWS infrastructure. + Applies terraform changes to provision or update infrastructure. ## Example ```bash diff --git a/lib/mix/tasks/terraform.build.ex b/lib/mix/tasks/terraform.build.ex index 14451c3b..aed99f25 100644 --- a/lib/mix/tasks/terraform.build.ex +++ b/lib/mix/tasks/terraform.build.ex @@ -17,6 +17,9 @@ defmodule Mix.Tasks.Terraform.Build do - `aws-bucket` - Region for aws (default: `#{@default_aws_release_bucket}`) - `aws-log-bucket` - Region for aws (default: `#{@default_aws_log_bucket}`) - `env` - Environment for terraform (default: `Mix.env()`) + - `render-dir` - Render the terraform files into this directory instead of + `directory`, skipping the tool preflight and `terraform init`. Used by the + render diff harness to compare output across revisions. - `quiet` - Supress output - `force` - Force create files without asking - `verbose` - Log extra details about the process @@ -26,26 +29,35 @@ defmodule Mix.Tasks.Terraform.Build do def run(args) do opts = args |> parse_args - |> Keyword.put_new(:directory, @terraform_default_path) - |> Keyword.put_new(:aws_region, @default_aws_region) - |> Keyword.put_new(:aws_release_bucket, @default_aws_release_bucket) + |> put_render_dir_paths() + # Runtime Config calls, not the module attributes: an attribute captures the value when + # the DEP compiles, so a consuming project's config change silently renders stale + # defaults until a forced deps.compile. Mix.env() has the same problem for :env — it is + # always :dev in a local shell regardless of `config :deploy_ex, :env`. + |> Keyword.put_new(:directory, DeployEx.Config.terraform_folder_path()) + |> Keyword.put_new(:aws_region, DeployEx.Config.aws_region()) + |> Keyword.put_new(:aws_release_bucket, DeployEx.Config.aws_release_bucket()) |> Keyword.put_new(:aws_log_bucket, DeployEx.Config.aws_log_bucket()) |> Keyword.put_new(:aws_release_state_bucket, DeployEx.Config.aws_release_state_bucket()) |> Keyword.put_new(:aws_release_state_lock_table, DeployEx.Config.aws_release_state_lock_table()) - |> Keyword.put_new(:env, Mix.env()) + |> Keyword.put_new(:env, DeployEx.Config.env()) no_logging = opts[:no_logging] || opts[:no_loki] || false opts = Keyword.put(opts, :no_logging, no_logging) + # --provider overrides the configured cloud_provider. Resolved once so the seed and the + # render can never disagree about which file set they are working from. + provider = resolve_provider(opts) + with :ok <- DeployExHelpers.check_valid_project(), - :ok <- DeployEx.ToolInstaller.ensure_installed(:terraform), + :ok <- ensure_terraform_installed(opts), {:ok, releases} <- DeployExHelpers.fetch_mix_releases(), - :ok <- ensure_terraform_directory_exists(opts[:directory]) do + :ok <- ensure_terraform_directory_exists(opts[:directory], provider, opts) do random_bytes = 6 |> :crypto.strong_rand_bytes |> Base.encode32(padding: false) terraform_app_releases_variables = releases |> Keyword.keys - |> Enum.map_join(",\n\n", &(&1 |> to_string |> generate_terraform_release_variables())) + |> Enum.map_join(",\n\n", &DeployEx.TerraformVariables.generate_terraform_release_variables(to_string(&1), provider)) params = %{ directory: opts[:directory], @@ -55,7 +67,7 @@ defmodule Mix.Tasks.Terraform.Build do aws_release_bucket: opts[:aws_release_bucket], use_db: !opts[:no_database], - db_password: !opts[:no_database] && generate_db_password(), + db_password: !opts[:no_database] && (opts[:db_password] || generate_db_password()), release_bucket_name: opts[:aws_release_bucket], logging_bucket_name: opts[:aws_log_bucket], @@ -65,7 +77,14 @@ defmodule Mix.Tasks.Terraform.Build do terraform_backend: DeployEx.Config.terraform_backend(), - pem_app_name: "#{DeployExHelpers.kebab_project_name()}-#{random_bytes}", + oci_region: DeployEx.Config.oci_setting(:region), + oci_namespace: DeployEx.Config.oci_setting(:namespace), + oci_state_bucket: DeployEx.Config.oci_setting(:release_state_bucket), + oci_state_key: DeployEx.Config.oci_release_state_key(), + oci_state_profile: DeployEx.Config.oci_setting(:state_profile), + oci_vcn_cidr: DeployEx.Config.oci_setting(:vcn_cidr) || "10.20.0.0/16", + + pem_app_name: opts[:pem_app_name] || existing_pem_app_name(opts[:directory]) || "#{DeployExHelpers.kebab_project_name()}-#{random_bytes}", app_name: DeployExHelpers.underscored_project_name(), kebab_app_name: DeployExHelpers.kebab_project_name(), @@ -78,29 +97,73 @@ defmodule Mix.Tasks.Terraform.Build do terraform_app_releases_variables: terraform_app_releases_variables, terraform_release_variables: terraform_app_releases_variables, - terraform_redis_variables: terraform_redis_variables(opts), - terraform_sentry_variables: terraform_sentry_variables(opts), - terraform_grafana_variables: terraform_grafana_variables(opts), - terraform_loki_variables: terraform_loki_variables(opts), - terraform_prometheus_variables: terraform_prometheus_variables(opts), + terraform_redis_variables: DeployEx.TerraformVariables.terraform_redis_variables(opts, provider), + terraform_clickhouse_variables: DeployEx.TerraformVariables.terraform_clickhouse_variables(opts, provider), + terraform_rabbitmq_variables: DeployEx.TerraformVariables.terraform_rabbitmq_variables(opts, provider), + terraform_sentry_variables: DeployEx.TerraformVariables.terraform_sentry_variables(opts, provider), + terraform_grafana_variables: DeployEx.TerraformVariables.terraform_grafana_variables(opts, provider), + terraform_loki_variables: DeployEx.TerraformVariables.terraform_loki_variables(opts, provider), + terraform_prometheus_variables: DeployEx.TerraformVariables.terraform_prometheus_variables(opts, provider), } - write_terraform_template_files(params, opts) + write_terraform_template_files(params, opts, provider) - DeployEx.Terraform.run_command_with_input( - "init", - params[:directory] - ) + if opts[:render_dir] do + :ok + else + DeployEx.Terraform.run_command_with_input( + "init", + params[:directory] + ) + end else {:error, e} -> Mix.raise(to_string(e)) end end + # Matches the flag against the registered providers rather than calling to_existing_atom/1. + # That function depends on whether something has already interned the atom, which is not + # guaranteed here — DeployEx.Cloud's registry is a compile-time attribute and the module may + # not be loaded yet when this task runs. Matching known keys also gives a usable error. + defp resolve_provider(opts) do + case opts[:provider] do + nil -> + DeployEx.Config.cloud_provider() + + name -> + known = DeployEx.Cloud.providers() + + case Enum.find(known, &(to_string(&1) === name)) do + nil -> Mix.raise("unknown provider #{inspect(name)}, expected one of #{inspect(known)}") + provider -> provider + end + end + end + + defp put_render_dir_paths(opts) do + case opts[:render_dir] do + nil -> opts + render_dir -> Keyword.put(opts, :directory, render_dir) + end + end + + defp ensure_terraform_installed(opts) do + if opts[:render_dir] do + :ok + else + DeployEx.ToolInstaller.ensure_installed(:terraform) + end + end + defp parse_args(args) do {opts, _extra_args} = OptionParser.parse!(args, aliases: [f: :force, q: :quit, d: :directory, v: :verbose], switches: [ directory: :string, + render_dir: :string, + provider: :string, + pem_app_name: :string, + db_password: :string, force: :boolean, quiet: :boolean, verbose: :boolean, @@ -112,6 +175,8 @@ defmodule Mix.Tasks.Terraform.Build do no_sentry: :boolean, no_grafana: :boolean, no_redis: :boolean, + clickhouse: :boolean, + rabbitmq: :boolean, no_prometheus: :boolean ] ) @@ -119,175 +184,86 @@ defmodule Mix.Tasks.Terraform.Build do opts end - defp ensure_terraform_directory_exists(directory) do - if File.exists?(directory) do - :ok - else - Mix.shell().info([:green, "* copying terraform into ", :reset, directory]) - - File.mkdir_p!(directory) - - "terraform" - |> DeployExHelpers.priv_folder() - |> File.cp_r!(directory) - - directory - |> Path.join("**/*.eex") - |> Path.wildcard - |> Enum.map(&File.rm!/1) - - :ok + # Seeds only the active provider's file set. A whole-tree copy would put every provider's + # templates into every user's ./deploys — an :aws user would find providers/oci/*.tf sitting + # in their terraform root, where tofu would try to load them. + # + # Syncs on EVERY build, not only when the directory is first created: a first-seed-only copy + # left existing trees permanently stale for static (non-.eex) files — new provider files + # never arrived and module updates never landed, while the .eex renders around them DID + # update, producing a tree that references module variables its stale modules lack. + defp ensure_terraform_directory_exists(directory, provider, opts) do + if !File.exists?(directory) do + Mix.shell().info([:green, "* copying ", to_string(provider), " terraform into ", :reset, directory]) end - end - - defp generate_terraform_release_variables(release_name) do - String.trim_trailing(""" - #{release_name} = { - name = "#{DeployEx.Utils.upper_title_case(release_name)}" - tags = { - Vendor = "Self" - Type = "Self Made" - } - - # Autoscaling Configuration (optional) - # Uncomment and configure to enable AWS Auto Scaling Groups - # autoscaling = { - # enable = true - # min_size = 1 - # max_size = 5 - # desired_capacity = 2 - # cpu_target_percent = 60 - # } - } - """, "\n") - end - defp terraform_redis_variables(opts) do - if opts[:no_redis] do - "" - else - """ - #{DeployExHelpers.underscored_project_name()}_redis = { - name = "#{DeployExHelpers.title_case_project_name()} Redis" - private_ip = "10.0.1.60" - enable_ebs = true - - # This is a suggestion for instance + priv_path = DeployExHelpers.priv_folder("terraform") - instance_type = "r7g.medium" + with {:ok, files} <- DeployEx.Cloud.PrivFileSet.files(provider, priv_path) do + File.mkdir_p!(directory) - instance_ebs_secondary_size = 16 + files + |> Enum.reject(fn {source, _dest} -> String.ends_with?(source, ".eex") end) + |> Enum.each(fn {source, dest} -> sync_priv_file(priv_path, directory, source, dest, opts) end) - tags = { - Vendor = "Redis" - Type = "Database" - DatabaseKey = "#{DeployExHelpers.underscored_project_name()}_redis" - } - }, - """ + :ok end end - defp terraform_sentry_variables(opts) do - if opts[:no_sentry] do - "" - else - """ - sentry = { - name = "Sentry Monitoring" - tags = { - Vendor = "Sentry" - Type = "Monitoring" - } - }, - """ - end - end + # Identical contents short-circuit silently — write_file would route them into the + # "overwrite declined" warning, which is false for an unchanged file. + defp sync_priv_file(priv_path, directory, source, dest, opts) do + contents = File.read!(Path.join(priv_path, source)) + target = Path.join(directory, dest) - defp terraform_loki_variables(opts) do - if opts[:no_logging] do - "" - else - """ - loki_aggregator = { - name = "Grafana Loki Logs" - instance_type = "t3.micro" - private_ip = "10.0.1.50" - - enable_ebs = true - instance_ebs_secondary_size = 8 - - tags = { - Vendor = "Grafana" - Type = "Monitoring" - MonitoringKey = "loki_logger" - } - }, - """ - end - end - - defp terraform_grafana_variables(opts) do - if opts[:no_grafana] do - "" + if File.exists?(target) and File.read!(target) === contents do + :ok else - """ - grafana_ui = { - name = "Grafana UI" - enable_ebs = true - enable_eip = true - instance_ebs_secondary_size = 8 - - tags = { - Vendor = "Grafana" - Type = "Monitoring" - MonitoringKey = "grafana_ui" - } - }, - """ + target |> Path.dirname() |> File.mkdir_p!() + DeployExHelpers.write_file(target, contents, Keyword.put(opts, :message, "* syncing #{target}")) end end - defp terraform_prometheus_variables(opts) do - if opts[:no_prometheus] do - "" - else - """ - prometheus_db = { - name = "Prometheus Metrics Database" - instance_type = "t3.micro" - enable_ebs = true - instance_ebs_secondary_size = 16 - private_ip = "10.0.1.40" - - tags = { - Vendor = "Grafana" - Type = "Monitoring" - MonitoringKey = "prometheus_db" - } - }, - """ - end + # A fresh random pem name every build makes each rebuild+apply REPLACE the key file under a + # new name while ansible.cfg still points at the previous one — every node goes UNREACHABLE + # with "no such identity" (measured). Reuse whatever name the existing render already + # carries; the random suffix is only for the first build. + defp existing_pem_app_name(directory) do + directory + |> Path.join("*.tf") + |> Path.wildcard() + |> Enum.find_value(fn file -> + case Regex.run(~r/"([A-Za-z0-9-]+)-key-pair\.pem"/, File.read!(file)) do + [_full, pem_app_name] -> pem_app_name + _no_match -> nil + end + end) end + # The AWS block advertises autoscaling, which has no OCI implementation yet — leaving that + # comment in an OCI tree would document a knob that silently does nothing. defp generate_db_password do "SuperSecretPassword#{Enum.random(111_111..999_999)}" end - defp write_terraform_template_files(params, opts) do + # Renders the active provider's .eex files. Non-AWS templates flatten on the way out — + # providers/oci/variables.tf.eex becomes variables.tf at the terraform root — because tofu + # only loads root-level .tf and runs in the configured terraform folder. + defp write_terraform_template_files(params, opts, provider) do terraform_path = DeployExHelpers.priv_folder("terraform") - terraform_path - |> Path.join("*.eex") - |> Path.wildcard - |> Enum.map(fn template_file -> - template = EEx.eval_file(template_file, assigns: params) - - template_file - |> String.replace(terraform_path, "") - |> String.replace(".eex", "") - |> then(&Path.join(params[:directory], &1)) - |> DeployExHelpers.write_file(template, opts) + with {:ok, files} <- DeployEx.Cloud.PrivFileSet.files(provider, terraform_path) do + files + |> Enum.filter(fn {source, _dest} -> String.ends_with?(source, ".eex") end) + |> Enum.each(fn {source, dest} -> + template = EEx.eval_file(Path.join(terraform_path, source), assigns: params) + target = Path.join(params[:directory], String.replace(dest, ".eex", "")) + + target |> Path.dirname() |> File.mkdir_p!() + DeployExHelpers.write_file(target, template, opts) end) + + :ok + end end end diff --git a/lib/mix/tasks/terraform.create_ebs_snapshot.ex b/lib/mix/tasks/terraform.create_ebs_snapshot.ex index d0c6e54a..92f34e36 100644 --- a/lib/mix/tasks/terraform.create_ebs_snapshot.ex +++ b/lib/mix/tasks/terraform.create_ebs_snapshot.ex @@ -113,30 +113,58 @@ defmodule Mix.Tasks.Terraform.CreateEbsSnapshot do end end - defp find_volumes_for_instances(region, instances) do + def find_volumes_for_instances(region, instances, opts \\ []) do instance_ids = Enum.map(instances, & &1["instanceId"]) filters = [ {"attachment.instance-id", instance_ids} ] - ExAws.EC2.describe_volumes(filters: filters) - |> ExAws.request(region: region) + with {:ok, volumes} <- fetch_all_volumes(region, [filters: filters], opts, %{instance_ids: instance_ids}) do + case volumes do + [] -> + {:error, ErrorMessage.not_found( + "No volumes found for instances", + %{instance_ids: instance_ids} + )} + + volumes -> + Mix.shell().info([ + :green, "Found ", :bright, "#{length(volumes)}", :reset, :green, + " volume(s) across all instances", :reset + ]) + {:ok, volumes} + end + end + end + + # DescribeVolumes caps a response and signals more via `nextToken`. A single request therefore + # truncates silently -- it returns `{:ok, partial}`, not an error -- which here determines which + # volumes get a snapshot while the task prints a confident "Found N volume(s)" from the + # truncated count. `AwsMachine.fetch_instances/2` already paginates the same EC2 Query API for + # the same reason. + defp fetch_all_volumes(region, base_opts, opts, details) do + fetch_volumes_page(region, base_opts, opts, details, nil, []) + end + + defp fetch_volumes_page(region, base_opts, opts, details, next_token, acc) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + + base_opts + |> maybe_put_ec2_opt(:max_results, opts[:max_results]) + |> maybe_put_ec2_opt(:next_token, next_token) + |> ExAws.EC2.describe_volumes() + |> request_fn.(region: region) |> case do {:ok, %{body: body}} -> case parse_volumes_response(body) do - {:ok, []} -> - {:error, ErrorMessage.not_found( - "No volumes found for instances", - %{instance_ids: instance_ids} - )} - {:ok, volumes} -> - Mix.shell().info([ - :green, "Found ", :bright, "#{length(volumes)}", :reset, :green, - " volume(s) across all instances", :reset - ]) - {:ok, volumes} + accumulated = acc ++ volumes + + case ec2_next_token(body, "DescribeVolumesResponse") do + nil -> {:ok, accumulated} + token -> fetch_volumes_page(region, base_opts, opts, details, token, accumulated) + end {:error, _} = error -> error end @@ -144,17 +172,29 @@ defmodule Mix.Tasks.Terraform.CreateEbsSnapshot do {:error, {:http_error, status_code, %{body: body}}} -> {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ "Error fetching volumes from AWS", - %{error_body: body, instance_ids: instance_ids} + Map.put(details, :error_body, body) ])} {:error, error} -> {:error, ErrorMessage.bad_request( "Failed to describe volumes", - %{error: error, instance_ids: instance_ids} + Map.put(details, :error, error) )} end end + # AWS signals more pages via `nextToken`, absent once the last page is reached -- not via a + # truthy/falsy flag, so presence (not truthiness) is what terminates the loop. + defp ec2_next_token(body, response_key) do + case XmlToMap.naive_map(body) do + %{^response_key => %{"nextToken" => token}} when is_binary(token) and token !== "" -> token + _no_more_pages -> nil + end + end + + defp maybe_put_ec2_opt(opts, _key, nil), do: opts + defp maybe_put_ec2_opt(opts, key, value), do: Keyword.put(opts, key, value) + defp filter_volumes_by_type(volumes, opts) do include_root = opts[:include_root] || false diff --git a/lib/mix/tasks/terraform.delete_ebs_snapshot.ex b/lib/mix/tasks/terraform.delete_ebs_snapshot.ex index 2cf937e3..45c38955 100644 --- a/lib/mix/tasks/terraform.delete_ebs_snapshot.ex +++ b/lib/mix/tasks/terraform.delete_ebs_snapshot.ex @@ -123,38 +123,17 @@ defmodule Mix.Tasks.Terraform.DeleteEbsSnapshot do end end - defp get_snapshots_by_ids(region, snapshot_ids) do - ExAws.EC2.describe_snapshots(snapshot_ids: snapshot_ids) - |> ExAws.request(region: region) - |> case do - {:ok, %{body: body}} -> - case parse_snapshots_response(body) do - {:ok, snapshots} -> - snapshot_data = Enum.map(snapshots, fn snapshot -> - %{ - snapshot_id: snapshot["snapshotId"], - volume_id: snapshot["volumeId"], - description: snapshot["description"], - start_time: snapshot["startTime"] - } - end) - {:ok, snapshot_data} - - {:error, _} = error -> error - end - - {:error, {:http_error, status_code, %{body: body}}} -> - {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ - "Error fetching snapshots from AWS", - %{error_body: body, snapshot_ids: snapshot_ids} - ])} - - {:error, error} -> - {:error, ErrorMessage.bad_request( - "Failed to describe snapshots", - %{error: error, snapshot_ids: snapshot_ids} - )} - end + # AWS rejects DescribeSnapshots calls that combine SnapshotIds with MaxResults + # ("InvalidParameterCombination: The parameter snapshotSet cannot be used with the parameter + # maxResults") -- confirmed against a live account. :max_results is dropped here so an opts + # value meant for the filter-based paginators can never be forwarded into an id lookup and + # blow up the request; a nextToken is still followed if AWS ever sends one. + def get_snapshots_by_ids(region, snapshot_ids, opts \\ []) do + opts = Keyword.delete(opts, :max_results) + + with {:ok, snapshots} <- fetch_all_snapshots(region, [snapshot_ids: snapshot_ids], opts, %{snapshot_ids: snapshot_ids}) do + {:ok, Enum.map(snapshots, &snapshot_summary/1)} + end end defp find_instances_by_ips(region, instance_ips) do @@ -177,26 +156,79 @@ defmodule Mix.Tasks.Terraform.DeleteEbsSnapshot do end end - defp find_volumes_for_instances(region, instances) do + def find_volumes_for_instances(region, instances, opts \\ []) do instance_ids = Enum.map(instances, & &1["instanceId"]) - + filters = [ {"attachment.instance-id", instance_ids} ] - ExAws.EC2.describe_volumes(filters: filters) - |> ExAws.request(region: region) + with {:ok, volumes} <- fetch_all_volumes(region, [filters: filters], opts, %{instance_ids: instance_ids}) do + case volumes do + [] -> + {:error, ErrorMessage.not_found( + "No volumes found for instances", + %{instance_ids: instance_ids} + )} + + volumes -> + {:ok, volumes} + end + end + end + + def find_snapshots_for_volumes(region, volumes, opts \\ []) do + volume_ids = Enum.map(volumes, & &1["volumeId"]) + + filters = [ + {"volume-id", volume_ids} + ] + + # `owner: ["self"]` is what makes this query terminate. Snapshots are the one EC2 resource + # with a public/shared universe, and AWS applies MaxResults to the UNSCOPED scan before + # applying the volume-id filter — so unscoped, page one comes back empty WITH a nextToken and + # the pagination loop walks every public snapshot in the region. + # + # It has to be the Owner REQUEST PARAMETER, not an `owner-id` filter: MEASURED, the filter + # form silently matches zero snapshots (self is not a valid filter value), which looks like a + # fast fix and is actually a query that can never find anything. + request_opts = [owner: ["self"], filters: filters] + + with {:ok, snapshots} <- fetch_all_snapshots(region, request_opts, opts, %{volume_ids: volume_ids}) do + filtered_snapshots = snapshots + |> filter_snapshots_by_age(opts[:max_age_days]) + |> Enum.map(&snapshot_summary/1) + + {:ok, filtered_snapshots} + end + end + + # DescribeVolumes/DescribeSnapshots cap a response and signal more via `nextToken`. A single + # request therefore truncates silently -- it returns `{:ok, partial}`, not an error -- which + # here feeds destructive snapshot-deletion selection directly. `AwsMachine.fetch_instances/2` + # already paginates the same EC2 Query API for the same reason. + defp fetch_all_volumes(region, base_opts, opts, details) do + fetch_volumes_page(region, base_opts, opts, details, nil, []) + end + + defp fetch_volumes_page(region, base_opts, opts, details, next_token, acc) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + + base_opts + |> maybe_put_ec2_opt(:max_results, opts[:max_results]) + |> maybe_put_ec2_opt(:next_token, next_token) + |> ExAws.EC2.describe_volumes() + |> request_fn.(region: region) |> case do {:ok, %{body: body}} -> case parse_volumes_response(body) do - {:ok, []} -> - {:error, ErrorMessage.not_found( - "No volumes found for instances", - %{instance_ids: instance_ids} - )} - {:ok, volumes} -> - {:ok, volumes} + accumulated = acc ++ volumes + + case ec2_next_token(body, "DescribeVolumesResponse") do + nil -> {:ok, accumulated} + token -> fetch_volumes_page(region, base_opts, opts, details, token, accumulated) + end {:error, _} = error -> error end @@ -204,42 +236,39 @@ defmodule Mix.Tasks.Terraform.DeleteEbsSnapshot do {:error, {:http_error, status_code, %{body: body}}} -> {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ "Error fetching volumes from AWS", - %{error_body: body, instance_ids: instance_ids} + Map.put(details, :error_body, body) ])} {:error, error} -> {:error, ErrorMessage.bad_request( "Failed to describe volumes", - %{error: error, instance_ids: instance_ids} + Map.put(details, :error, error) )} end end - defp find_snapshots_for_volumes(region, volumes, opts) do - volume_ids = Enum.map(volumes, & &1["volumeId"]) - - filters = [ - {"volume-id", volume_ids} - ] + defp fetch_all_snapshots(region, base_opts, opts, details) do + fetch_snapshots_page(region, base_opts, opts, details, nil, []) + end - ExAws.EC2.describe_snapshots(filters: filters) - |> ExAws.request(region: region) + defp fetch_snapshots_page(region, base_opts, opts, details, next_token, acc) do + request_fn = opts[:request_fn] || (&ExAws.request/2) + + base_opts + |> maybe_put_ec2_opt(:max_results, opts[:max_results]) + |> maybe_put_ec2_opt(:next_token, next_token) + |> ExAws.EC2.describe_snapshots() + |> request_fn.(region: region) |> case do {:ok, %{body: body}} -> case parse_snapshots_response(body) do {:ok, snapshots} -> - filtered_snapshots = snapshots - |> filter_snapshots_by_age(opts[:max_age_days]) - |> Enum.map(fn snapshot -> - %{ - snapshot_id: snapshot["snapshotId"], - volume_id: snapshot["volumeId"], - description: snapshot["description"], - start_time: snapshot["startTime"] - } - end) - - {:ok, filtered_snapshots} + accumulated = acc ++ snapshots + + case ec2_next_token(body, "DescribeSnapshotsResponse") do + nil -> {:ok, accumulated} + token -> fetch_snapshots_page(region, base_opts, opts, details, token, accumulated) + end {:error, _} = error -> error end @@ -247,17 +276,38 @@ defmodule Mix.Tasks.Terraform.DeleteEbsSnapshot do {:error, {:http_error, status_code, %{body: body}}} -> {:error, apply(ErrorMessage, ErrorMessage.http_code_reason_atom(status_code), [ "Error fetching snapshots from AWS", - %{error_body: body, volume_ids: volume_ids} + Map.put(details, :error_body, body) ])} {:error, error} -> {:error, ErrorMessage.bad_request( "Failed to describe snapshots", - %{error: error, volume_ids: volume_ids} + Map.put(details, :error, error) )} end end + # AWS signals more pages via `nextToken`, absent once the last page is reached -- not via a + # truthy/falsy flag, so presence (not truthiness) is what terminates the loop. + defp ec2_next_token(body, response_key) do + case XmlToMap.naive_map(body) do + %{^response_key => %{"nextToken" => token}} when is_binary(token) and token !== "" -> token + _no_more_pages -> nil + end + end + + defp maybe_put_ec2_opt(opts, _key, nil), do: opts + defp maybe_put_ec2_opt(opts, key, value), do: Keyword.put(opts, key, value) + + defp snapshot_summary(snapshot) do + %{ + snapshot_id: snapshot["snapshotId"], + volume_id: snapshot["volumeId"], + description: snapshot["description"], + start_time: snapshot["startTime"] + } + end + defp filter_snapshots_by_age(snapshots, nil), do: snapshots defp filter_snapshots_by_age(snapshots, max_age_days) do cutoff_date = DateTime.utc_now() |> DateTime.add(-max_age_days, :day) diff --git a/lib/mix/tasks/terraform.drop.ex b/lib/mix/tasks/terraform.drop.ex index 8fa14502..3bbfba87 100644 --- a/lib/mix/tasks/terraform.drop.ex +++ b/lib/mix/tasks/terraform.drop.ex @@ -5,10 +5,10 @@ defmodule Mix.Tasks.Terraform.Drop do @shortdoc "Destroys all resources built by terraform" @moduledoc """ - Destroys all AWS infrastructure resources managed by Terraform. + Destroys all infrastructure resources managed by Terraform. This is a destructive operation that will tear down all provisioned resources - including EC2 instances, load balancers, security groups, and other infrastructure. + including compute instances, load balancers, security groups, and other infrastructure. ## Example ```bash @@ -31,7 +31,14 @@ defmodule Mix.Tasks.Terraform.Drop do cmd = "destroy #{DeployEx.Terraform.parse_args(args, :destroy)}" cmd = if opts[:auto_approve], do: "#{cmd} --auto-approve", else: cmd - DeployEx.Terraform.run_command_with_input(cmd, opts[:directory]) + # Mix does NOT fail a task based on run/1's return value — only a raise produces a + # non-zero exit. Returning the error tuple made a FAILED destroy exit 0. MEASURED: a + # destroy that left an object-storage bucket behind ("409-BucketNotEmpty") still + # reported success, so infrastructure you believe is gone is still there and billing. + case DeployEx.Terraform.run_command_with_input(cmd, opts[:directory]) do + :ok -> :ok + {:error, error} -> Mix.raise(to_string(error)) + end end end diff --git a/lib/mix/tasks/terraform.refresh.ex b/lib/mix/tasks/terraform.refresh.ex index c3f881fd..02db8a1a 100644 --- a/lib/mix/tasks/terraform.refresh.ex +++ b/lib/mix/tasks/terraform.refresh.ex @@ -23,10 +23,15 @@ defmodule Mix.Tasks.Terraform.Refresh do |> Keyword.put_new(:directory, @terraform_default_path) with :ok <- DeployExHelpers.check_valid_project() do - DeployEx.Terraform.run_command_with_input( - "refresh #{DeployEx.Terraform.parse_args(args, :refresh)}", - opts[:directory] - ) + # Mix ignores run/1's return value — only a raise sets a non-zero exit. See + # terraform.drop for the measured consequence of returning the tuple instead. + case DeployEx.Terraform.run_command_with_input( + "refresh #{DeployEx.Terraform.parse_args(args, :refresh)}", + opts[:directory] + ) do + :ok -> :ok + {:error, error} -> Mix.raise(to_string(error)) + end end end diff --git a/mix.exs b/mix.exs index d65b1167..4e5a6b1e 100644 --- a/mix.exs +++ b/mix.exs @@ -35,6 +35,7 @@ defmodule DeployEx.MixProject do [ {:jason, "~> 1.3"}, {:error_message, "~> 0.2"}, + {:nimble_options, "~> 1.0"}, {:ex_aws, "~> 2.3"}, {:ex_aws_s3, "~> 2.3"}, {:ex_aws_dynamo, "~> 4.2"}, diff --git a/priv/ansible/app_setup_playbook.yaml.eex b/priv/ansible/app_setup_playbook.yaml.eex index 03248cb7..c5af74c3 100644 --- a/priv/ansible/app_setup_playbook.yaml.eex +++ b/priv/ansible/app_setup_playbook.yaml.eex @@ -8,10 +8,11 @@ - beam_linux_tuning - log_cleanup - pip3 - - awscli - - ipv6 + <%= if @cloud_provider == :oci do %>- oci_cli + <% else %>- awscli + <% end %>- ipv6 <%= unless @no_prometheus do %>- prometheus_exporter <% end %> <%= unless @no_logging do %>- grafana_alloy <% end %> - - save_ami + <%= if @cloud_provider !== :oci do %>- save_ami<% end %> diff --git a/priv/ansible/providers/oci/README.md b/priv/ansible/providers/oci/README.md new file mode 100644 index 00000000..0b3a668f --- /dev/null +++ b/priv/ansible/providers/oci/README.md @@ -0,0 +1,25 @@ +# OCI ansible file set + +Selected by `DeployEx.Cloud.PrivFileSet` when `mix ansible.build --provider oci` (or +`config :deploy_ex, cloud_provider: :oci`) runs. Flattens onto the ansible root exactly +like the terraform provider set — `providers/oci/ansible.cfg.eex` becomes `ansible.cfg`, +`providers/oci/oci.yaml.eex` becomes `oci.yaml`. + +- `ansible.cfg.eex` — same shape as the root AWS config, except `remote_user = ubuntu` + (OCI's Ubuntu images have no `admin` user) and `inventory = ./oci.yaml`. No `[inventory]` + section: the static file parses through ansible-core's built-in `yaml` plugin, so nothing + needs enabling. +- `oci.yaml.eex` — static inventory render template. `ansible.build.ex` queries the `oci` + CLI directly (no `oracle.oci` collection dependency) for running instances in the + configured compartment, filters to this project (`Group` freeform tag), and composes + host groups + hostvars from the same four tag keys the AWS `aws_ec2` plugin's + `keyed_groups` reads (`MonitoringKey`, `InstanceGroup`, `DatabaseKey`, `QaNode`) so + playbooks and `--limit` targeting work unmodified across providers. + +Everything else (roles, setup playbooks, playbook/group_vars templates) is shared with AWS +and lives at the ansible root — OCI has no role variants yet, so it gets those files +byte-identical. + +**Static, not dynamic**: unlike `aws_ec2.yaml.eex`, this is a snapshot, not a live plugin +query. Regenerate it (`mix ansible.build`) after any instance create/scale/terminate or +`--limit`/group targeting will miss the change. diff --git a/priv/ansible/providers/oci/ansible.cfg.eex b/priv/ansible/providers/oci/ansible.cfg.eex new file mode 100644 index 00000000..af4f412c --- /dev/null +++ b/priv/ansible/providers/oci/ansible.cfg.eex @@ -0,0 +1,12 @@ +[defaults] +remote_user = ubuntu +inventory = ./oci.yaml +roles_path = ./roles +deprecation_warnings = False +private_key_file = <%= @pem_file_path %> +host_key_checking = False +# log_path = /var/log/syslog + +[ssh_connection] +ssh_args = -o StrictHostKeyChecking=accept-new +pipelining = True diff --git a/priv/ansible/providers/oci/group_vars/all.yaml.eex b/priv/ansible/providers/oci/group_vars/all.yaml.eex new file mode 100644 index 00000000..edc44c08 --- /dev/null +++ b/priv/ansible/providers/oci/group_vars/all.yaml.eex @@ -0,0 +1,34 @@ +oci_release_bucket: <%= @oci_release_bucket %> +oci_namespace: <%= @oci_namespace %> + +# The VCN CIDR — roles that hold an intra-network allowlist (clickhouse's ) read +# this instead of their AWS-VPC-shaped default. Must match terraform's vcn_cidr; both default +# to 10.20.0.0/16 and both read config :deploy_ex, :oci, vcn_cidr when set. +clickhouse_allowed_network_cidr: "<%= @oci_vcn_cidr %>" + +# grafana_loki/prometheus_db still read S3-shaped credentials — OCI logging/monitoring +# storage is a separate, not-yet-built phase, so these stay as AWS-shaped placeholders +# (unused unless those roles are pointed at an OCI-backed store) rather than leaving the +# variable undefined for roles that already expect it. +aws_credentials: + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" +<%= if @is_logging_enabled do %> + +loki_logger_s3_region: <%= DeployEx.Config.aws_log_region() %> +loki_logger_s3_bucket_name: <%= DeployEx.Config.aws_log_bucket() %> +loki_logger_retention: 30d + +env: <%= DeployEx.Config.env() %> + +grafana_loki_url: "http://10.0.1.50:3100"<% end %> +<%= if @is_prometheus_enabled do %> +grafana_prometheus_url: "http://10.0.1.40:9090" + +prometheus_scrape_region: <%= DeployEx.Config.aws_region() %> +prometheus_retention_size: 4GB +prometheus_retention_time: 7d +<% end %> + +system_env: + - PATH=/root/.asdf/shims:/root/.asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/priv/ansible/providers/oci/oci.yaml.eex b/priv/ansible/providers/oci/oci.yaml.eex new file mode 100644 index 00000000..0c2d1fb9 --- /dev/null +++ b/priv/ansible/providers/oci/oci.yaml.eex @@ -0,0 +1,11 @@ +--- +# Static inventory, generated by `mix ansible.build --provider oci` from the live instance +# list in the configured compartment (see ansible.build.ex `fetch_oci_instances/1`). +# +# AWS uses the dynamic `aws_ec2` plugin instead (see ../../aws_ec2.yaml.eex) because EC2 +# tags can be read live at ansible-run-time. OCI has no equivalent inventory plugin we're +# willing to add as a collection dependency, so this file is a point-in-time snapshot and +# must be regenerated after any instance create/scale/terminate (re-run `mix ansible.build`). +all: +<%= @hosts_section %> +<%= @children_section %> diff --git a/priv/ansible/providers/oci/roles/deploy_node/defaults/main.yaml b/priv/ansible/providers/oci/roles/deploy_node/defaults/main.yaml new file mode 100644 index 00000000..b7779917 --- /dev/null +++ b/priv/ansible/providers/oci/roles/deploy_node/defaults/main.yaml @@ -0,0 +1,18 @@ +app_name: "" +bucket_name: "{{ oci_release_bucket }}" + +# instance_principal is the production mode: the node authenticates as itself via a dynamic +# group, with no credential on disk, mirroring an EC2 instance profile. It requires a dynamic +# group and policy in the TENANCY (IAM writes land in the home region, not the resource +# region), which a compartment-scoped operator may not be able to create. Override to +# `api_key` where the node carries an ~/.oci/config instead — that is also how a CI runner +# reaches object storage. +oci_cli_auth: instance_principal +target_release_sha: "" +release_prefix: "" +release_state_prefix: "release-state" +qa_node_suffix: "" +app_port: 80 +extra_env: [] +system_env: [] +open_files_limit: 65536 diff --git a/priv/ansible/providers/oci/roles/deploy_node/files/find_oci_release_by_sha.sh b/priv/ansible/providers/oci/roles/deploy_node/files/find_oci_release_by_sha.sh new file mode 100644 index 00000000..6013f716 --- /dev/null +++ b/priv/ansible/providers/oci/roles/deploy_node/files/find_oci_release_by_sha.sh @@ -0,0 +1,40 @@ +#! /usr/bin/env bash +# Usage: find_oci_release_by_sha.sh +set -uo pipefail + +BUCKET_NAME="$1" +NAMESPACE="$2" +APP_NAME="$3" +RELEASE_PREFIX="$4" +TARGET_SHA="$5" + +# A failed lookup MUST NOT reach stdout. MEASURED: with instance principals unconfigured, the +# CLI's "ServiceError:" banner was taken as the release name and the caller ran +# `oci os object get --name 'ServiceError:'`, turning an auth failure into a confusing 404 on +# a nonsense object. +find_matching() { + local prefix="$1" + local output status + + output=$(oci os object list --namespace "$NAMESPACE" --bucket-name "$BUCKET_NAME" \ + --prefix "$prefix" --all --query 'data[*].name' --output json 2>&1) + status=$? + + if [ $status -ne 0 ]; then + echo "find_oci_release_by_sha.sh: listing '$prefix' in '$BUCKET_NAME' failed (exit $status)" >&2 + echo "$output" >&2 + return $status + fi + + # The oci CLI prints NOTHING (not "[]") when a list matches no resources, and jq treats + # empty input as producing no output, so no match is not an error here. + printf '%s' "$output" | jq -r '.[]' 2>/dev/null | grep "$TARGET_SHA" | head -n 1 +} + +match=$(find_matching "${RELEASE_PREFIX:+$RELEASE_PREFIX/}$APP_NAME") || exit $? + +if [ -z "$match" ] && [ -n "$RELEASE_PREFIX" ]; then + match=$(find_matching "$APP_NAME") || exit $? +fi + +echo "$match" diff --git a/priv/ansible/providers/oci/roles/deploy_node/files/latest_oci_release.sh b/priv/ansible/providers/oci/roles/deploy_node/files/latest_oci_release.sh new file mode 100644 index 00000000..0197db66 --- /dev/null +++ b/priv/ansible/providers/oci/roles/deploy_node/files/latest_oci_release.sh @@ -0,0 +1,44 @@ +#! /usr/bin/env bash +# Usage: latest_oci_release.sh +# +# OCI Object Storage equivalent of latest_aws_release.sh. Object names returned by the +# oci CLI are already relative to the bucket (unlike `aws s3 ls`, which prints +# "bucket/key"), so no bucket-name stripping is needed here. +set -uo pipefail + +BUCKET_NAME="$1" +NAMESPACE="$2" +APP_NAME="$3" +RELEASE_PREFIX="$4" + +# A failed lookup MUST NOT reach stdout. MEASURED: with instance principals unconfigured, the +# CLI's "ServiceError:" banner was taken as the release name and the caller ran +# `oci os object get --name 'ServiceError:'`, turning an auth failure into a confusing 404 on +# a nonsense object. Every lookup therefore checks the exit status and refuses to print +# anything that is not a plausible object key. +list_objects() { + local prefix="$1" + local output status + + output=$(oci os object list --namespace "$NAMESPACE" --bucket-name "$BUCKET_NAME" \ + --prefix "$prefix" --all --query 'data[*].name' --output json 2>&1) + status=$? + + if [ $status -ne 0 ]; then + echo "latest_oci_release.sh: listing '$prefix' in '$BUCKET_NAME' failed (exit $status)" >&2 + echo "$output" >&2 + return $status + fi + + # The oci CLI prints NOTHING (not "[]") when a list matches no resources, and jq treats + # empty input as producing no output, so an empty bucket is not an error here. + printf '%s' "$output" | jq -r '.[]' 2>/dev/null | sort -r | head -n 1 +} + +release=$(list_objects "${RELEASE_PREFIX:+$RELEASE_PREFIX/}$APP_NAME") || exit $? + +if [ -z "$release" ] && [ -n "$RELEASE_PREFIX" ]; then + release=$(list_objects "$APP_NAME") || exit $? +fi + +echo "$release" diff --git a/priv/ansible/providers/oci/roles/deploy_node/files/update_oci_release_state.sh b/priv/ansible/providers/oci/roles/deploy_node/files/update_oci_release_state.sh new file mode 100644 index 00000000..86fd79a7 --- /dev/null +++ b/priv/ansible/providers/oci/roles/deploy_node/files/update_oci_release_state.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Usage: update_oci_release_state.sh +# +# OCI Object Storage equivalent of update_release_state.sh. Tracks which release is deployed +# by maintaining two objects in the SAME bucket as the releases themselves: +# //current_release.txt +# //release_history.txt +# +# If the current release matches object_key, prints "unchanged" and exits. +# Otherwise updates both files and prints "changed". + +set -euo pipefail + +BUCKET="$1" +NAMESPACE="$2" +STATE_PREFIX="$3" +APP_NAME="$4" +OBJECT_KEY="$5" + +CURRENT_KEY="${STATE_PREFIX}/${APP_NAME}/current_release.txt" +HISTORY_KEY="${STATE_PREFIX}/${APP_NAME}/release_history.txt" + +CURRENT_FILE="/tmp/${APP_NAME}_current_release.txt" +HISTORY_FILE="/tmp/${APP_NAME}_release_history.txt" + +# Fetch current release (may not exist yet) +existing="" +if oci os object get --namespace "$NAMESPACE" --bucket-name "$BUCKET" --name "$CURRENT_KEY" --file "$CURRENT_FILE" 2>/dev/null; then + existing=$(tr -d '[:space:]' < "$CURRENT_FILE") +fi + +# Skip if unchanged +if [ "$existing" = "$OBJECT_KEY" ]; then + echo "unchanged" + exit 0 +fi + +# Fetch or create history +if ! oci os object get --namespace "$NAMESPACE" --bucket-name "$BUCKET" --name "$HISTORY_KEY" --file "$HISTORY_FILE" 2>/dev/null; then + touch "$HISTORY_FILE" +fi + +# Append old release to history (if there was one) +if [ -n "$existing" ]; then + echo "$existing" >> "$HISTORY_FILE" +fi + +# Write new current release +echo "$OBJECT_KEY" > "$CURRENT_FILE" + +# Upload both (the oci CLI prints the object's etag/metadata as JSON on success — matches +# the AWS script's `--quiet` intent by discarding it, keeping stdout to just the final line) +oci os object put --namespace "$NAMESPACE" --bucket-name "$BUCKET" --name "$CURRENT_KEY" --file "$CURRENT_FILE" --force > /dev/null +oci os object put --namespace "$NAMESPACE" --bucket-name "$BUCKET" --name "$HISTORY_KEY" --file "$HISTORY_FILE" --force > /dev/null + +echo "changed" diff --git a/priv/ansible/providers/oci/roles/deploy_node/tasks/main.yaml b/priv/ansible/providers/oci/roles/deploy_node/tasks/main.yaml new file mode 100644 index 00000000..8490213c --- /dev/null +++ b/priv/ansible/providers/oci/roles/deploy_node/tasks/main.yaml @@ -0,0 +1,100 @@ +- name: setup_node + block: + - name: Populate service facts + service_facts: + + - name: Find OCI file for {{ app_name }} in Object Storage {% if target_release_sha | length > 0 %} with sha {{ target_release_sha }} {% else %} latest {% endif %} and set it into variable + ansible.builtin.script: + cmd: > + {% if target_release_sha | length > 0 %} + find_oci_release_by_sha.sh {{ bucket_name }} {{ oci_namespace }} {{ app_name }} "{{ release_prefix }}" {{ target_release_sha }} + {% else %} + latest_oci_release.sh {{ bucket_name }} {{ oci_namespace }} {{ app_name }} "{{ release_prefix }}" + {% endif %} + environment: + OCI_CLI_AUTH: "{{ oci_cli_auth }}" + register: oci_file_name + + # "No release found" is a distinct, actionable state and must not be reported as a + # download failure. Without this the empty stdout flows into stdout_lines[0] and surfaces + # as an index error, or worse, as a 404 on whatever text the lookup happened to print. + - name: Fail clearly when no release exists for {{ app_name }} + ansible.builtin.fail: + msg: >- + No release found for {{ app_name }} in bucket {{ bucket_name }} + {% if target_release_sha | length > 0 %}matching sha {{ target_release_sha }}{% endif %}. + Upload one with `mix deploy_ex.upload` before deploying. + when: oci_file_name.stdout | trim | length == 0 + + - name: Set local release path + set_fact: + oci_object_key: "{{ oci_file_name.stdout_lines[0] }}" + local_release_file: "{{ app_name }}/{{ oci_file_name.stdout_lines[0] | basename }}" + + - name: Create release directories + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - /srv/{{ app_name }} + - /srv/unpack-directory + + - name: Download {{ app_name }} from OCI Object Storage + ansible.builtin.command: + cmd: >- + oci os object get --namespace {{ oci_namespace }} --bucket-name {{ bucket_name }} + --name "{{ oci_object_key }}" --file "/srv/{{ local_release_file }}" + creates: "/srv/{{ local_release_file }}" + environment: + OCI_CLI_AUTH: "{{ oci_cli_auth }}" + + - name: Untar {{ local_release_file }} + unarchive: + src: /srv/{{ local_release_file }} + dest: /srv/unpack-directory + remote_src: true + + - name: Update release state for {{ app_name }} + ansible.builtin.script: + cmd: update_oci_release_state.sh {{ bucket_name }} {{ oci_namespace }} {{ release_state_prefix }} {{ app_name }} {{ oci_object_key }} + environment: + OCI_CLI_AUTH: "{{ oci_cli_auth }}" + register: release_state_result + changed_when: "'changed' in release_state_result.stdout" + + - name: Add erlang_systemd.service file to /etc/systemd/system/{{ app_name }}.service + template: + src: erlang_systemd.service.j2 + dest: /etc/systemd/system/{{ app_name }}.service + owner: root + group: root + mode: '0644' + + - name: Enable {{ app_name }} service and reload systemd (out of critical path) + systemd: + name: "{{ app_name }}" + enabled: true + daemon_reload: true + + - name: Atomic swap into /srv/{{ app_name }} and restart {{ app_name }} + shell: | + set -euo pipefail + rm -rf /srv/{{ app_name }}.old + systemctl stop {{ app_name }}.service || true + if [ -d /srv/{{ app_name }} ]; then + mv /srv/{{ app_name }} /srv/{{ app_name }}.old + fi + mv /srv/unpack-directory /srv/{{ app_name }} + systemctl start {{ app_name }}.service + args: + executable: /bin/bash + + - name: Cleanup previous release (post-swap, no downtime impact) + file: + path: /srv/{{ app_name }}.old + state: absent + + become: true diff --git a/priv/ansible/providers/oci/roles/oci_cli/defaults/main.yaml b/priv/ansible/providers/oci/roles/oci_cli/defaults/main.yaml new file mode 100644 index 00000000..3f39a01e --- /dev/null +++ b/priv/ansible/providers/oci/roles/oci_cli/defaults/main.yaml @@ -0,0 +1,6 @@ +oci_cli_version: 3.90.1 + +# Its own virtualenv, not the system interpreter — see tasks/main.yaml for why a plain pip +# install cannot work on Ubuntu 24.04. +oci_cli_venv: /opt/oci-cli +oci_cli_bin: /usr/local/bin/oci diff --git a/priv/ansible/providers/oci/roles/oci_cli/tasks/main.yaml b/priv/ansible/providers/oci/roles/oci_cli/tasks/main.yaml new file mode 100644 index 00000000..4549f5f1 --- /dev/null +++ b/priv/ansible/providers/oci/roles/oci_cli/tasks/main.yaml @@ -0,0 +1,47 @@ +- name: oci_cli + block: + - name: Check if oci cli is installed + stat: + path: "{{ oci_cli_bin }}" + register: oci_cli_exe + + - name: Check installed oci cli version + command: "{{ oci_cli_bin }} --version" + register: oci_cli_current_version + when: oci_cli_exe.stat.exists + changed_when: false + failed_when: false + + - name: Install jq and python venv support + apt: + name: + - jq + - python3-venv + update_cache: true + + # oci-cli goes in its own virtualenv rather than into the system interpreter. On Ubuntu + # 24.04 a plain `pip install --break-system-packages oci-cli` MEASURABLY fails: oci-cli + # requires urllib3>=2.6.3, pip tries to replace the distro's urllib3 2.0.7, and dpkg-owned + # packages have no RECORD file, so the uninstall aborts with + # "Cannot uninstall urllib3 2.0.7, RECORD file not found". A venv sidesteps the conflict + # entirely and cannot damage system Python. + - name: Create the oci cli virtualenv + command: "python3 -m venv {{ oci_cli_venv }}" + args: + creates: "{{ oci_cli_venv }}/bin/python" + + - name: Install oci-cli {{ oci_cli_version }} + pip: + name: "oci-cli=={{ oci_cli_version }}" + virtualenv: "{{ oci_cli_venv }}" + when: >- + (not oci_cli_exe.stat.exists) or + (oci_cli_version not in (oci_cli_current_version.stdout | default(''))) + + - name: Link {{ oci_cli_bin }} onto the venv entrypoint + file: + src: "{{ oci_cli_venv }}/bin/oci" + dest: "{{ oci_cli_bin }}" + state: link + + become: true diff --git a/priv/ansible/roles/beam_linux_tuning/tasks/main.yaml b/priv/ansible/roles/beam_linux_tuning/tasks/main.yaml index b8fa838d..c8df7c91 100644 --- a/priv/ansible/roles/beam_linux_tuning/tasks/main.yaml +++ b/priv/ansible/roles/beam_linux_tuning/tasks/main.yaml @@ -1,5 +1,19 @@ - name: beam_linux_tuning block: + # Cloud images run unattended-upgrades on first boot, so a setup started right after + # terraform.apply races it for the dpkg lock and dies with "Could not get lock + # /var/lib/dpkg/lock-frontend. It is held by process N (apt)" — MEASURED on a freshly + # created node. `apt-get check` exits non-zero while the lock is held, so retrying it is + # a portable wait needing no extra packages. This role runs first in every setup + # playbook, so guarding here covers every apt task that follows it. + - name: Wait for apt to become available + ansible.builtin.command: apt-get check + register: apt_available + until: apt_available.rc == 0 + retries: 30 + delay: 10 + changed_when: false + - name: Gather active swap devices command: swapon --show=NAME --noheadings register: active_swap @@ -58,4 +72,33 @@ - name: Increase TCP Congestion Window shell: defrt=`ip route | grep "^default" | head -1` && ip route change $defrt initcwnd 10 when: rc_local_file.changed + + # OCI's platform images ship an iptables INPUT chain that REJECTs everything but SSH, + # persisted through netfilter-persistent — so even with the VCN wide open, an app node + # gets "No route to host" from redis, clickhouse and every BEAM peer (measured). Debian + # AMIs on AWS have no such default. Network policy is the cloud's job (security list / + # NSG); the host firewall is reset to accept-all and the persisted rules rewritten so a + # reboot does not bring the reject back. + - name: Check for the OCI image's default host firewall + stat: + path: /etc/iptables/rules.v4 + register: oci_iptables_rules + + - name: Detect a REJECT in the INPUT chain + shell: iptables -S INPUT | grep -c -- '-j REJECT' || true + register: oci_iptables_rejects + changed_when: false + when: oci_iptables_rules.stat.exists and ansible_facts['distribution'] == 'Ubuntu' + + - name: Open the host firewall — the VCN security list is the policy layer + when: + - oci_iptables_rules.stat.exists + - ansible_facts['distribution'] == 'Ubuntu' + - (oci_iptables_rejects.stdout | default('0') | int) > 0 + block: + - name: Flush the reject rules + shell: iptables -F INPUT && iptables -P INPUT ACCEPT && ip6tables -F INPUT && ip6tables -P INPUT ACCEPT + + - name: Persist an accept-all ruleset so a reboot does not restore the reject + shell: iptables-save > /etc/iptables/rules.v4 && ip6tables-save > /etc/iptables/rules.v6 become: true diff --git a/priv/ansible/roles/clickhouse/README.md b/priv/ansible/roles/clickhouse/README.md new file mode 100644 index 00000000..c1643dc8 --- /dev/null +++ b/priv/ansible/roles/clickhouse/README.md @@ -0,0 +1,152 @@ +# clickhouse + +> **STATUS: verified against a live host. One item is untestable without +> credentials this project does not hold.** +> +> Verified 2026-08-12 on throwaway OCI Ubuntu 24.04 hosts: +> +> | item | result | +> |---|---| +> | role runs green | PASSED — `failed=0`, service reached `active (running)` | +> | version pin | PASSED — `apt` history shows `clickhouse-server=24.8.14.39` installed fresh | +> | apt repo + GPG key URL | PASSED — the install above proves both resolve | +> | `clickhouse` system user/group | PASSED — `uid=999(clickhouse) gid=988(clickhouse)` | +> | cold tier OFF by default | PASSED — `config.d` held only `listen.xml` | +> | `SELECT 1` over TCP and HTTP | PASSED — `clickhouse-client` and `curl :8123` both return `1` | +> | default user from the configured CIDR | PASSED — connected over the host's own private IP (not loopback) with the CIDR set to the test VCN, so the CIDR entry is what was exercised | +> | cold tier ENABLED boots cleanly | PASSED — `NRestarts=0`, `active (running)`, `system.disks` shows `s3_cold ObjectStorage` and `system.storage_policies` the expected `tiered`/`hot`/`s3_cold` shape, booted with deliberately fake credentials — confirming `skip_access_check` behaves as its header claims | +> | second run is `changed=0` | NOT MET, and benign: the changes are `awscli` handlers plus this role's `Update apt cache`, which `cache_valid_time: 0` makes report changed every run by design (copied from `redis_server`). No clickhouse-role task drifts. | +> | cold tier against a REAL bucket | **UNTESTABLE HERE** — needs OCI Customer Secret Keys, an IAM write in the tenancy root | +> +> **On the loopback entries.** ClickHouse's `` check is literal and +> never implicitly permits 127.0.0.1, so an on-box healthcheck fails +> `AUTHENTICATION_FAILED` even with a confirmed-empty password — diagnosed from +> `preprocessed_configs/users.xml`, which showed `` correctly empty +> while the CIDR excluded loopback. Loopback is therefore listed unconditionally +> alongside the configured CIDR. Both paths are now confirmed working live. +> +> **Correction to an earlier version of this file.** It claimed the ClickHouse +> 24.8 deb's unit was broken on Ubuntu 24.04 systemd. That was WRONG. The service +> started cleanly on this exact host and package before any edit. The crash-loop +> appeared only after a `--` sequence landed inside an XML comment in +> `zz-allow-default-network.xml.j2`; XML 1.0 forbids that anywhere in a comment +> body, so Poco refused the file and the daemon exited before signalling +> readiness. systemd reports that as `Failed with result 'protocol'` — identical +> to a genuine `Type=notify` mismatch, which is why reading only `journalctl` led +> to the wrong conclusion. The ClickHouse error log named the file and line +> outright. Two lessons worth keeping: diagnose a daemon from ITS OWN log, not +> just systemd's view of it; and an elimination experiment that removes the wrong +> file proves nothing (`config.d/listen.xml` was removed while the actually +> broken `users.d/` fragment stayed in place). + +Installs a pinned `clickhouse-server` (apt), configures it via `config.d`/`users.d` +drop-in fragments (never touching the package's stock `config.xml`), and manages +it as a systemd service. Written to match the app-side contract in +`opgg_umbrella/config/runtime.exs` (`CLICKHOUSE_URL_OVERRIDE`, +`CLICKHOUSE_CLUSTER_OVERRIDE`, `CLICKHOUSE_POOL_SIZE`, +`CLICKHOUSE_MIGRATE_URL_OVERRIDE`, `CLICKHOUSE_STORAGE_POLICY_OVERRIDE`) and +`opgg_umbrella/docker/clickhouse/storage-s3-tiered.xml` (the dev/doc reference +for the cold-storage tier this role deploys server-side). + +## What it does + +- Adds the official ClickHouse apt repo + GPG key, pins the install to + `clickhouse_version.*` via `/etc/apt/preferences.d/clickhouse.pref` (major.minor + pin, so patch releases can land but a version bump never silently upgrades + the pinned line), and installs `clickhouse-server`/`clickhouse-client`. +- Renders `config.d/listen.xml` — listen host, HTTP/TCP/interserver ports, + data dir, and a server-wide memory ceiling. +- Renders `users.d/zz-allow-default-network.xml` — opens the built-in + `default` user (no password) to `clickhouse_allowed_network_cidr`, matching + the app's `Ch` client (see `defaults/main.yaml`). +- Optionally renders `config.d/storage-s3-tiered.xml` — the S3 cold-storage + tier's server-side `` — **off by default**. +- Enables and starts the `clickhouse-server` systemd unit the package ships. + +Every task is idempotent (`copy`/`template`/`apt`/`systemd` with no `command`/ +`shell` outside the one GPG-key step, which is itself guarded by a `stat` +check) — re-running the role with unchanged vars changes nothing. + +## Cold-storage tier: operator runbook + +`clickhouse_s3_cold_storage_enabled` defaults to `false`. Leave it there until +all of the following are true, in this order: + +1. A real S3 (AWS) or OCI Object Storage (via its S3-compatibility endpoint) + bucket exists for cold parts, with lifecycle/retention configured. +2. `clickhouse_s3_cold_bucket_endpoint` points at it, and (per the auth mode + below) either the node's IAM role or + `clickhouse_s3_cold_access_key_id`/`clickhouse_s3_cold_secret_access_key` + grant `s3:GetObject`/`PutObject`/`DeleteObject`/`ListBucket` on the bucket. +3. This role runs with `clickhouse_s3_cold_storage_enabled: true` and + clickhouse-server has restarted (the `template` task notifies the + `restart clickhouse-server` handler automatically). +4. **Only then** does ops flip `CLICKHOUSE_STORAGE_POLICY_OVERRIDE` on the + app (unset defaults to `'tiered'` per `runtime.exs` — set it to + `disabled` explicitly for any deploy that precedes step 3). + +Skipping the order in step 4 is the exact failure `storage-s3-tiered.xml.j2`'s +header warns about: a fresh `CREATE TABLE ... SETTINGS storage_policy = +'tiered'` against a server that has never defined the `tiered` policy fails +`NO_SUCH_POLICY`. This role cannot enforce that ordering across deploy +pipelines by itself — it can only guarantee that when it DOES define the +policy, the policy is on disk and loaded before the role returns. + +## AWS vs OCI: the credential asymmetry is real, not a bug + +`clickhouse_s3_cold_auth_mode` selects one of two mutually exclusive blocks in +`storage-s3-tiered.xml.j2`: + +| Mode | Renders | Works on | +|---|---|---| +| `instance_role` | `true` | AWS only | +| `static_keys` (default) | ``/`` | AWS and OCI | + +ClickHouse's `s3` disk type only speaks the S3 API. On OCI, the only endpoint +that understands it is OCI Object Storage's **S3-compatibility API** +(`https://.compat.objectstorage..oraclecloud.com//`), +and **that API does not accept OCI instance principals** — it requires +long-lived Customer Secret Keys (a static SigV4 access key/secret pair, +created under the OCI user's "Customer Secret Keys"). This is a documented +limitation of OCI's S3-compat surface, not something this role works around by +choice, and not something an OCI instance's own native-API instance-principal +support (used elsewhere for `oci os` CLI calls) changes — the S3-compat +endpoint is a separate auth surface. `clickhouse_s3_cold_auth_mode` therefore +defaults to `static_keys`: it's the only mode that works everywhere this role +might run. Set it to `instance_role` explicitly on AWS if you want the +node-IAM-role credential path instead of static keys there. + +Do not attempt to make `instance_role` work on OCI — it cannot, and the +failure mode (silently falling back to no credentials, or the disk failing to +authenticate) is worse than requiring an explicit static-keys config. + +### IAM/credential grants per provider + +- **AWS, `instance_role`**: the node's IAM instance profile needs + `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:ListBucket` on the + cold-tier bucket. No static keys stored anywhere. +- **AWS or OCI, `static_keys`**: `clickhouse_s3_cold_access_key_id` / + `clickhouse_s3_cold_secret_access_key` need the same four permissions — + on AWS an IAM user's access key; on OCI a Customer Secret Key belonging to + a user/policy scoped to the bucket's compartment. + +## Variables + +See `defaults/main.yaml` for the full list and inline documentation — +version, listen host, ports, data dir, memory ratio, allowed network CIDR, +and the cold-tier toggle/endpoint/auth-mode/credential vars. + +## Not done here (deliberately out of scope) + +- Provisioning the actual EC2/OCI compute node this role runs on, or tagging + it so `mix ansible.setup` targets it (that's `mix terraform.build`'s + `DatabaseKey`/`MonitoringKey` tag wiring — see + `priv/ansible/setup/clickhouse.yaml` for the host group this role expects + to exist). +- Creating the cold-tier bucket, its lifecycle policy, or the IAM/OCI + credentials themselves — those are operator/Terraform actions, listed + above as prerequisites. +- ClickHouse user/ACL management beyond the single `default` user the app + connects as. Add a dedicated `users.d` fragment here if a second + (e.g. migrate-only) ClickHouse user is ever needed to match + `CLICKHOUSE_MIGRATE_URL_OVERRIDE`'s separate-credentials intent. diff --git a/priv/ansible/roles/clickhouse/defaults/main.yaml b/priv/ansible/roles/clickhouse/defaults/main.yaml new file mode 100644 index 00000000..16263cc4 --- /dev/null +++ b/priv/ansible/roles/clickhouse/defaults/main.yaml @@ -0,0 +1,37 @@ +clickhouse_version: "24.8" + +clickhouse_listen_host: "0.0.0.0" +clickhouse_http_port: 8123 +clickhouse_tcp_port: 9000 +clickhouse_interserver_http_port: 9009 +clickhouse_data_dir: /var/lib/clickhouse + +# Fraction of system RAM clickhouse-server is allowed to use (the server's own +# built-in default is 0.9; repeated here so it is one obvious place to tune +# per instance size rather than hunting through config.d). +clickhouse_max_server_memory_usage_to_ram_ratio: 0.9 + +# CIDR allowed to connect as the passwordless `default` user, mirroring the +# `Ch` client the app connects with (see README.md). Scope this to the +# VPC/VCN range, never 0.0.0.0/0 or ::/0. +clickhouse_allowed_network_cidr: "10.0.0.0/16" + +# S3 cold-storage tier (opgg_umbrella S3 §4.3 / R16). OFF by default: a +# config.d file naming an unreachable endpoint is a boot risk, and flipping +# the app's CLICKHOUSE_STORAGE_POLICY_OVERRIDE to 'tiered' before the server +# defines the policy fails table creation with NO_SUCH_POLICY. See README.md +# for the operator runbook before setting this true. +clickhouse_s3_cold_storage_enabled: false +clickhouse_s3_cold_bucket_endpoint: "" +clickhouse_s3_cold_move_factor: 0.1 + +# "static_keys" (long-lived Customer Secret Keys) is the ONLY mode that works +# against OCI Object Storage's S3-compatibility API — OCI instance principals +# are not supported there. "instance_role" is AWS-only, and requires the +# node's IAM role to grant s3:GetObject/PutObject/DeleteObject/ListBucket on +# the bucket. Default is static_keys because it is the one mode that works on +# every provider this role may run on; set instance_role explicitly on AWS if +# you want the node-role credential path instead. See README.md. +clickhouse_s3_cold_auth_mode: static_keys +clickhouse_s3_cold_access_key_id: "" +clickhouse_s3_cold_secret_access_key: "" diff --git a/priv/ansible/roles/clickhouse/handlers/main.yaml b/priv/ansible/roles/clickhouse/handlers/main.yaml new file mode 100644 index 00000000..9fc5f1ed --- /dev/null +++ b/priv/ansible/roles/clickhouse/handlers/main.yaml @@ -0,0 +1,6 @@ +- name: restart clickhouse-server + become: true + systemd: + name: clickhouse-server + state: restarted + daemon_reload: true diff --git a/priv/ansible/roles/clickhouse/tasks/main.yaml b/priv/ansible/roles/clickhouse/tasks/main.yaml new file mode 100644 index 00000000..3ca765cc --- /dev/null +++ b/priv/ansible/roles/clickhouse/tasks/main.yaml @@ -0,0 +1,122 @@ +- name: clickhouse + block: + - name: Install ClickHouse apt prerequisites + apt: + name: + - apt-transport-https + - ca-certificates + - gnupg + state: present + update_cache: true + + - name: Check ClickHouse GPG key + stat: + path: /usr/share/keyrings/clickhouse-keyring.gpg + register: clickhouse_gpg_key + + - name: Add ClickHouse gpg key + shell: curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg + when: not clickhouse_gpg_key.stat.exists + + - name: Add ClickHouse apt repository + copy: + dest: /etc/apt/sources.list.d/clickhouse.list + mode: 0644 + content: | + deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main + + # Pins the install (and every future `apt upgrade`) to the same major.minor + # this role was written against, without needing to know the exact patch + # version — matching the pinned major.minor of the clickhouse/clickhouse-server + # Docker image tag used elsewhere, without silently upgrading past it. + - name: Pin ClickHouse packages to {{ clickhouse_version }}.* + copy: + dest: /etc/apt/preferences.d/clickhouse.pref + mode: 0644 + content: | + Package: clickhouse-server clickhouse-client clickhouse-common-static + Pin: version {{ clickhouse_version }}.* + Pin-Priority: 1001 + + - name: Update apt cache + apt: + update_cache: true + cache_valid_time: 0 + + # DEBIAN_FRONTEND/CLICKHOUSE_SKIP_USER_SETUP steer the package's postinst + # away from its interactive default-user password prompt. The + # zz-allow-default-network.xml.j2 override below is the real guarantee — + # it always wins regardless of what the postinst decided — but skipping the + # prompt keeps this task from ever blocking on stdin. + - name: Install ClickHouse server and client + apt: + name: + - clickhouse-server + - clickhouse-client + state: present + environment: + DEBIAN_FRONTEND: noninteractive + CLICKHOUSE_SKIP_USER_SETUP: "1" + + - name: Ensure config.d and users.d directories exist + file: + path: "{{ item }}" + state: directory + owner: clickhouse + group: clickhouse + mode: 0755 + loop: + - /etc/clickhouse-server/config.d + - /etc/clickhouse-server/users.d + + - name: Add listen/port/data-dir/memory config to config.d + template: + src: listen.xml.j2 + dest: /etc/clickhouse-server/config.d/listen.xml + owner: clickhouse + group: clickhouse + mode: 0644 + notify: restart clickhouse-server + + # Named zz- so it merges AFTER anything the package postinst wrote to + # users.d/default-user.xml (config.d/users.d fragments merge alphabetically, + # last wins) — see the file's own header comment for why both the password + # and the network need clearing, not just one. + - name: Allow the default user from the configured network + template: + src: zz-allow-default-network.xml.j2 + dest: /etc/clickhouse-server/users.d/zz-allow-default-network.xml + owner: clickhouse + group: clickhouse + mode: 0644 + notify: restart clickhouse-server + + - name: Deploy S3 cold-storage tier config + template: + src: storage-s3-tiered.xml.j2 + dest: /etc/clickhouse-server/config.d/storage-s3-tiered.xml + owner: clickhouse + group: clickhouse + mode: 0644 + when: clickhouse_s3_cold_storage_enabled + notify: restart clickhouse-server + + - name: Remove S3 cold-storage tier config when disabled + file: + path: /etc/clickhouse-server/config.d/storage-s3-tiered.xml + state: absent + when: not clickhouse_s3_cold_storage_enabled + notify: restart clickhouse-server + + - name: Enable clickhouse-server service + systemd: + name: clickhouse-server + enabled: true + + - name: Start clickhouse-server service + systemd: + daemon_reload: true + name: clickhouse-server + state: started + + become: true diff --git a/priv/ansible/roles/clickhouse/templates/listen.xml.j2 b/priv/ansible/roles/clickhouse/templates/listen.xml.j2 new file mode 100644 index 00000000..c26485cd --- /dev/null +++ b/priv/ansible/roles/clickhouse/templates/listen.xml.j2 @@ -0,0 +1,19 @@ + + + + {{ clickhouse_listen_host }} + {{ clickhouse_http_port }} + {{ clickhouse_tcp_port }} + {{ clickhouse_interserver_http_port }} + + {{ clickhouse_data_dir }}/ + {{ clickhouse_data_dir }}/tmp/ + {{ clickhouse_data_dir }}/user_files/ + {{ clickhouse_data_dir }}/format_schemas/ + + {{ clickhouse_max_server_memory_usage_to_ram_ratio }} + diff --git a/priv/ansible/roles/clickhouse/templates/storage-s3-tiered.xml.j2 b/priv/ansible/roles/clickhouse/templates/storage-s3-tiered.xml.j2 new file mode 100644 index 00000000..1df32a11 --- /dev/null +++ b/priv/ansible/roles/clickhouse/templates/storage-s3-tiered.xml.j2 @@ -0,0 +1,64 @@ + + + + + + + s3 + {{ clickhouse_s3_cold_bucket_endpoint }} +{% if clickhouse_s3_cold_auth_mode == 'instance_role' %} + true +{% else %} + {{ clickhouse_s3_cold_access_key_id }} + {{ clickhouse_s3_cold_secret_access_key }} +{% endif %} + + true + {{ clickhouse_data_dir }}/disks/s3_cold/ + + + + + + + + default + + + s3_cold + + + {{ clickhouse_s3_cold_move_factor }} + + + + diff --git a/priv/ansible/roles/clickhouse/templates/zz-allow-default-network.xml.j2 b/priv/ansible/roles/clickhouse/templates/zz-allow-default-network.xml.j2 new file mode 100644 index 00000000..7b7642bf --- /dev/null +++ b/priv/ansible/roles/clickhouse/templates/zz-allow-default-network.xml.j2 @@ -0,0 +1,54 @@ + + + + + + + + + + ::1 + 127.0.0.1 + {{ clickhouse_allowed_network_cidr }} + + + + diff --git a/priv/ansible/roles/grafana_alloy/tasks/main.yaml b/priv/ansible/roles/grafana_alloy/tasks/main.yaml index a85eecec..f87a3551 100644 --- a/priv/ansible/roles/grafana_alloy/tasks/main.yaml +++ b/priv/ansible/roles/grafana_alloy/tasks/main.yaml @@ -29,10 +29,12 @@ path: /root/promtail_positions.yaml state: absent + # `args: warn: false` was removed in ansible-core 2.14 and is now a hard error: + # "Unsupported parameters for (ansible.legacy.command) module: warn". MEASURED against + # ansible-core 2.18.4, where it fails the play. The warning it suppressed was the + # use-a-module-instead nag, which no longer exists. - name: Remove old promtail binary shell: rm -f /root/promtail-* - args: - warn: false - name: Reload systemd after promtail cleanup systemd: @@ -44,6 +46,16 @@ path: ~/alloy-{{ alloy_architecture }} register: alloy + # Alloy ships as a .zip, and ansible's unarchive shells out to unzip for those. Ubuntu's + # base image has no unzip, so the task fails with "Unable to find required 'unzip' or + # 'zipinfo' binary in the path" after a confusing wall of tar errors — MEASURED on Ubuntu + # 24.04. The Debian AMI this role grew up on happened to include it. + - name: Install unzip, required to extract the Alloy release + apt: + name: unzip + update_cache: true + when: not alloy.stat.exists + - name: Download Alloy {{ alloy_architecture }}/{{ alloy_version }} unarchive: src: https://github.com/grafana/alloy/releases/download/{{ alloy_version }}/alloy-{{ alloy_architecture }}.zip diff --git a/priv/ansible/roles/ipv6/tasks/main.yaml b/priv/ansible/roles/ipv6/tasks/main.yaml index 244d9199..a90a384b 100644 --- a/priv/ansible/roles/ipv6/tasks/main.yaml +++ b/priv/ansible/roles/ipv6/tasks/main.yaml @@ -8,6 +8,18 @@ group: root mode: '0644' + # The awscli role creates ~/.aws while writing credentials, so on AWS this directory + # happened to already exist by the time the copy below ran. That was incidental, not + # guaranteed — MEASURED on an OCI node, this task fails with "Destination directory + # /root/.aws does not exist" and takes the whole setup play down with it. + - name: Ensure the aws config directory exists + ansible.builtin.file: + path: ~/.aws + state: directory + owner: root + group: root + mode: '0755' + - name: Add dualstack to boto config ansible.builtin.copy: content: | diff --git a/priv/ansible/roles/rabbitmq_server/defaults/main.yaml b/priv/ansible/roles/rabbitmq_server/defaults/main.yaml new file mode 100644 index 00000000..7e8b14ce --- /dev/null +++ b/priv/ansible/roles/rabbitmq_server/defaults/main.yaml @@ -0,0 +1,56 @@ +# Versions are pinned to GitHub Releases artifacts because RabbitMQ's apt mirrors +# (ppa1.rabbitmq.com / ppa*.novemberain.com) were unresolvable or 404 from both a laptop and an +# OCI node on 2026-08-18, while the GitHub Releases path was verified end to end (Erlang 27 + +# RabbitMQ 4.1.8 installed and active on Ubuntu 24.04). Ubuntu's own archive carries 3.12, +# which predates the 4.x quorum-queue semantics the app relies on. +rabbitmq_version: 4.1.8 +rabbitmq_erlang_version: 27.3.4.16 +rabbitmq_erlang_release_tag: "{{ rabbitmq_erlang_version }}-ubuntu/{{ ansible_facts['distribution_release'] }}" +rabbitmq_erlang_tarball_url: "https://github.com/rabbitmq/erlang-debian-package/releases/download/{{ rabbitmq_erlang_release_tag }}/erlang-{{ rabbitmq_erlang_version }}-ubuntu-{{ ansible_facts['distribution_release'] }}.tar.gz" +rabbitmq_deb_url: "https://github.com/rabbitmq/rabbitmq-server/releases/download/v{{ rabbitmq_version }}/rabbitmq-server_{{ rabbitmq_version }}-1_all.deb" + +# The subset of the Erlang debs RabbitMQ needs; the tarball ships 41 and the rest (wx, jinterface, +# examples, manpages …) is dead weight on a broker. +rabbitmq_erlang_packages: + - erlang-base + - erlang-asn1 + - erlang-crypto + - erlang-eldap + - erlang-ftp + - erlang-inets + - erlang-mnesia + - erlang-os-mon + - erlang-parsetools + - erlang-public-key + - erlang-runtime-tools + - erlang-snmp + - erlang-ssl + - erlang-syntax-tools + - erlang-tftp + - erlang-tools + - erlang-xmerl + +rabbitmq_vhost: tft + +# The application's AMQP identity: full configure/write/read on the vhost. Password comes from +# the consuming project's (gitignored) group vars — an empty default means the user is NOT +# created, so a missing secret fails loudly at the app's first connect instead of shipping a +# guessable credential. +rabbitmq_app_user: opgg_app +rabbitmq_app_password: "" + +# The management-API identity the app's BrokerMgmt uses (RABBITMQ_MGMT_RO_USER). It needs the +# `management` tag plus read AND write on the vhost — BrokerMgmt.publish_message posts through +# the HTTP API — so "read-only" is the app's name for it, not its permission set. `configure` +# stays empty: it can neither declare nor delete queues. +rabbitmq_mgmt_user: tft_admin_ro +rabbitmq_mgmt_password: "" + +# `guest` is loopback-only upstream (rabbitmq.conf loopback_users) and stays that way here; +# the dev docker-compose widens it, prod does not. +rabbitmq_management_listener_port: 15672 +rabbitmq_amqp_port: 5672 + +# Quorum queues want a real disk floor; the upstream default (50MB) is a laptop number. +rabbitmq_disk_free_limit: 2GB +rabbitmq_vm_memory_high_watermark: 0.6 diff --git a/priv/ansible/roles/rabbitmq_server/tasks/main.yaml b/priv/ansible/roles/rabbitmq_server/tasks/main.yaml new file mode 100644 index 00000000..5bdc63dc --- /dev/null +++ b/priv/ansible/roles/rabbitmq_server/tasks/main.yaml @@ -0,0 +1,145 @@ +- name: rabbitmq + tags: rabbitmq + become: true + block: + - name: Populate service facts + service_facts: + + - name: Check installed rabbitmq version + command: dpkg-query -W -f='${Version}' rabbitmq-server + register: rabbitmq_installed + changed_when: false + failed_when: false + + - name: Check installed erlang version + command: dpkg-query -W -f='${Version}' erlang-base + register: erlang_installed + changed_when: false + failed_when: false + + # Erlang comes from RabbitMQ's own debian-package releases (a tarball of debs per Ubuntu + # codename), unpacked and installed by path so apt resolves the intra-Erlang dependencies. + - name: Install Erlang {{ rabbitmq_erlang_version }} + when: not erlang_installed.stdout.startswith(rabbitmq_erlang_version) + block: + - name: Create erlang download directory + file: + path: /opt/deploy_ex/erlang + state: directory + mode: '0755' + + - name: Download erlang debs tarball + get_url: + url: "{{ rabbitmq_erlang_tarball_url }}" + dest: /opt/deploy_ex/erlang/erlang.tar.gz + mode: '0644' + timeout: 120 + + - name: Unpack erlang debs + unarchive: + src: /opt/deploy_ex/erlang/erlang.tar.gz + dest: /opt/deploy_ex/erlang + remote_src: true + + - name: Locate the erlang debs to install + find: + paths: /opt/deploy_ex/erlang + patterns: "{{ rabbitmq_erlang_packages | map('regex_replace', '$', '_*.deb') | list }}" + recurse: true + register: erlang_debs + + - name: Install erlang debs + apt: + deb: "{{ item }}" + loop: "{{ erlang_debs.files | map(attribute='path') | sort }}" + + - name: Download rabbitmq-server {{ rabbitmq_version }} + get_url: + url: "{{ rabbitmq_deb_url }}" + dest: /opt/deploy_ex/rabbitmq-server_{{ rabbitmq_version }}.deb + mode: '0644' + timeout: 120 + when: rabbitmq_installed.stdout != rabbitmq_version ~ '-1' + + - name: Install rabbitmq-server + apt: + deb: /opt/deploy_ex/rabbitmq-server_{{ rabbitmq_version }}.deb + when: rabbitmq_installed.stdout != rabbitmq_version ~ '-1' + + - name: Configure rabbitmq + template: + src: rabbitmq.conf.j2 + dest: /etc/rabbitmq/rabbitmq.conf + owner: rabbitmq + group: rabbitmq + mode: '0644' + register: rabbitmq_conf + + # Plain rabbitmqctl/rabbitmq-plugins throughout: deploy_ex takes no ansible collection + # dependencies (see ansible.build's inventory note), and every command below is idempotent + # or guarded so reruns do not churn. + - name: Enable the management plugin + command: rabbitmq-plugins enable --offline rabbitmq_management + register: rabbitmq_mgmt_plugin + changed_when: "'started' in rabbitmq_mgmt_plugin.stdout or 'enabled' in rabbitmq_mgmt_plugin.stdout" + + - name: Enable and (re)start rabbitmq-server + systemd: + name: rabbitmq-server + enabled: true + state: "{{ 'restarted' if (rabbitmq_conf.changed or rabbitmq_mgmt_plugin.changed) else 'started' }}" + daemon_reload: true + + - name: Wait for the broker + command: rabbitmq-diagnostics -q check_port_connectivity + register: rabbitmq_ready + retries: 20 + delay: 5 + until: rabbitmq_ready.rc == 0 + changed_when: false + + - name: List vhosts + command: rabbitmqctl -q list_vhosts name + register: rabbitmq_vhosts + changed_when: false + + - name: Create the {{ rabbitmq_vhost }} vhost + command: rabbitmqctl add_vhost {{ rabbitmq_vhost }} + when: rabbitmq_vhost not in rabbitmq_vhosts.stdout_lines + + - name: List users + command: rabbitmqctl -q list_users + register: rabbitmq_users + changed_when: false + + # Created once; a rerun does not rewrite the password, so the app's credential does not + # churn under it. Rotate by deleting the user and rerunning. + - name: Create the application user + command: rabbitmqctl add_user {{ rabbitmq_app_user }} {{ rabbitmq_app_password | quote }} + when: rabbitmq_app_password != "" and rabbitmq_app_user not in (rabbitmq_users.stdout_lines | map('regex_replace', '\\s.*$', '') | list) + no_log: true + + - name: Grant the application user full permissions on {{ rabbitmq_vhost }} + command: rabbitmqctl set_permissions -p {{ rabbitmq_vhost }} {{ rabbitmq_app_user }} ".*" ".*" ".*" + when: rabbitmq_app_password != "" + changed_when: false + + - name: Create the management-API user + command: rabbitmqctl add_user {{ rabbitmq_mgmt_user }} {{ rabbitmq_mgmt_password | quote }} + when: rabbitmq_mgmt_password != "" and rabbitmq_mgmt_user not in (rabbitmq_users.stdout_lines | map('regex_replace', '\\s.*$', '') | list) + no_log: true + + - name: Tag the management-API user + command: rabbitmqctl set_user_tags {{ rabbitmq_mgmt_user }} management + when: rabbitmq_mgmt_password != "" + changed_when: false + + - name: Grant the management-API user read+write (no configure) on {{ rabbitmq_vhost }} + command: rabbitmqctl set_permissions -p {{ rabbitmq_vhost }} {{ rabbitmq_mgmt_user }} "^$" ".*" ".*" + when: rabbitmq_mgmt_password != "" + changed_when: false + + - name: Refuse to leave the broker without an application user + fail: + msg: "rabbitmq_app_password is empty — set it in the consuming project's (gitignored) group vars so the {{ rabbitmq_app_user }} user is created" + when: rabbitmq_app_password == "" diff --git a/priv/ansible/roles/rabbitmq_server/templates/rabbitmq.conf.j2 b/priv/ansible/roles/rabbitmq_server/templates/rabbitmq.conf.j2 new file mode 100644 index 00000000..224253cb --- /dev/null +++ b/priv/ansible/roles/rabbitmq_server/templates/rabbitmq.conf.j2 @@ -0,0 +1,17 @@ +# Managed by deploy_ex (roles/rabbitmq_server). Single-node broker; quorum queues are +# declared by the application with x-queue-type=quorum and work on one node (Raft majority +# of 1). Adding nodes buys HA, not correctness — and a 2-node cluster is the one layout to +# avoid (majority of 2 is 2, so losing either halts every queue). + +listeners.tcp.default = {{ rabbitmq_amqp_port }} +management.tcp.port = {{ rabbitmq_management_listener_port }} +management.tcp.ip = 0.0.0.0 + +# guest stays loopback-only; the app authenticates as {{ rabbitmq_app_user }}. +loopback_users.guest = true + +disk_free_limit.absolute = {{ rabbitmq_disk_free_limit }} +vm_memory_high_watermark.relative = {{ rabbitmq_vm_memory_high_watermark }} + +# Consumers that vanish mid-delivery re-queue promptly. +heartbeat = 60 diff --git a/priv/ansible/roles/redis_server/tasks/main.yaml b/priv/ansible/roles/redis_server/tasks/main.yaml index 4e2c0135..e50bc7dc 100644 --- a/priv/ansible/roles/redis_server/tasks/main.yaml +++ b/priv/ansible/roles/redis_server/tasks/main.yaml @@ -4,6 +4,15 @@ - name: Populate service facts service_facts: + # The redis-stack-server binary (/opt/redis-stack) is a Debian-only artefact: it links + # libssl1.1, which Ubuntu 24.04 does not ship, and packages.redis.io publishes no + # redis-stack-server for noble at all. Redis 8 folded the stack modules into redis-server + # itself, so an Ubuntu host with redis_stack_enabled installs redis-server 8.x from + # packages.redis.io and runs it under the plain redis unit — same features, no /opt binary. + - name: Decide whether the redis-stack binary path applies + set_fact: + redis_use_stack_binary: "{{ redis_stack_enabled and ansible_facts['distribution'] == 'Debian' }}" + - name: Configure vm.overcommit_memory for Redis sysctl: name: vm.overcommit_memory @@ -31,6 +40,8 @@ mode: '0755' insertbefore: 'exit 0' + # redis_version is a Debian package revision (…+deb13u1) that does not exist in Ubuntu's + # archive; Ubuntu gets its redis from packages.redis.io further down. - name: Install Redis packages apt: name: @@ -39,6 +50,7 @@ - redis={{ redis_version }} state: present update_cache: true + when: ansible_facts['distribution'] == 'Debian' - name: Install GPG apt: @@ -65,9 +77,15 @@ - name: Add source repository into sources list shell: | echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb bullseye main" | tee /etc/apt/sources.list.d/redis.list - when: redis_stack_enabled and not redis_stack_source_list.stat.exists + when: redis_stack_enabled and not redis_stack_source_list.stat.exists and ansible_facts['distribution'] == 'Debian' + + - name: Add source repository into sources list (Ubuntu codename) + shell: | + echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb {{ ansible_facts['distribution_release'] }} main" | tee /etc/apt/sources.list.d/redis.list + when: redis_stack_enabled and not redis_stack_source_list.stat.exists and ansible_facts['distribution'] == 'Ubuntu' - name: Add bullseye repo (deb822) for libssl1.1 only + when: ansible_facts['distribution'] == 'Debian' deb822_repository: name: debian-bullseye types: [deb] @@ -78,6 +96,7 @@ state: present - name: Pin libssl1.1 so only that package can come from bullseye + when: ansible_facts['distribution'] == 'Debian' copy: dest: /etc/apt/preferences.d/libssl1.1.pref mode: '0644' @@ -103,6 +122,7 @@ apt: name: libssl1.1 state: present + when: ansible_facts['distribution'] == 'Debian' - name: Install Redis Stack apt: @@ -110,7 +130,16 @@ update_cache: true default_release: "bullseye" state: present - when: redis_stack_enabled + when: redis_use_stack_binary + + - name: Install Redis 8 from packages.redis.io (Ubuntu) + apt: + name: + - redis-server + - redis-tools + update_cache: true + state: present + when: ansible_facts['distribution'] == 'Ubuntu' - name: Add redis.conf file to /etc/redis/redis.conf template: @@ -127,7 +156,7 @@ owner: root group: root mode: 0644 - when: redis_stack_enabled + when: redis_use_stack_binary - name: Run ulimit -n 65536 shell: ulimit -n 65536 # noqa command-instead-of-shell @@ -161,26 +190,26 @@ systemd: name: redis-server state: stopped - when: redis_stack_enabled + when: redis_use_stack_binary - name: Disable redis-server service when redis stack is enabled systemd: name: redis-server enabled: false - when: redis_stack_enabled + when: redis_use_stack_binary - name: Check if redis.service is a symlink stat: path: /etc/systemd/system/redis.service register: redis_service_unit - when: redis_stack_enabled + when: redis_use_stack_binary - name: Remove redis.service symlink when redis stack is enabled file: path: /etc/systemd/system/redis.service state: absent when: - - redis_stack_enabled + - redis_use_stack_binary - redis_service_unit.stat.exists | default(false) - redis_service_unit.stat.islnk | default(false) @@ -188,7 +217,7 @@ systemd: name: redis-server masked: true - when: redis_stack_enabled + when: redis_use_stack_binary - name: Stop and disable redis alias service when redis stack is enabled systemd: @@ -196,13 +225,53 @@ state: stopped enabled: false failed_when: false - when: redis_stack_enabled + when: redis_use_stack_binary - name: Start redis service systemd: name: redis state: started - when: not redis_stack_enabled + when: not redis_use_stack_binary and ansible_facts['distribution'] == 'Debian' + + # The Debian units run redis as root, and /data arrives from the EBS mount role there. The + # Ubuntu package unit runs as User=redis under ProtectSystem, so redis.conf's `dir /data` + # needs the directory to exist, be owned by redis, and be whitelisted for the unit — + # otherwise the restart fails with a bare "No such file or directory" (measured). + - name: Create /data for redis (Ubuntu) + file: + path: /data + state: directory + owner: redis + group: redis + mode: '0750' + when: ansible_facts['distribution'] == 'Ubuntu' + + - name: Create the redis-server unit drop-in directory (Ubuntu) + file: + path: /etc/systemd/system/redis-server.service.d + state: directory + mode: '0755' + when: ansible_facts['distribution'] == 'Ubuntu' + + - name: Allow the redis-server unit to write /data (Ubuntu) + copy: + dest: /etc/systemd/system/redis-server.service.d/data-dir.conf + mode: '0644' + content: | + [Service] + ReadWritePaths=-/data + when: ansible_facts['distribution'] == 'Ubuntu' + + # restarted, not started: the packages.redis.io install already brought redis up on the + # stock conf, so a plain `started` would leave our /etc/redis/redis.conf unread until the + # next reboot. + - name: Restart redis-server with the managed config (Ubuntu) + systemd: + name: redis-server + daemon_reload: true + enabled: true + state: restarted + when: ansible_facts['distribution'] == 'Ubuntu' - name: Reload, Enable & Stop redis stack service systemd: @@ -210,13 +279,13 @@ daemon_reload: true enabled: true state: stopped - when: redis_stack_enabled + when: redis_use_stack_binary - name: Start redis stack service systemd: name: redis-stack state: started - when: redis_stack_enabled + when: redis_use_stack_binary - name: Stop redis-exporter service systemd: diff --git a/priv/ansible/roles/redis_server/templates/redis.conf b/priv/ansible/roles/redis_server/templates/redis.conf index e2ea6e94..acef352d 100644 --- a/priv/ansible/roles/redis_server/templates/redis.conf +++ b/priv/ansible/roles/redis_server/templates/redis.conf @@ -84,7 +84,15 @@ bind 0.0.0.0 # you are sure you want clients from other hosts to connect to Redis # even if no authentication is configured, nor a specific set of interfaces # are explicitly listed using the "bind" directive. +{% if ansible_facts['distribution'] == 'Ubuntu' %} +# Redis 8 enforces protected mode against every non-loopback client when no password is set, +# so app nodes get -DENIED. Reachability is the cloud's job here (VCN security list, no +# public 6379 at the edge), so protected mode is switched off rather than adding a password +# every app would then need. Debian's stack path keeps the upstream default. +protected-mode no +{% else %} protected-mode yes +{% endif %} # Accept connections on the specified port, default is 6379 (IANA #815344). # If port 0 is specified Redis will not listen on a TCP socket. @@ -233,7 +241,11 @@ daemonize no # UPSTART_JOB or NOTIFY_SOCKET environment variables # Note: these supervision methods only signal "process is ready." # They do not enable continuous pings back to your supervisor. +{% if ansible_facts['distribution'] == 'Ubuntu' %} +supervised systemd +{% else %} supervised no +{% endif %} # If a pid file is specified, Redis writes it where specified at startup # and removes it at exit. diff --git a/priv/ansible/setup/clickhouse.yaml b/priv/ansible/setup/clickhouse.yaml new file mode 100644 index 00000000..061bde9f --- /dev/null +++ b/priv/ansible/setup/clickhouse.yaml @@ -0,0 +1,9 @@ +- hosts: database_*_clickhouse + roles: + - beam_linux_tuning + - pip3 + - awscli + - log_cleanup + - prometheus_exporter + - clickhouse + - ipv6 diff --git a/priv/ansible/setup/rabbitmq.yaml b/priv/ansible/setup/rabbitmq.yaml new file mode 100644 index 00000000..9ee12c01 --- /dev/null +++ b/priv/ansible/setup/rabbitmq.yaml @@ -0,0 +1,9 @@ +- hosts: database_*_rabbitmq + roles: + - beam_linux_tuning + - pip3 + - awscli + - log_cleanup + - prometheus_exporter + - rabbitmq_server + - ipv6 diff --git a/priv/terraform/providers/oci/README.md b/priv/terraform/providers/oci/README.md new file mode 100644 index 00000000..121eace8 --- /dev/null +++ b/priv/terraform/providers/oci/README.md @@ -0,0 +1,159 @@ +# OCI environment + +An Oracle Cloud environment — VCN, internet gateway, route table, security list, public subnet, +one OCI compute instance per app entry in `var._project` (mirroring AWS's per-app +`module "ec2_instance" { for_each = ... }`), a release bucket, and the dynamic group/policy +instance principals need to reach it. Rendered by `mix terraform.build --provider oci` via +`DeployEx.Cloud.PrivFileSet` — non-`.eex` files here are copied as-is, `.eex` files render and +flatten onto the terraform root (`instance.tf.eex` -> `instance.tf`, etc). + +The original single-instance skeleton this replaced was verified against a live tenancy: 6 +resources created, instance reached RUNNING, all 6 destroyed, compartment confirmed empty. The +current multi-app shape has been verified with `tofu init`/`validate`/`plan` only — see the plan +doc's v21+ amendments for exact commands and output. **`tofu apply` has not been run against this +shape.** + +## What's here vs. AWS + +Deliberately minimal compared to the AWS `aws-instance` module — no EBS snapshot restore, no +autoscaling. Per-app instances support: instance count, shape, ocpus, memory, image OCID +(auto-detected if unset), boot volume size, public IP, ssh key, load balancer, and freeform tags. +Cloud-init / release bootstrapping (AWS's `cloud_init_data.yaml.tftpl`) is also not ported yet — +it needs the `oci` CLI instance-principal flow (Phase 3, `cli_adapter` in +`DeployEx.Cloud.Providers.Oci` is still `nil`), not the AMI-style `awscli` bootstrap AWS uses. + +## Load balancer + +Set `load_balancer = { enable = true, ... }` inside an app's entry in `_project` and +`mix terraform.apply` provisions an OCI Network Load Balancer (`oci_network_load_balancer_*`) in +`modules/oci-instance/load_balancer.tf` — one NLB per LB-enabled app, listening on 80 (and 443 +when `enable_https = true`) and forwarding to that app's instances. + +```hcl +load_balancer = { + enable = true + enable_https = false + reserved_ip_ocid = null + + health_check = { + path = "/health" + return_code = 200 + https_return_code = 200 + unhealthy_threshold = 3 + timeout = 3 + interval = 10 + } +} +``` + +**Gate divergence (D1):** AWS only creates a load balancer when +`enable && (instance_count > 1 || autoscaling.enable)` — OCI gates on `enable` alone. OCI has no +autoscaling and defaults `instance_count` to `1`, so copying AWS's gate would make +`load_balancer.enable = true` a silent no-op for the common single-instance case. + +**Security posture (D3):** the NSG that opens 80/443 is created *per app*, attached to both the +NLB and the LB-enabled app's own instances — not a rule on the shared `network.tf` security list. +`network.tf` is untouched by this feature. Consequence: **ports 80/443 are reachable directly on +an LB-backed app's instances**, not only through the NLB — OCI's NSG and security-list rules are +additive (union), so the LB-scoped NSG opens those ports on the instance's VNIC regardless of the +NLB path. Apps without `load_balancer.enable = true` are unaffected. + +**Portability (D6):** `health_check.return_code` and `health_check.https_return_code` are +OCI-only keys. An AWS-shaped `load_balancer` block runs unchanged on OCI, but an OCI block that +sets `return_code` / `https_return_code` is rejected by AWS's typed `variables.tf` — portability +is one-way (AWS -> OCI, not OCI -> AWS). + +**Keys AWS has that OCI ignores**, and why: + +| Key | Why ignored on OCI | +|---|---| +| `port` | Dead on AWS too — declared but never wired into a target group. OCI's listener is always 80/443. | +| `instance_port` | Same as above — always falls back to 80/443 on both providers. | +| `health_check.protocol` | AWS hardcodes `HTTP` on the 80 check and `HTTPS` on the 443 check. OCI derives the same thing: `HTTP`/`HTTPS` when `health_check.path` is set, `TCP` (connect-only) when it is not — OCI's health checker cannot be omitted the way AWS's can. | +| `health_check.matcher` / `health_check.https_matcher` | AWS accepts a status-code *range* string (`"200-299,301"`). OCI's `return_code` is a single number — not representable, so it is a distinct key rather than a lossy mapping. | +| `health_check.healthy_threshold` | OCI has one threshold (`retries`) that governs both directions — see `health_check.unhealthy_threshold` below. | + +**Keys that map directly:** + +| Key | Maps to | +|---|---| +| `health_check.unhealthy_threshold` | `health_checker.retries` — OCI's own docs describe this as the retry count before *and* after a state flip, covering both AWS thresholds | +| `health_check.timeout` | `health_checker.timeout_in_millis` (seconds × 1000) | +| `health_check.interval` | `health_checker.interval_in_millis` (seconds × 1000) | + +An unset health-check key passes `null` and takes OCI's own provider default (3 retries, 3s +timeout, 10s interval) rather than transplanting AWS's defaults (2/5s/20s) — deliberate, to avoid +a second set of magic numbers to keep in sync across providers. + +`reserved_ip_ocid` is OCI-only, with no AWS equivalent: an NLB's public IP is ephemeral by +default and can change on NLB replacement. Set it to a pre-created reserved public IP OCID to +pin the address DNS targets. + +## Operator notes — adopting the load balancer on an existing tree + +- **Never `--force` an adopted tree.** `mix terraform.build --force` discards hand-edited drift + in `variables.tf` (including any `load_balancer` block you've already enabled) and can close + public ingress mid-cutover. Rebuild without `--force`, then re-apply your `load_balancer` + edits if the regenerated defaults collided with them. +- **Opt-in flags are part of tree identity.** `--clickhouse` / `--rabbitmq` (and any other + opt-in rebuild flag) must be passed on *every* `terraform.build` run against a tree that + already has those nodes — a bare rebuild plans their destruction. +- **The load-balancer enable bit lives in the regenerated default map.** Enabling it is declared + drift against the generator's default output, the same way any other hand-edited + `_project` value is — expected, not a bug to chase. + +## Use + +```bash +cp terraform.tfvars.example terraform.tfvars # fill in — gitignored +tofu init +tofu plan +tofu apply +tofu destroy +``` + +Auth is an OCI API key (non-interactive, no browser). Generate and upload one with: + +```bash +openssl genrsa -out ~/.oci/oci_api_key.pem 2048 +openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem +oci iam user api-key upload --user-id --key-file ~/.oci/oci_api_key_public.pem \ + --region +``` + +A freshly uploaded key takes a minute or two to propagate; `NotAuthenticated` immediately after +upload usually means "wait", not "wrong". + +## OCI constraints that differ from AWS + +Both of these failed mid-apply during development, with resources already created: + +- **`dns_label` on a VCN is capped at 15 characters**, must be alphanumeric, and must start with a + letter. AWS VPCs have no equivalent, so a project name that is fine on AWS breaks VCN creation + here. `providers.tf` normalizes and clamps it rather than assuming the name fits. +- **OCI rejects every CIDR inside `0.0.0.0/8`.** The AWS habit of using `0.0.0.0/32` as a + "matches nothing" sentinel is invalid. Absence is expressed by omitting the rule entirely — the + SSH ingress rule is a `dynamic` block that produces nothing when `ssh_ingress_cidr` is empty. + +Two more things worth knowing: + +- **IAM writes go to the tenancy's HOME region**, which is often not where resources live. + Creating compartments, API keys, dynamic groups or policies against the wrong region fails with + `NotAllowed — "Please go to your home region"`. +- **Some regions have exactly one availability domain** (ap-seoul-1, ap-chuncheon-1), so there is + no multi-AD spread to configure. + +## Release bucket + instance principals + +`bucket.tf` creates one `oci_objectstorage_bucket` for releases — there is no separate +release-state bucket, since `release-state` is just an object prefix inside this same bucket +(matches `priv/ansible/roles/deploy_node/defaults/main.yaml`). `iam.tf` creates the dynamic +group (matches every instance in `compartment_ocid`) and policy (read on the bucket, manage on +its `release-state/*` prefix) instances need to read/write it via instance principals — no +Customer Secret Keys involved. Both IAM resources use the `oci.home` provider alias (see next +section) since they are IAM writes. + +## State + +Local state, deliberately. This is a throwaway environment; remote state belongs with the real +backend work rather than pointing at an existing bucket. diff --git a/priv/terraform/providers/oci/bucket.tf b/priv/terraform/providers/oci/bucket.tf new file mode 100644 index 00000000..fc723d65 --- /dev/null +++ b/priv/terraform/providers/oci/bucket.tf @@ -0,0 +1,23 @@ +# Release bucket. There is no separate "release state" bucket — release-state is an object +# prefix inside this same bucket (see priv/ansible/roles/deploy_node/defaults/main.yaml +# release_state_prefix), matching the AWS shape exactly. +# +# NOTE: OCI has no equivalent of aws_s3_bucket's `force_destroy`. A destroy against a bucket +# holding releases fails with "409-BucketNotEmpty, Bucket ... is not empty" — MEASURED — and +# leaves the bucket behind after every other resource is already gone. Emptying it first is +# left as a deliberate operator step rather than automated: the AWS side force-destroys +# release history on teardown, and silently doing the same here would delete every release +# artifact plus the current_release/release_history markers with no confirmation. +# +# To tear down completely: +# oci os object bulk-delete --bucket-name --force +# mix terraform.drop +resource "oci_objectstorage_bucket" "releases" { + compartment_id = var.compartment_ocid + namespace = var.namespace + name = var.release_bucket_name + + freeform_tags = merge(local.common_tags, { + Name = "Releases" + }) +} diff --git a/priv/terraform/providers/oci/database.tf b/priv/terraform/providers/oci/database.tf new file mode 100644 index 00000000..4f2bba48 --- /dev/null +++ b/priv/terraform/providers/oci/database.tf @@ -0,0 +1,114 @@ +# OCI Database with PostgreSQL — the managed-postgres analogue of the AWS RDS path. +# One DB system per resource_databases entry; an empty map (the default) creates nothing, +# so this needs no render-time gating the way the AWS template does. + +resource "random_password" "psql_admin" { + for_each = var.resource_databases + + length = 32 + special = false +} + +# The psql service rejects public subnets outright (400: "is not a private subnet"), so the +# DB gets its own private subnet: no public IPs, and a route table with no routes — nothing +# outside the VCN can be reached from it, and nodes reach the DB endpoint intra-VCN. +resource "oci_core_route_table" "database_private" { + count = length(var.resource_databases) > 0 ? 1 : 0 + + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${local.name_prefix}-database-rt" + + freeform_tags = local.common_tags +} + +resource "oci_core_subnet" "database_private" { + count = length(var.resource_databases) > 0 ? 1 : 0 + + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + cidr_block = var.database_subnet_cidr + display_name = "${local.name_prefix}-database-subnet" + dns_label = "db" + prohibit_public_ip_on_vnic = true + route_table_id = oci_core_route_table.database_private[0].id + security_list_ids = [oci_core_security_list.public.id] + + freeform_tags = local.common_tags +} + +# The subnet security list only admits SSH, and OCI filters intra-subnet traffic too — the +# nodes reach postgres through this NSG, not through subnet membership. +resource "oci_core_network_security_group" "database" { + count = length(var.resource_databases) > 0 ? 1 : 0 + + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${local.name_prefix}-database-nsg" + + freeform_tags = local.common_tags +} + +resource "oci_core_network_security_group_security_rule" "database_ingress" { + count = length(var.resource_databases) > 0 ? 1 : 0 + + network_security_group_id = oci_core_network_security_group.database[0].id + direction = "INGRESS" + protocol = "6" + source = var.vcn_cidr + source_type = "CIDR_BLOCK" + + tcp_options { + destination_port_range { + min = 5432 + max = 5432 + } + } +} + +# Regional durability is only offered in 3-AD regions — single-AD regions (ap-seoul-1, +# ap-chuncheon-1) reject it with 400-InvalidParameter and require an explicit availability +# domain instead, so AD-pinned is the default here. The flex shape sizes via +# instance_ocpu_count/memory rather than a fixed-shape name. The admin password is generated +# into state — state is remote and private, matching the AWS random_password approach. +resource "oci_psql_db_system" "database" { + for_each = var.resource_databases + + compartment_id = var.compartment_ocid + display_name = "${each.value.name}-${var.environment}" + shape = try(each.value.shape, "PostgreSQL.VM.Standard.E6.Flex") + db_version = try(each.value.db_version, "16") + + instance_ocpu_count = try(each.value.instance_ocpu_count, 2) + instance_memory_size_in_gbs = try(each.value.instance_memory_size_in_gbs, 32) + + credentials { + username = each.value.database_username + + password_details { + password_type = "PLAIN_TEXT" + password = random_password.psql_admin[each.key].result + } + } + + network_details { + subnet_id = oci_core_subnet.database_private[0].id + nsg_ids = [oci_core_network_security_group.database[0].id] + } + + storage_details { + system_type = "OCI_OPTIMIZED_STORAGE" + is_regionally_durable = try(each.value.regionally_durable, false) + availability_domain = try(each.value.regionally_durable, false) ? null : var.availability_domain + } + + freeform_tags = local.common_tags +} + +output "databases" { + description = "Database Info" + value = { for k in sort(keys(var.resource_databases)) : k => { + "endpoint" : oci_psql_db_system.database[k].network_details[0].primary_db_endpoint_private_ip, + "username" : var.resource_databases[k].database_username + } } +} diff --git a/priv/terraform/providers/oci/iam.tf b/priv/terraform/providers/oci/iam.tf new file mode 100644 index 00000000..eceab75b --- /dev/null +++ b/priv/terraform/providers/oci/iam.tf @@ -0,0 +1,38 @@ +# Instance principal access to the release bucket. +# +# Dynamic groups and policies are IAM writes — MEASURED to always land in the tenancy's HOME +# region regardless of which region the resources they reference live in (see B3 in +# docs/superpowers/plans/2026-08-03-multi-cloud-oci.md). Both resources below use the `oci.home` +# provider alias declared in providers.tf. + +resource "oci_identity_dynamic_group" "instances" { + provider = oci.home + + compartment_id = var.tenancy_ocid + name = "${var.project_name}-${var.environment}-instances" + description = "Instances in the ${var.project_name} ${var.environment} compartment (release bucket access)" + + # Matches every instance in the resource compartment. Freeform-tag-based matching rules are + # not an option — OCI dynamic group rules only match defined tags, not freeform tags. + matching_rule = "ALL {instance.compartment.id = '${var.compartment_ocid}'}" + + freeform_tags = local.common_tags +} + +# Policy is attached at the resource compartment (not the tenancy root) so it grants no more +# than instances in this project actually need — least-privilege scope for the bucket they +# read releases from and write release-state markers to. +resource "oci_identity_policy" "instance_release_bucket_access" { + provider = oci.home + + compartment_id = var.compartment_ocid + name = "${var.project_name}-${var.environment}-release-bucket-access" + description = "Lets ${var.project_name} ${var.environment} instances read releases and write release-state" + + statements = [ + "Allow dynamic-group ${oci_identity_dynamic_group.instances.name} to read objects in compartment id ${var.compartment_ocid} where target.bucket.name = '${var.release_bucket_name}'", + "Allow dynamic-group ${oci_identity_dynamic_group.instances.name} to manage objects in compartment id ${var.compartment_ocid} where all {target.bucket.name = '${var.release_bucket_name}', target.object.name = 'release-state/*'}", + ] + + freeform_tags = local.common_tags +} diff --git a/priv/terraform/providers/oci/instance.tf.eex b/priv/terraform/providers/oci/instance.tf.eex new file mode 100644 index 00000000..1671e21e --- /dev/null +++ b/priv/terraform/providers/oci/instance.tf.eex @@ -0,0 +1,53 @@ +# Latest Ubuntu 24.04 image for the configured shape, mirroring the AWS `data.aws_ami.base` +# fallback. Per-app entries may still pin `image_ocid` explicitly. +data "oci_core_images" "base" { + compartment_id = var.compartment_ocid + operating_system = "Canonical Ubuntu" + operating_system_version = "24.04" + shape = var.instance_shape + sort_by = "TIMECREATED" + sort_order = "DESC" +} + +module "oci_instance" { + source = "./modules/oci-instance" + + for_each = var.<%= @app_name %>_project + + resource_group = var.resource_group + environment = var.environment + compartment_ocid = var.compartment_ocid + availability_domain = var.availability_domain + subnet_id = oci_core_subnet.public.id + nsg_ids = [oci_core_network_security_group.ssh.id] + vcn_id = oci_core_vcn.main.id + + instance_name = each.value.name + instance_count = try(each.value.instance_count, null) + + instance_shape = try(each.value.shape, var.instance_shape) + instance_ocpus = try(each.value.ocpus, var.instance_ocpus) + instance_memory_gbs = try(each.value.memory_gbs, var.instance_memory_gbs) + + instance_image_ocid = coalesce( + try(each.value.image_ocid, null), + var.instance_image_ocid, + try(data.oci_core_images.base.images[0].id, null) + ) + + boot_volume_size_gbs = try(each.value.boot_volume_size_gbs, null) + assign_public_ip = try(each.value.assign_public_ip, var.assign_public_ip) + ssh_public_key = local.ssh_public_key + + enable_load_balancer = try(each.value.load_balancer.enable, false) + enable_load_balancer_https = try(each.value.load_balancer.enable_https, true) + load_balancer_health_check_path = try(each.value.load_balancer.health_check.path, "") + load_balancer_health_check_return_code = try(each.value.load_balancer.health_check.return_code, null) + load_balancer_health_check_https_return_code = try(each.value.load_balancer.health_check.https_return_code, null) + load_balancer_health_check_retries = try(each.value.load_balancer.health_check.unhealthy_threshold, null) + load_balancer_health_check_timeout_seconds = try(each.value.load_balancer.health_check.timeout, null) + load_balancer_health_check_interval_seconds = try(each.value.load_balancer.health_check.interval, null) + reserved_ip_ocid = try(each.value.load_balancer.reserved_ip_ocid, null) + + tags = try(each.value.tags, {}) +} diff --git a/priv/terraform/providers/oci/key-pair.tf.eex b/priv/terraform/providers/oci/key-pair.tf.eex new file mode 100644 index 00000000..d1d58eec --- /dev/null +++ b/priv/terraform/providers/oci/key-pair.tf.eex @@ -0,0 +1,35 @@ +# Generate PEM +# +# Mirrors the AWS tree's key-pair-main.tf so `mix ansible.build` finds a private key under +# the `*pem` glob it searches. Without this an operator must supply +# `ssh_public_key` AND hand-name the matching private key, with nothing telling them the +# convention — ansible.build fails the PEM lookup only after it has already written the +# inventory, and ansible.setup then dies with a bare "command failed" because ansible.cfg +# was never written. +# +# Unlike AWS there is no key-pair RESOURCE to create: OCI injects public keys through +# instance metadata, so this generates and saves the key, and instance.tf does the +# injecting via local.ssh_public_key. +# +# Supplying `ssh_public_key` in tfvars skips generation entirely — a bring-your-own key +# stays supported, and no private key is written in that case. +resource "tls_private_key" "key_pair" { + count = var.ssh_public_key == "" ? 1 : 0 + + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "local_file" "ssh_key" { + count = var.ssh_public_key == "" ? 1 : 0 + + filename = "<%= @pem_app_name %>-key-pair.pem" + content = tls_private_key.key_pair[0].private_key_pem + file_permission = "0400" +} + +locals { + # Empty when neither supplied nor generated, which instance.tf treats as "install no key" + # rather than passing an empty ssh_authorized_keys and silently making the box unreachable. + ssh_public_key = var.ssh_public_key != "" ? var.ssh_public_key : try(tls_private_key.key_pair[0].public_key_openssh, "") +} diff --git a/priv/terraform/providers/oci/modules/oci-instance/load_balancer.tf b/priv/terraform/providers/oci/modules/oci-instance/load_balancer.tf new file mode 100644 index 00000000..d835a4d8 --- /dev/null +++ b/priv/terraform/providers/oci/modules/oci-instance/load_balancer.tf @@ -0,0 +1,163 @@ +locals { + lb_count = var.enable_load_balancer ? 1 : 0 + lb_https_count = var.enable_load_balancer && var.enable_load_balancer_https ? 1 : 0 + lb_has_path = var.load_balancer_health_check_path != "" + + lb_timeout_in_millis = var.load_balancer_health_check_timeout_seconds == null ? null : var.load_balancer_health_check_timeout_seconds * 1000 + lb_interval_in_millis = var.load_balancer_health_check_interval_seconds == null ? null : var.load_balancer_health_check_interval_seconds * 1000 +} + +resource "oci_core_network_security_group" "load_balancer" { + count = local.lb_count + + compartment_id = var.compartment_ocid + vcn_id = var.vcn_id + display_name = "${local.kebab_instance_name}-lb-nsg" + + freeform_tags = merge({ + Name = "${local.kebab_instance_name}-lb-nsg" + Group = var.resource_group + InstanceGroup = local.snake_instance_name + Environment = var.environment + ManagedBy = "DeployEx" + }, var.tags) +} + +resource "oci_core_network_security_group_security_rule" "load_balancer_http" { + count = local.lb_count + + network_security_group_id = oci_core_network_security_group.load_balancer[0].id + direction = "INGRESS" + protocol = "6" + source = "0.0.0.0/0" + source_type = "CIDR_BLOCK" + + tcp_options { + destination_port_range { + min = 80 + max = 80 + } + } +} + +resource "oci_core_network_security_group_security_rule" "load_balancer_https" { + count = local.lb_https_count + + network_security_group_id = oci_core_network_security_group.load_balancer[0].id + direction = "INGRESS" + protocol = "6" + source = "0.0.0.0/0" + source_type = "CIDR_BLOCK" + + tcp_options { + destination_port_range { + min = 443 + max = 443 + } + } +} + +resource "oci_network_load_balancer_network_load_balancer" "main" { + count = local.lb_count + + compartment_id = var.compartment_ocid + display_name = "${local.kebab_instance_name}-nlb" + subnet_id = var.subnet_id + is_private = false + + network_security_group_ids = oci_core_network_security_group.load_balancer[*].id + + # Unset (null) leaves the public IP EPHEMERAL — it can change on NLB replacement. Only wired + # when the caller supplies an already-created reserved public IP OCID. + dynamic "reserved_ips" { + for_each = var.reserved_ip_ocid == null ? [] : [var.reserved_ip_ocid] + + content { + id = reserved_ips.value + } + } + + freeform_tags = merge({ + Name = "${local.kebab_instance_name}-nlb" + Group = var.resource_group + InstanceGroup = local.snake_instance_name + Environment = var.environment + ManagedBy = "DeployEx" + }, var.tags) +} + +resource "oci_network_load_balancer_backend_set" "http" { + count = local.lb_count + + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.main[0].id + name = "http" + policy = "FIVE_TUPLE" + is_preserve_source = true + + health_checker { + protocol = local.lb_has_path ? "HTTP" : "TCP" + port = 80 + url_path = local.lb_has_path ? var.load_balancer_health_check_path : null + return_code = local.lb_has_path ? coalesce(var.load_balancer_health_check_return_code, 200) : null + retries = var.load_balancer_health_check_retries + timeout_in_millis = local.lb_timeout_in_millis + interval_in_millis = local.lb_interval_in_millis + } +} + +resource "oci_network_load_balancer_backend_set" "https" { + count = local.lb_https_count + + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.main[0].id + name = "https" + policy = "FIVE_TUPLE" + is_preserve_source = true + + health_checker { + protocol = local.lb_has_path ? "HTTPS" : "TCP" + port = 443 + url_path = local.lb_has_path ? var.load_balancer_health_check_path : null + return_code = local.lb_has_path ? coalesce(var.load_balancer_health_check_https_return_code, 200) : null + retries = var.load_balancer_health_check_retries + timeout_in_millis = local.lb_timeout_in_millis + interval_in_millis = local.lb_interval_in_millis + } +} + +resource "oci_network_load_balancer_backend" "http" { + count = local.lb_count * var.instance_count + + backend_set_name = oci_network_load_balancer_backend_set.http[0].name + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.main[0].id + target_id = oci_core_instance.main[count.index].id + port = 80 +} + +resource "oci_network_load_balancer_backend" "https" { + count = local.lb_https_count * var.instance_count + + backend_set_name = oci_network_load_balancer_backend_set.https[0].name + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.main[0].id + target_id = oci_core_instance.main[count.index].id + port = 443 +} + +resource "oci_network_load_balancer_listener" "http" { + count = local.lb_count + + name = "http" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.main[0].id + default_backend_set_name = oci_network_load_balancer_backend_set.http[0].name + port = 80 + protocol = "TCP" +} + +resource "oci_network_load_balancer_listener" "https" { + count = local.lb_https_count + + name = "https" + network_load_balancer_id = oci_network_load_balancer_network_load_balancer.main[0].id + default_backend_set_name = oci_network_load_balancer_backend_set.https[0].name + port = 443 + protocol = "TCP" +} diff --git a/priv/terraform/providers/oci/modules/oci-instance/main.tf b/priv/terraform/providers/oci/modules/oci-instance/main.tf new file mode 100644 index 00000000..2f57ad3c --- /dev/null +++ b/priv/terraform/providers/oci/modules/oci-instance/main.tf @@ -0,0 +1,44 @@ +locals { + snake_instance_name = lower(replace(var.instance_name, " ", "_")) + kebab_instance_name = lower(replace(var.instance_name, " ", "-")) +} + +resource "oci_core_instance" "main" { + count = var.instance_count + + compartment_id = var.compartment_ocid + availability_domain = var.availability_domain + display_name = "${var.instance_name}-${var.environment}-${count.index}" + shape = var.instance_shape + + shape_config { + ocpus = var.instance_ocpus + memory_in_gbs = var.instance_memory_gbs + } + + source_details { + source_type = "image" + source_id = var.instance_image_ocid + boot_volume_size_in_gbs = var.boot_volume_size_gbs + } + + create_vnic_details { + subnet_id = var.subnet_id + nsg_ids = concat(var.nsg_ids, oci_core_network_security_group.load_balancer[*].id) + assign_public_ip = var.assign_public_ip + display_name = "${local.kebab_instance_name}-vnic-${count.index}" + hostname_label = "${local.kebab_instance_name}-${count.index}" + } + + # ssh_authorized_keys is omitted entirely when no key is supplied — passing an empty string + # makes the instance unreachable with no indication why. + metadata = var.ssh_public_key == "" ? {} : { ssh_authorized_keys = var.ssh_public_key } + + freeform_tags = merge({ + Name = "${var.instance_name}-${var.environment}-${count.index}" + Group = var.resource_group + InstanceGroup = local.snake_instance_name + Environment = var.environment + ManagedBy = "DeployEx" + }, var.tags) +} diff --git a/priv/terraform/providers/oci/modules/oci-instance/outputs.tf b/priv/terraform/providers/oci/modules/oci-instance/outputs.tf new file mode 100644 index 00000000..3ab0c788 --- /dev/null +++ b/priv/terraform/providers/oci/modules/oci-instance/outputs.tf @@ -0,0 +1,22 @@ +output "instance_ids" { + description = "OCIDs of the created instances" + value = oci_core_instance.main[*].id +} + +output "public_ips" { + description = "Public IPs of the created instances" + value = oci_core_instance.main[*].public_ip +} + +output "private_ips" { + description = "Private IPs of the created instances" + value = oci_core_instance.main[*].private_ip +} + +output "load_balancer_public_ips" { + description = "Public IPs of the load balancer. Empty list when enable_load_balancer is false." + value = [ + for ip in flatten(oci_network_load_balancer_network_load_balancer.main[*].ip_addresses) : ip.ip_address + if ip.is_public + ] +} diff --git a/priv/terraform/providers/oci/modules/oci-instance/variables.tf b/priv/terraform/providers/oci/modules/oci-instance/variables.tf new file mode 100644 index 00000000..c6569d28 --- /dev/null +++ b/priv/terraform/providers/oci/modules/oci-instance/variables.tf @@ -0,0 +1,177 @@ +### General ### +############### + +variable "resource_group" { + description = "Group tag for all resources" + type = string + nullable = false +} + +variable "environment" { + description = "Environment tag value" + type = string + nullable = false +} + +variable "compartment_ocid" { + description = "Compartment OCID instances are created in" + type = string + nullable = false +} + +variable "availability_domain" { + description = "Availability domain name" + type = string + nullable = false +} + +variable "subnet_id" { + description = "Subnet OCID instances attach to" + type = string + nullable = false +} + +variable "nsg_ids" { + description = "Network security group OCIDs attached to each instance's VNIC" + type = list(string) + default = [] + nullable = false +} + +variable "instance_name" { + description = "Instance name itself" + type = string + nullable = false +} + +variable "tags" { + description = "Freeform tags to merge onto every resource" + type = map(any) + default = {} + nullable = false +} + +### Instances ### +################# + +variable "instance_count" { + description = "Instance count, default 1" + type = number + default = 1 + nullable = false +} + +variable "instance_shape" { + description = "Flex shape, default VM.Standard.E5.Flex" + type = string + default = "VM.Standard.E5.Flex" + nullable = false +} + +variable "instance_ocpus" { + description = "OCPU count for the flex shape, default 1" + type = number + default = 1 + nullable = false +} + +variable "instance_memory_gbs" { + description = "Memory in GB for the flex shape, default 8" + type = number + default = 8 + nullable = false +} + +variable "instance_image_ocid" { + description = "Image OCID" + type = string + nullable = false +} + +variable "boot_volume_size_gbs" { + description = "Boot volume size in GB, default 50 (OCI's minimum)" + type = number + default = 50 + nullable = false +} + +variable "assign_public_ip" { + description = "Whether instances get a public IP" + type = bool + default = true + nullable = false +} + +variable "ssh_public_key" { + description = "Public key injected into the instances. Empty installs no key, making them unreachable by SSH." + type = string + default = "" + nullable = false +} + +### Load balancer ### +###################### + +variable "vcn_id" { + description = "VCN OCID the load-balancer NSG attaches to. Only used when enable_load_balancer is true." + type = string + default = "" + nullable = false +} + +variable "enable_load_balancer" { + description = "Whether to create a network load balancer for this app. Gates on this flag alone, unlike AWS's enable_elb && (instance_count > 1 || autoscaling) — OCI has no autoscaling and instance_count defaults to 1, so copying AWS's gate would make the flag a silent no-op for the common case." + type = bool + default = false + nullable = false +} + +variable "enable_load_balancer_https" { + description = "Whether to also create the 443 backend set, backend, and listener" + type = bool + default = true + nullable = false +} + +variable "load_balancer_health_check_path" { + description = "HTTP(S) health check path. Empty performs a TCP connect check instead of an HTTP(S) one." + type = string + default = "" + nullable = false +} + +variable "load_balancer_health_check_return_code" { + description = "Expected HTTP return code for the 80 health check when load_balancer_health_check_path is set. Unset takes 200." + type = number + default = null +} + +variable "load_balancer_health_check_https_return_code" { + description = "Expected HTTP return code for the 443 health check when load_balancer_health_check_path is set. Unset takes 200." + type = number + default = null +} + +variable "load_balancer_health_check_retries" { + description = "Health check retries before a backend is marked unhealthy. Unset takes the provider default (3)." + type = number + default = null +} + +variable "load_balancer_health_check_timeout_seconds" { + description = "Health check timeout in seconds. Unset takes the provider default (3s)." + type = number + default = null +} + +variable "load_balancer_health_check_interval_seconds" { + description = "Health check interval in seconds. Unset takes the provider default (10s)." + type = number + default = null +} + +variable "reserved_ip_ocid" { + description = "OCID of a reserved public IP to attach to the NLB. Unset leaves the public IP ephemeral." + type = string + default = null +} diff --git a/priv/terraform/providers/oci/modules/oci-instance/versions.tf b/priv/terraform/providers/oci/modules/oci-instance/versions.tf new file mode 100644 index 00000000..d8d3db67 --- /dev/null +++ b/priv/terraform/providers/oci/modules/oci-instance/versions.tf @@ -0,0 +1,12 @@ +# Explicit on purpose: without this, a child module that only implies its provider from a +# resource-type prefix (oci_core_instance -> "oci") falls back to the LEGACY hashicorp/oci +# registry namespace instead of inheriting the root's oracle/oci requirement — MEASURED, +# `tofu init` tries to resolve both and downloads hashicorp/oci alongside oracle/oci. +terraform { + required_providers { + oci = { + source = "oracle/oci" + version = "~> 7.0" + } + } +} diff --git a/priv/terraform/providers/oci/network.tf b/priv/terraform/providers/oci/network.tf new file mode 100644 index 00000000..a82c3475 --- /dev/null +++ b/priv/terraform/providers/oci/network.tf @@ -0,0 +1,104 @@ +resource "oci_core_vcn" "main" { + compartment_id = var.compartment_ocid + cidr_blocks = [var.vcn_cidr] + display_name = "${local.name_prefix}-vcn" + + # OCI caps dnsLabel at 15 chars, must be alphanumeric, and must start with a letter — none of + # which AWS imposes on a VPC. A project name that is fine on AWS silently breaks VCN creation + # here, so this normalizes rather than assuming the name already fits. + dns_label = local.vcn_dns_label + + freeform_tags = local.common_tags +} + +resource "oci_core_internet_gateway" "main" { + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${local.name_prefix}-igw" + enabled = true + + freeform_tags = local.common_tags +} + +resource "oci_core_route_table" "public" { + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${local.name_prefix}-rt" + + route_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_internet_gateway.main.id + } + + freeform_tags = local.common_tags +} + +# OCI security lists are stateful, so only ingress needs enumerating for inbound flows. +resource "oci_core_security_list" "public" { + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${local.name_prefix}-sl" + + egress_security_rules { + destination = "0.0.0.0/0" + protocol = "all" + } + + # Unlike AWS, OCI filters traffic BETWEEN instances on the same subnet — with only the SSH + # rule below, an app node cannot reach redis (6379), clickhouse (8123) or any BEAM peer + # (measured: 6379 refused server → redis). This is the analogue of the AWS app security + # group's self-referencing rule: everything inside the VCN trusts everything else, and only + # the public edge is gated. Includes ICMP so path-MTU discovery works between nodes. + ingress_security_rules { + source = var.vcn_cidr + protocol = "all" + } + + # No SSH rule at all when no CIDR is supplied. OCI rejects any CIDR inside 0.0.0.0/8, so the + # AWS trick of using 0.0.0.0/32 as a "matches nothing" sentinel is invalid here — absence has + # to be expressed by omitting the rule. protocol 6 is TCP. + dynamic "ingress_security_rules" { + for_each = var.ssh_ingress_cidr == "" ? [] : [var.ssh_ingress_cidr] + + content { + source = ingress_security_rules.value + protocol = "6" + + tcp_options { + min = 22 + max = 22 + } + } + } + + freeform_tags = local.common_tags +} + +# Holds the SSH ingress rule `mix deploy_ex.ssh.authorize` toggles. A network security group +# rather than a second security list: `oci network nsg rules add`/`remove` operate on individual +# rules by content or ID, so the whitelist toggle never has to read the full rule set and write +# it back the way `oci_core_security_list` above would require. No default rules here — the +# security list already grants open egress and any static SSH CIDR to every instance in the +# subnet; this NSG exists solely for the dynamic per-run rule. See +# `DeployEx.Cloud.OciSecurityGroup` for the client that manages it. +resource "oci_core_network_security_group" "ssh" { + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${local.name_prefix}-nsg" + + freeform_tags = local.common_tags +} + +resource "oci_core_subnet" "public" { + compartment_id = var.compartment_ocid + vcn_id = oci_core_vcn.main.id + cidr_block = var.subnet_cidr + display_name = "${local.name_prefix}-subnet" + dns_label = "public" + route_table_id = oci_core_route_table.public.id + security_list_ids = [oci_core_security_list.public.id] + prohibit_public_ip_on_vnic = !var.assign_public_ip + + freeform_tags = local.common_tags +} diff --git a/priv/terraform/providers/oci/outputs.tf b/priv/terraform/providers/oci/outputs.tf new file mode 100644 index 00000000..3952c97e --- /dev/null +++ b/priv/terraform/providers/oci/outputs.tf @@ -0,0 +1,39 @@ +output "vcn_id" { + description = "VCN OCID" + value = oci_core_vcn.main.id +} + +output "subnet_id" { + description = "Public subnet OCID" + value = oci_core_subnet.public.id +} + +output "ssh_nsg_id" { + description = "OCID of the network security group mix deploy_ex.ssh.authorize manages" + value = oci_core_network_security_group.ssh.id +} + +output "instance_ids" { + description = "Compute instance OCIDs, keyed by app name" + value = { for app, mod in module.oci_instance : app => mod.instance_ids } +} + +output "instance_public_ips" { + description = "Public IPs, keyed by app name (empty when assign_public_ip is false)" + value = { for app, mod in module.oci_instance : app => mod.public_ips } +} + +output "instance_private_ips" { + description = "Private IPs within the subnet, keyed by app name" + value = { for app, mod in module.oci_instance : app => mod.private_ips } +} + +output "load_balancer_public_ips" { + description = "Load balancer public IPs, keyed by app name (empty list when load_balancer.enable is false)" + value = { for app, mod in module.oci_instance : app => mod.load_balancer_public_ips } +} + +output "release_bucket_name" { + description = "Name of the release bucket" + value = oci_objectstorage_bucket.releases.name +} diff --git a/priv/terraform/providers/oci/providers.tf.eex b/priv/terraform/providers/oci/providers.tf.eex new file mode 100644 index 00000000..4c6c1650 --- /dev/null +++ b/priv/terraform/providers/oci/providers.tf.eex @@ -0,0 +1,71 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + oci = { + source = "oracle/oci" + version = "~> 7.0" + } + } + +<%= if @terraform_backend === :s3 and @oci_state_bucket do %> + # OCI has no native terraform backend — state goes through the S3-compatibility endpoint. + # Auth is a Customer Secret Key (an AWS-shaped credential pair), NOT the OCI API key: the + # compat endpoint does not accept OCI signatures or instance principals. The skip_* flags + # exist because this endpoint is not AWS — no STS, no metadata service, no AWS account — + # and skip_s3_checksum because OCI rejects the checksum headers newer AWS SDKs send. + # No lock table: OCI has no DynamoDB, so concurrent applies are unguarded. + backend "s3" { + bucket = "<%= @oci_state_bucket %>" + key = "<%= @oci_state_key %>" + region = "<%= @oci_region %>"<%= if @oci_state_profile do %> + profile = "<%= @oci_state_profile %>"<% end %> + + endpoints = { + s3 = "https://<%= @oci_namespace %>.compat.objectstorage.<%= @oci_region %>.oraclecloud.com" + } + + skip_region_validation = true + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true + skip_s3_checksum = true + use_path_style = true + } +<% end %>} + +provider "oci" { + tenancy_ocid = var.tenancy_ocid + user_ocid = var.user_ocid + fingerprint = var.fingerprint + private_key_path = pathexpand(var.private_key_path) + region = var.region +} + +# IAM writes (dynamic groups, policies) always land in the tenancy's home region — see iam.tf. +provider "oci" { + alias = "home" + + tenancy_ocid = var.tenancy_ocid + user_ocid = var.user_ocid + fingerprint = var.fingerprint + private_key_path = pathexpand(var.private_key_path) + region = var.home_region +} + +locals { + name_prefix = "${var.project_name}-${var.environment}" + + # Strip everything OCI disallows, then clamp to the 15-char limit. + vcn_dns_label = substr( + lower(replace(local.name_prefix, "/[^A-Za-z0-9]/", "")), + 0, + 15 + ) + + common_tags = { + "Group" = var.resource_group + "Environment" = var.environment + "ManagedBy" = "DeployEx" + } +} diff --git a/priv/terraform/providers/oci/terraform.tfvars.example b/priv/terraform/providers/oci/terraform.tfvars.example new file mode 100644 index 00000000..bc7f6e6f --- /dev/null +++ b/priv/terraform/providers/oci/terraform.tfvars.example @@ -0,0 +1,30 @@ +# Copy to terraform.tfvars and fill in. terraform.tfvars is gitignored — it identifies your +# tenancy and user, so it must never be committed. +# +# Find these with: +# tenancy / user / fingerprint : cat ~/.oci/config +# compartment : oci iam compartment list --all +# availability_domain : oci iam availability-domain list --compartment-id +# namespace : oci os ns get +# home_region : oci iam region-subscription list (the one marked is-home-region) +# image : oci compute image list --compartment-id \ +# --operating-system "Canonical Ubuntu" \ +# --operating-system-version "24.04" --shape VM.Standard.E6.Flex + +tenancy_ocid = "ocid1.tenancy.oc1..aaaa..." +user_ocid = "ocid1.user.oc1..aaaa..." +fingerprint = "aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99" +private_key_path = "~/.oci/oci_api_key.pem" +compartment_ocid = "ocid1.compartment.oc1..aaaa..." + +region = "ap-seoul-1" +home_region = "ap-chuncheon-1" +availability_domain = "Abcd:AP-SEOUL-1-AD-1" +namespace = "axxxxxxxxxxx" + +# Optional — auto-detected as the latest Ubuntu 24.04 image for instance_shape when unset. +# instance_image_ocid = "ocid1.image.oc1.ap-seoul-1.aaaa..." + +# Port 22 stays closed until you set this. Use your own address, not 0.0.0.0/0. +# ssh_ingress_cidr = "203.0.113.4/32" +# ssh_public_key = "ssh-ed25519 AAAA..." diff --git a/priv/terraform/providers/oci/variables.tf.eex b/priv/terraform/providers/oci/variables.tf.eex new file mode 100644 index 00000000..39235a70 --- /dev/null +++ b/priv/terraform/providers/oci/variables.tf.eex @@ -0,0 +1,196 @@ +######################################## +# Identity / auth +# +# No defaults — these identify a specific tenancy and user. Supply them via terraform.tfvars +# (gitignored) or TF_VAR_* environment variables. See terraform.tfvars.example. +######################################## + +variable "tenancy_ocid" { + description = "OCID of the tenancy resources are billed to" + type = string +} + +variable "user_ocid" { + description = "OCID of the user whose API key signs requests" + type = string +} + +variable "fingerprint" { + description = "Fingerprint of the API public key uploaded to that user" + type = string +} + +variable "private_key_path" { + description = "Path to the API private key. Never commit this file." + type = string + default = "~/.oci/oci_api_key.pem" +} + +variable "compartment_ocid" { + description = "Compartment everything is created in. Use a dedicated one so teardown is contained." + type = string +} + +######################################## +# Placement +######################################## + +variable "region" { + description = "Region resources are created in. IAM writes always go to the tenancy's HOME region, which may differ." + type = string +} + +variable "home_region" { + description = "Tenancy's home region. IAM writes (dynamic groups, policies) always go here, regardless of `region`. Find it with: oci iam region-subscription list" + type = string +} + +variable "availability_domain" { + description = "Availability domain name. Some regions (ap-seoul-1, ap-chuncheon-1) have exactly one." + type = string +} + +variable "namespace" { + description = "Object storage namespace for the tenancy. Find it with: oci os ns get" + type = string +} + +######################################## +# Naming / tagging +######################################## + +variable "project_name" { + description = "Prefix for every resource name" + type = string + default = "<%= @kebab_app_name %>" +} + +variable "environment" { + description = "Environment tag value" + type = string + default = "<%= @environment %>" +} + +variable "resource_group" { + description = "Group tag value. OCI freeform tags permit spaces, so this is an identity encoding of the canonical tag." + type = string + default = "<%= DeployEx.Utils.upper_title_case(@app_name) %> Backend" +} + +######################################## +# Network +######################################## + +variable "vcn_cidr" { + description = "CIDR for the VCN. Rendered from config :deploy_ex, :oci, vcn_cidr so ansible's intra-network allowlists (clickhouse) agree with it." + type = string + default = "<%= @oci_vcn_cidr %>" +} + +variable "subnet_cidr" { + description = "CIDR for the public subnet" + type = string + default = "10.20.1.0/24" +} + +variable "database_subnet_cidr" { + description = "CIDR for the private database subnet — the managed psql service rejects public subnets. Only created when resource_databases is non-empty." + type = string + default = "10.20.2.0/24" +} + +variable "ssh_ingress_cidr" { + description = "Who may reach port 22. Empty creates NO ssh rule; OCI rejects any CIDR inside 0.0.0.0/8, so a sentinel CIDR is not an option." + type = string + default = "" +} + +######################################## +# Compute +######################################## + +variable "instance_shape" { + description = "Flex shape. VM.Standard.A1.Flex is the always-free ARM shape and needs an aarch64 image." + type = string + default = "VM.Standard.E6.Flex" +} + +variable "instance_ocpus" { + description = "OCPU count for the flex shape" + type = number + default = 1 +} + +variable "instance_memory_gbs" { + description = "Memory for the flex shape" + type = number + default = 8 +} + +variable "instance_image_ocid" { + description = "Image OCID override. Region-specific — list with: oci compute image list --compartment-id --operating-system 'Canonical Ubuntu'. Leave unset to auto-detect the latest Ubuntu 24.04 image for instance_shape." + type = string + default = null +} + +variable "ssh_public_key" { + description = "Bring-your-own public key to inject into instances. Leave empty (the default) and key-pair.tf generates a keypair and writes the private half next to this file, named so `mix ansible.build` finds it. Set this only if you manage the key yourself — no private key is written in that case." + type = string + default = "" +} + +variable "assign_public_ip" { + description = "Whether the instance gets a public IP" + type = bool + default = true +} + +######################################## +# Release bucket +######################################## + +variable "release_bucket_name" { + description = "Name of the object storage bucket holding release artifacts" + type = string + default = "<%= @release_bucket_name %>" +} + +######################################## +# Per-app instances +######################################## + +# Deliberately `any` rather than a fully-typed object(...) like the AWS `_project` +# variable. Entries legitimately differ in shape — only the redis entry carries a DatabaseKey +# tag, only grafana sets assign_public_ip, and a per-app entry may omit sizing entirely to +# inherit the top-level defaults. `map(any)` does not help: OpenTofu still unifies every value +# to ONE element type and fails with "all map elements must have the same type" the moment two +# entries diverge. `any` skips that unification. +# +# The cost is real and worth stating: unlike AWS's map(object({...})), NOTHING here rejects a +# misspelled or unsupported key. The oci-instance module reads what it understands via try() +# and ignores the rest, so `shappe = "..."` is accepted and quietly does nothing. +variable "<%= @app_name %>_project" { + description = "Map of project names to configuration. Recognized keys: name, instance_count, shape, ocpus, memory_gbs, image_ocid, boot_volume_size_gbs, assign_public_ip, load_balancer, tags. Unrecognized keys are ignored without error." + type = any + + default = { +<%= @terraform_sentry_variables %> +<%= @terraform_redis_variables %> +<%= @terraform_clickhouse_variables %> +<%= @terraform_rabbitmq_variables %> +<%= @terraform_grafana_variables %> +<%= @terraform_prometheus_variables %> +<%= @terraform_loki_variables %> +<%= @terraform_release_variables %> + } +} + +# Managed OCI Database with PostgreSQL instances — see database.tf. Same `any` reasoning as +# the project variable above. Recognized keys: name, database_username, and optionally shape, +# db_version, instance_ocpu_count, instance_memory_size_in_gbs. Empty map creates nothing. +variable "resource_databases" { + description = "Map of managed PostgreSQL databases. Recognized keys: name, database_username, shape, db_version, instance_ocpu_count, instance_memory_size_in_gbs." + type = any + + default = {} +} diff --git a/test/deploy_ex/ansible_xml_templates_test.exs b/test/deploy_ex/ansible_xml_templates_test.exs new file mode 100644 index 00000000..23de7ca6 --- /dev/null +++ b/test/deploy_ex/ansible_xml_templates_test.exs @@ -0,0 +1,47 @@ +defmodule DeployEx.AnsibleXmlTemplatesTest do + use ExUnit.Case, async: true + + # An XML config fragment that does not parse is not a config problem — it is a service that + # never starts. MEASURED: a `--` sequence inside an XML comment (XML 1.0 forbids it anywhere + # in a comment body) made ClickHouse's Poco parser refuse + # `users.d/zz-allow-default-network.xml` outright, so the daemon died before signalling + # readiness and systemd reported "Failed with result 'protocol'" — indistinguishable at the + # systemd layer from a Type=notify incompatibility, and it cost a long misdiagnosis. + # + # Ansible renders these with Jinja2, which nothing here can execute, so the Jinja constructs + # are substituted out and the surrounding XML is parsed. That is enough to catch malformed + # markup, unbalanced tags, and illegal comments — the failures that stop a daemon booting. + @templates Path.wildcard("priv/ansible/**/*.xml.j2") + + test "there are XML templates to check, so this test cannot pass vacuously" do + refute Enum.empty?(@templates) + end + + for template <- @templates do + test "#{template} renders parseable XML" do + xml = unquote(template) |> File.read!() |> strip_jinja() + + assert {:ok, _parsed} = parse_xml(xml), + "#{unquote(template)} does not parse as XML once Jinja constructs are removed. " <> + "A config.d/users.d fragment that fails to parse stops the service booting." + end + end + + defp strip_jinja(contents) do + contents + |> String.replace(~r/\{\#.*?\#\}/s, "") + |> String.replace(~r/\{%.*?%\}/s, "") + |> String.replace(~r/\{\{.*?\}\}/s, "placeholder") + end + + # xmerl is fed the UTF-8 BYTES, not Unicode codepoints. String.to_charlist/1 would hand it + # codepoints, and it rejects anything above its assumed single-byte range — an em-dash in a + # comment would fail as `bad_character, 8212` even though it is perfectly legal XML. + defp parse_xml(xml) do + {parsed, _rest} = :xmerl_scan.string(:binary.bin_to_list(xml), quiet: true) + + {:ok, parsed} + catch + kind, reason -> {:error, {kind, reason}} + end +end diff --git a/test/deploy_ex/aws_database_pagination_test.exs b/test/deploy_ex/aws_database_pagination_test.exs new file mode 100644 index 00000000..9b2e7766 --- /dev/null +++ b/test/deploy_ex/aws_database_pagination_test.exs @@ -0,0 +1,78 @@ +defmodule DeployEx.AwsDatabasePaginationTest do + @moduledoc """ + Behavioural test for AwsDatabase.fetch_aws_databases/1 pagination. + + DescribeDBInstances caps a response and signals more via `Marker`. A single request returns + `{:ok, partial}` silently, never an error — this test replaces the recursion with a single + request to prove the loop actually threads `:request_fn` through and consumes every page. + + Responses are queued in the process dictionary — per-PID isolated, no setup or teardown. + """ + + use ExUnit.Case, async: true + + alias DeployEx.AwsDatabase + + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + Process.put(:call_count, (Process.get(:call_count) || 0) + 1) + + head + end + + defp call_count, do: Process.get(:call_count) || 0 + + defp recording_request_fn do + fn _request, _config -> next_response() end + end + + defp rds_page(identifiers, marker) do + instances = + Enum.map_join(identifiers, "", fn identifier -> + "" <> + "#{identifier}" <> + "
#{identifier}.example.com
5432" <> + "postgres" <> + "#{identifier}db" <> + "" <> + "" + end) + + marker_xml = if marker, do: "#{marker}", else: "" + + {:ok, + %{ + body: + "" <> + "#{instances}#{marker_xml}" <> + "" + }} + end + + describe "fetch_aws_databases/1 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([rds_page(["db-1", "db-2"], "db-2"), rds_page(["db-3"], nil)]) + + assert {:ok, instances} = AwsDatabase.fetch_aws_databases(request_fn: recording_request_fn()) + + assert Enum.map(instances, & &1.identifier) === ["db-1", "db-2", "db-3"] + end + + test "terminates when the response omits Marker" do + queue_responses([rds_page(["only"], nil)]) + + assert {:ok, [_only]} = AwsDatabase.fetch_aws_databases(request_fn: recording_request_fn()) + assert call_count() === 1, "an absent Marker must not drive a second request" + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([rds_page(["db-1"], "db-1"), {:error, {"InternalError", "boom"}}]) + + assert {:error, %ErrorMessage{}} = + AwsDatabase.fetch_aws_databases(request_fn: recording_request_fn()) + end + end +end diff --git a/test/deploy_ex/aws_dynamodb_pagination_test.exs b/test/deploy_ex/aws_dynamodb_pagination_test.exs new file mode 100644 index 00000000..400bf079 --- /dev/null +++ b/test/deploy_ex/aws_dynamodb_pagination_test.exs @@ -0,0 +1,64 @@ +defmodule DeployEx.AwsDynamodbPaginationTest do + @moduledoc """ + Behavioural test for AwsDynamodb.list_tables/2 pagination. + + ListTables caps a response and signals more via `LastEvaluatedTableName`. A single request + returns `{:ok, partial}` silently, never an error — this test replaces the recursion with a + single request to prove the loop actually threads `:request_fn` through and consumes every + page. Live account has zero tables today, so this is the only way to exercise the loop. + + Responses are queued in the process dictionary — per-PID isolated, no setup or teardown. + """ + + use ExUnit.Case, async: true + + alias DeployEx.AwsDynamodb + + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + Process.put(:call_count, (Process.get(:call_count) || 0) + 1) + + head + end + + defp call_count, do: Process.get(:call_count) || 0 + + defp recording_request_fn do + fn _request, _config -> next_response() end + end + + defp dynamo_page(table_names, last_evaluated) do + body = + if last_evaluated, + do: %{"TableNames" => table_names, "LastEvaluatedTableName" => last_evaluated}, + else: %{"TableNames" => table_names} + + {:ok, body} + end + + describe "list_tables/2 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([dynamo_page(["a", "b"], "b"), dynamo_page(["c"], nil)]) + + assert AwsDynamodb.list_tables("us-east-1", request_fn: recording_request_fn()) === + {:ok, ["a", "b", "c"]} + end + + test "terminates when the response omits LastEvaluatedTableName" do + queue_responses([dynamo_page(["only"], nil)]) + + assert AwsDynamodb.list_tables("us-east-1", request_fn: recording_request_fn()) === {:ok, ["only"]} + assert call_count() === 1, "an absent LastEvaluatedTableName must not drive a second request" + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([dynamo_page(["a"], "a"), {:error, {:http_error, 500, "boom"}}]) + + assert {:error, %ErrorMessage{}} = + AwsDynamodb.list_tables("us-east-1", request_fn: recording_request_fn()) + end + end +end diff --git a/test/deploy_ex/aws_infrastructure_conformance_test.exs b/test/deploy_ex/aws_infrastructure_conformance_test.exs new file mode 100644 index 00000000..b8c61b26 --- /dev/null +++ b/test/deploy_ex/aws_infrastructure_conformance_test.exs @@ -0,0 +1,42 @@ +defmodule DeployEx.AwsInfrastructureConformanceTest do + use ExUnit.Case, async: true + + alias DeployEx.AwsInfrastructure + + test "declares the Cloud.Infrastructure behaviour" do + behaviours = AwsInfrastructure.module_info(:attributes)[:behaviour] || [] + + assert DeployEx.Cloud.Infrastructure in behaviours + end + + test "exports every callback the behaviour declares" do + Code.ensure_loaded!(AwsInfrastructure) + + missing = + DeployEx.Cloud.Infrastructure.behaviour_info(:callbacks) + |> Enum.reject(fn {name, arity} -> function_exported?(AwsInfrastructure, name, arity) end) + + assert missing === [], "AwsInfrastructure is missing callbacks: #{inspect(missing)}" + end + + test "the AWS descriptor resolves infrastructure to this module" do + assert DeployEx.Cloud.capability(:infrastructure) === {:ok, AwsInfrastructure} + end + + test "keeps the public functions its existing call sites use" do + Code.ensure_loaded!(AwsInfrastructure) + + for {name, arity} <- [ + find_subnet_ids: 1, + find_key_pair_name: 1, + find_iam_instance_profile: 1, + find_vpc_id: 1, + find_latest_ami: 1, + gather_infrastructure: 1, + find_primary_subnet_id: 2 + ] do + assert function_exported?(AwsInfrastructure, name, arity), + "AwsInfrastructure.#{name}/#{arity} disappeared" + end + end +end diff --git a/test/deploy_ex/aws_infrastructure_test.exs b/test/deploy_ex/aws_infrastructure_test.exs index 7ef59f93..719ad4c8 100644 --- a/test/deploy_ex/aws_infrastructure_test.exs +++ b/test/deploy_ex/aws_infrastructure_test.exs @@ -4,15 +4,33 @@ defmodule DeployEx.AwsInfrastructureTest do alias DeployEx.AwsInfrastructure describe "find_iam_instance_profile/1" do - test "returns expected profile name based on resource group" do - assert {:ok, "my-project-instance-profile"} === - AwsInfrastructure.find_iam_instance_profile(resource_group: "My_Project") + # This test used to call find_iam_instance_profile/1 with no request_fn, which meant the + # SUITE HIT THE REAL AWS IAM API — it failed with a list of actual profiles from whatever + # account happened to be configured. Tests must not depend on a live account. + test "an explicitly configured profile short-circuits the lookup entirely" do + never_called = fn _operation, _opts -> flunk("should not have called AWS") end + + assert AwsInfrastructure.find_iam_instance_profile( + iam_instance_profile: "preset-profile", + request_fn: never_called + ) === {:ok, "preset-profile"} + end - assert {:ok, "test-backend-instance-profile"} === - AwsInfrastructure.find_iam_instance_profile(resource_group: "Test Backend") + test "returns the environment default when AWS reports it exists" do + default = "deploy-ex-ec2-instance-profile-#{DeployEx.Config.env()}" - assert {:ok, "simple-instance-profile"} === - AwsInfrastructure.find_iam_instance_profile(resource_group: "Simple") + assert AwsInfrastructure.find_iam_instance_profile( + request_fn: instance_profiles_response([default, "unrelated"]) + ) === {:ok, default} + end + + test "reports what IS available when the default is absent, rather than a bare not_found" do + assert {:error, %ErrorMessage{code: :not_found} = error} = + AwsInfrastructure.find_iam_instance_profile( + request_fn: instance_profiles_response(["something-else"]) + ) + + assert error.details.available === ["something-else"] end end @@ -112,73 +130,39 @@ defmodule DeployEx.AwsInfrastructureTest do end end - describe "parse_key_pairs_response/2" do - test "parses key pair from list" do - xml = """ - - - - - my-project-key-pair - abc123 - - - other-key-pair - def456 - - - - """ + describe "find_key_pair_name/1" do + # parse_key_pairs_response/2 was deleted when key-pair parsing moved inline, but its four + # tests stayed and kept the suite red. These exercise the CURRENT public path instead. + test "picks the newest matching key pair" do + response = key_pairs_response(["my-project-AAA-key-pair", "my-project-ZZZ-key-pair"]) - assert {:ok, "my-project-key-pair"} === AwsInfrastructure.parse_key_pairs_response(xml, "my-project-key-pair") + assert AwsInfrastructure.find_key_pair_name( + project_name: "my-project", + request_fn: response + ) === {:ok, "my-project-ZZZ-key-pair"} end - test "parses single key pair" do - xml = """ - - - - - single-key - abc123 - - - - """ - - assert {:ok, "single-key"} === AwsInfrastructure.parse_key_pairs_response(xml, "single-key") + test "parses a single-element key set, which AWS returns unwrapped" do + assert AwsInfrastructure.find_key_pair_name( + project_name: "my-project", + request_fn: key_pairs_response(["my-project-solo-key-pair"]) + ) === {:ok, "my-project-solo-key-pair"} end - test "returns error when key pair not found in list" do - xml = """ - - - - - other-key - abc123 - - - another-key - def456 - - - - """ - - assert {:error, %ErrorMessage{code: :not_found, message: "key pair my-key not found"}} = - AwsInfrastructure.parse_key_pairs_response(xml, "my-key") + test "ignores key pairs belonging to other projects" do + assert {:error, %ErrorMessage{code: :not_found}} = + AwsInfrastructure.find_key_pair_name( + project_name: "my-project", + request_fn: key_pairs_response(["other-project-key-pair"]) + ) end - test "returns error for empty key set" do - xml = """ - - - - - """ - - assert {:error, %ErrorMessage{code: :not_found}} = AwsInfrastructure.parse_key_pairs_response(xml, "my-key") + test "an empty key set is not_found, not a crash" do + assert {:error, %ErrorMessage{code: :not_found}} = + AwsInfrastructure.find_key_pair_name( + project_name: "my-project", + request_fn: key_pairs_response([]) + ) end end @@ -283,4 +267,33 @@ defmodule DeployEx.AwsInfrastructureTest do assert {:error, %ErrorMessage{code: :not_found}} = AwsInfrastructure.parse_images_response(xml) end end + + defp instance_profiles_response(names) do + profiles = Enum.map_join(names, "", &"#{&1}") + + body = """ + + + false + #{profiles} + + + """ + + fn _operation, _opts -> {:ok, %{body: body}} end + end + + defp key_pairs_response(key_names) do + items = Enum.map_join(key_names, "", &"#{&1}") + key_set = if Enum.empty?(key_names), do: "", else: "#{items}" + + body = """ + + + #{key_set} + + """ + + fn _operation, _opts -> {:ok, %{body: body}} end + end end diff --git a/test/deploy_ex/aws_load_balancer_test.exs b/test/deploy_ex/aws_load_balancer_test.exs new file mode 100644 index 00000000..bc7d06af --- /dev/null +++ b/test/deploy_ex/aws_load_balancer_test.exs @@ -0,0 +1,79 @@ +defmodule DeployEx.AwsLoadBalancerTest do + @moduledoc """ + Behavioural tests for `describe_target_groups/1` pagination. + + DescribeTargetGroups caps a response and signals more via NextMarker. A single request + returns `{:ok, partial}` silently, never an error — this test replaces the recursion with a + single request to prove it actually threads `:request_fn` through and consumes every page. + """ + + use ExUnit.Case, async: true + + alias DeployEx.AwsLoadBalancer + + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + Process.put(:call_count, (Process.get(:call_count) || 0) + 1) + head + end + + defp call_count, do: Process.get(:call_count) || 0 + + defp recording_request_fn do + fn _request, _config -> next_response() end + end + + defp tg_page(names, next_marker) do + target_groups = + Enum.map(names, fn name -> + %{ + target_group_arn: "arn:aws:elasticloadbalancing:us-east-1:1:targetgroup/#{name}", + target_group_name: name, + port: 4000, + protocol: "HTTP", + vpc_id: "vpc-1", + health_check_path: "/health", + health_check_port: "4000", + health_check_protocol: "HTTP" + } + end) + + {:ok, %{body: %{target_groups: target_groups, next_marker: next_marker}}} + end + + describe "describe_target_groups/1 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([tg_page(["a"], "TOKEN"), tg_page(["b"], "")]) + + assert {:ok, target_groups} = AwsLoadBalancer.describe_target_groups(request_fn: recording_request_fn()) + + assert Enum.map(target_groups, & &1.name) === ["a", "b"] + end + + test "terminates when next_marker is the empty string" do + queue_responses([tg_page(["only"], "")]) + + assert {:ok, [_only]} = AwsLoadBalancer.describe_target_groups(request_fn: recording_request_fn()) + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([tg_page(["a"], "TOKEN"), {:error, {:http_error, 500, %{body: "boom"}}}]) + + assert {:error, %ErrorMessage{}} = + AwsLoadBalancer.describe_target_groups(request_fn: recording_request_fn()) + end + end + + describe "find_target_groups_by_app/2 pagination" do + test "a matching target group past page one is still found, not silently dropped" do + queue_responses([tg_page(["other-app"], "TOKEN"), tg_page(["my_app"], "")]) + + assert {:ok, [%{name: "my_app"}]} = + AwsLoadBalancer.find_target_groups_by_app("my_app", request_fn: recording_request_fn()) + end + end +end diff --git a/test/deploy_ex/aws_security_group_test.exs b/test/deploy_ex/aws_security_group_test.exs new file mode 100644 index 00000000..32681b0c --- /dev/null +++ b/test/deploy_ex/aws_security_group_test.exs @@ -0,0 +1,137 @@ +defmodule DeployEx.AwsSecurityGroupTest do + use ExUnit.Case, async: true + + alias DeployEx.AwsSecurityGroup + + describe "Cloud.Security conformance" do + test "declares the behaviour" do + assert DeployEx.Cloud.Security in (AwsSecurityGroup.module_info(:attributes)[:behaviour] || []) + end + + test "exports every callback the behaviour declares" do + Code.ensure_loaded!(AwsSecurityGroup) + + missing = + DeployEx.Cloud.Security.behaviour_info(:callbacks) + |> Enum.reject(fn {name, arity} -> + function_exported?(AwsSecurityGroup, name, arity) + end) + + assert missing === [], "AwsSecurityGroup is missing callbacks: #{inspect(missing)}" + end + + test "the AWS descriptor resolves security to this module" do + assert DeployEx.Cloud.capability(:security) === {:ok, AwsSecurityGroup} + end + end + + describe "classify_ingress_error/3" do + test "maps an already-exists body to a conflict" do + body = "the rule already exists" + + assert {:error, %ErrorMessage{code: :conflict}} = + AwsSecurityGroup.classify_ingress_error(400, body, %{}) + end + + test "maps a does-not-exist body to not_found" do + body = "the rule does not exist" + + assert {:error, %ErrorMessage{code: :not_found}} = + AwsSecurityGroup.classify_ingress_error(400, body, %{}) + end + + test "falls back to the http code for any other message" do + body = "something else broke" + + assert {:error, %ErrorMessage{message: "something else broke"}} = + AwsSecurityGroup.classify_ingress_error(403, body, %{}) + end + + test "carries the supplied details through" do + body = "nope" + details = %{security_group_id: "sg-123", cidr: "1.2.3.4/32"} + + assert {:error, %ErrorMessage{details: ^details}} = + AwsSecurityGroup.classify_ingress_error(400, body, details) + end + end + + describe "describe_security_groups/2 pagination (via find_security_group/1)" do + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + Process.put(:call_count, (Process.get(:call_count) || 0) + 1) + head + end + + defp call_count, do: Process.get(:call_count) || 0 + + defp recording_request_fn do + fn _request, _config -> next_response() end + end + + defp sg_page(groups, next_token) do + items = + Enum.map_join(groups, "", fn {group_id, group_name} -> + "#{group_id}#{group_name}" <> + "vpc-1" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, + %{ + body: + "#{items}" <> + "#{token}" + }} + end + + test "a security group past page one is still found rather than reporting not_found" do + queue_responses([ + sg_page([{"sg-other", "other-sg"}], "TOKEN"), + sg_page([{"sg-mine", "myapp-sg"}], nil) + ]) + + assert AwsSecurityGroup.find_security_group(project_name: "myapp", request_fn: recording_request_fn()) === + {:ok, %{id: "sg-mine", vpc_id: "vpc-1", name: "myapp-sg"}} + end + + test "terminates when no nextToken comes back" do + queue_responses([sg_page([{"sg-mine", "myapp-sg"}], nil)]) + + assert {:ok, %{id: "sg-mine"}} = + AwsSecurityGroup.find_security_group(project_name: "myapp", request_fn: recording_request_fn()) + + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial match" do + queue_responses([ + sg_page([{"sg-other", "other-sg"}], "TOKEN"), + {:error, {:http_error, 500, %{body: "boom"}}} + ]) + + assert {:error, %ErrorMessage{}} = + AwsSecurityGroup.find_security_group(project_name: "myapp", request_fn: recording_request_fn()) + end + end + + describe "AwsIpWhitelister compatibility" do + test "keeps its public API so its two task call sites are untouched" do + Code.ensure_loaded!(DeployEx.AwsIpWhitelister) + + assert function_exported?(DeployEx.AwsIpWhitelister, :authorize, 3) + assert function_exported?(DeployEx.AwsIpWhitelister, :deauthorize, 3) + end + + test "holds no ExAws calls of its own — they moved into AwsSecurityGroup" do + source = File.read!("lib/deploy_ex/aws_ip_whitelister.ex") + + refute source =~ "ExAws.", + "aws_ip_whitelister.ex must hold zero ExAws calls after P0.2 absorbs them" + end + end +end diff --git a/test/deploy_ex/cloud/oci_object_store_test.exs b/test/deploy_ex/cloud/oci_object_store_test.exs new file mode 100644 index 00000000..b6f6310a --- /dev/null +++ b/test/deploy_ex/cloud/oci_object_store_test.exs @@ -0,0 +1,182 @@ +defmodule DeployEx.Cloud.OciObjectStoreTest do + use ExUnit.Case, async: true + + alias DeployEx.Cloud.OciObjectStore + + @compartment "ocid1.compartment.oc1..test" + + # Captures the command the store built and replays a canned CLI response. Process.put keeps + # it per-test-PID, so the suite stays async without a registry or an ETS table. + defp stub(output) do + [ + run_fn: fn command, _cwd -> + Process.put(:last_command, command) + + case output do + {:error, _} = error -> error + stdout -> {:ok, stdout} + end + end + ] + end + + defp last_command, do: Process.get(:last_command) + + defp cli_failure(output) do + {:error, + ErrorMessage.internal_server_error("oci exited 1", %{output: output, code: 1, command: "oci"})} + end + + describe "list_objects/2" do + test "reads names out of a real --all payload" do + payload = """ + { + "data": [ + {"name": "app/2026-a.tar.gz", "size": 9}, + {"name": "app/2026-b.tar.gz", "size": 9} + ], + "prefixes": [] + } + """ + + assert OciObjectStore.list_objects("bucket", stub(payload)) === + {:ok, ["app/2026-a.tar.gz", "app/2026-b.tar.gz"]} + end + + test "a payload with NO data key is an empty listing, not a crash" do + # MEASURED: `oci os object list` on a prefix matching nothing prints `{"prefixes": []}` + # with the data key absent entirely, so Map.get/3 must supply the default. + assert OciObjectStore.list_objects("bucket", stub(~s({"prefixes": []}))) === {:ok, []} + end + + test "empty stdout is an empty listing, not a JSON decode error" do + # MEASURED: the oci CLI prints NOTHING for an empty result set. Treating that as a decode + # failure makes a first-run empty bucket indistinguishable from a broken command. + assert OciObjectStore.list_objects("bucket", stub("")) === {:ok, []} + end + + test "always passes --all, which is OCI's pagination-to-completion flag" do + OciObjectStore.list_objects("bucket", stub(~s({"data": []}))) + + assert last_command() =~ "--all", + "without --all the CLI returns one page and reports a fraction of the releases " <> + "as if it were all of them, with no error" + end + + test "a prefix narrows the listing" do + OciObjectStore.list_objects("bucket", stub(~s({"data": []})) ++ [prefix: "app/"]) + + assert last_command() =~ "--prefix 'app/'" + end + + test "an empty prefix adds no flag rather than an empty one" do + OciObjectStore.list_objects("bucket", stub(~s({"data": []})) ++ [prefix: ""]) + + refute last_command() =~ "--prefix" + end + end + + describe "namespace" do + # Resolving the namespace internally costs a tenancy read that a least-privilege CI + # credential does not have: it fails every call with "Unable to retrieve namespace + # internally". Auto-resolution only works when the credential is over-privileged, so this + # is invisible until someone does the right thing and scopes the CI user down. + test "is passed explicitly on object calls rather than left to auto-resolution" do + OciObjectStore.list_objects("bucket", stub(~s({"data": []})) ++ [oci_namespace: "axm8ic8kr5of"]) + + assert last_command() =~ "--namespace 'axm8ic8kr5of'" + end + + test "is passed on bucket calls too" do + opts = stub(~s({"data": []})) ++ [oci_namespace: "axm8ic8kr5of", oci_compartment_id: @compartment] + + OciObjectStore.list_containers(opts) + + assert last_command() =~ "--namespace 'axm8ic8kr5of'" + end + + test "is omitted when unset, so an unconfigured project still relies on auto-resolution" do + OciObjectStore.list_objects("bucket", stub(~s({"data": []}))) + + refute last_command() =~ "--namespace" + end + end + + describe "provider-shaped opts" do + test "an AWS-shaped :region does NOT become the OCI region" do + # AwsManager threads opts[:aws_region] (default us-west-2) through to whichever store is + # active. Reading that as the OCI region sends every call to a region the tenancy is not + # in, and the failure looks like a permissions problem rather than a wrong endpoint. + OciObjectStore.list_objects("bucket", stub(~s({"data": []})) ++ [region: "us-west-2"]) + + refute last_command() =~ "us-west-2" + end + + test ":oci_region is the explicit override that does apply" do + OciObjectStore.list_objects("bucket", stub(~s({"data": []})) ++ [oci_region: "ap-seoul-1"]) + + assert last_command() =~ "--region ap-seoul-1" + end + + test "instance_principal auth is selectable for on-instance use" do + OciObjectStore.list_objects("bucket", stub(~s({"data": []})) ++ [oci_auth: "instance_principal"]) + + assert last_command() =~ "OCI_CLI_AUTH=instance_principal" + end + end + + describe "error classification" do + test "a 404 service error becomes :not_found" do + output = ~s(ServiceError:\n{"status": 404, "message": "The service returned error code 404"}) + + assert {:error, %ErrorMessage{code: :not_found}} = + OciObjectStore.get_object("bucket", "nope", stub(cli_failure(output))) + end + + test "a 409 service error becomes :conflict so an existing bucket is distinguishable" do + output = ~s(ServiceError:\n{"status": 409, "message": "bucket already exists"}) + + assert {:error, %ErrorMessage{code: :conflict}} = + OciObjectStore.create_container("bucket", stub(cli_failure(output)) ++ [oci_compartment_id: @compartment]) + end + + test "a failure with no ServiceError body passes the original error through" do + assert {:error, %ErrorMessage{code: :internal_server_error}} = + OciObjectStore.delete_object("bucket", "key", stub(cli_failure("command not found: oci"))) + end + end + + describe "bucket operations" do + test "a missing compartment_id is a bad_request naming the config key, not a crash" do + assert {:error, %ErrorMessage{code: :bad_request} = error} = + OciObjectStore.list_containers(stub(~s({"data": []}))) + + assert error.message =~ "compartment_id" + end + + test "list_containers returns maps carrying :name, matching the S3 store's callers" do + payload = ~s({"data": [{"name": "a-bucket", "time-created": "2026-08-09T00:00:00Z"}]}) + opts = stub(payload) ++ [oci_compartment_id: @compartment] + + assert OciObjectStore.list_containers(opts) === + {:ok, [%{name: "a-bucket", creation_date: "2026-08-09T00:00:00Z"}]} + end + end + + describe "put_object_tags/4" do + test "is an honest :not_implemented rather than an UndefinedFunctionError" do + # OCI has no object tagging and no update-metadata subcommand. Leaving the optional + # callback out would surface at the qa_release call site as an UndefinedFunctionError. + assert {:error, %ErrorMessage{code: :not_implemented}} = + OciObjectStore.put_object_tags("bucket", "key", %{"qa" => "true"}) + end + end + + describe "shell argument quoting" do + test "object keys are quoted so a key with a space cannot split into two arguments" do + OciObjectStore.list_objects("my bucket", stub(~s({"data": []}))) + + assert last_command() =~ "--bucket-name 'my bucket'" + end + end +end diff --git a/test/deploy_ex/cloud/oci_security_group_test.exs b/test/deploy_ex/cloud/oci_security_group_test.exs new file mode 100644 index 00000000..cbdeaecd --- /dev/null +++ b/test/deploy_ex/cloud/oci_security_group_test.exs @@ -0,0 +1,175 @@ +defmodule DeployEx.Cloud.OciSecurityGroupTest do + use ExUnit.Case, async: true + + alias DeployEx.Cloud.OciSecurityGroup + + @compartment "ocid1.compartment.oc1..test" + @nsg_id "ocid1.networksecuritygroup.oc1.ap-seoul-1.test" + + # Same seam OciObjectStoreTest uses: Process.put keeps the captured command per-test-PID, so + # the suite stays async without a registry or an ETS table. + defp stub(output) do + [ + run_fn: fn command, _cwd -> + Process.put(:last_command, command) + + case output do + {:error, _} = error -> error + stdout -> {:ok, stdout} + end + end + ] + end + + defp last_command, do: Process.get(:last_command) + + defp cli_failure(output) do + {:error, + ErrorMessage.internal_server_error("oci exited 1", %{output: output, code: 1, command: "oci"})} + end + + describe "Cloud.Security conformance" do + test "declares the behaviour" do + assert DeployEx.Cloud.Security in (OciSecurityGroup.module_info(:attributes)[:behaviour] || []) + end + + test "exports every callback the behaviour declares" do + Code.ensure_loaded!(OciSecurityGroup) + + missing = + DeployEx.Cloud.Security.behaviour_info(:callbacks) + |> Enum.reject(fn {name, arity} -> function_exported?(OciSecurityGroup, name, arity) end) + + assert missing === [], "OciSecurityGroup is missing callbacks: #{inspect(missing)}" + end + + test "the OCI descriptor resolves security to this module" do + assert DeployEx.Cloud.capability(:security, provider: :oci) === {:ok, OciSecurityGroup} + end + end + + describe "authorize_ingress/3" do + test "adds an ingress rule scoped to the given cidr and port 22" do + assert OciSecurityGroup.authorize_ingress(@nsg_id, "203.0.113.5/32", stub("")) === :ok + + assert last_command() =~ "network nsg rules add" + assert last_command() =~ "--nsg-id '#{@nsg_id}'" + assert last_command() =~ ~s("direction":"INGRESS") + assert last_command() =~ ~s("protocol":"6") + assert last_command() =~ ~s("source":"203.0.113.5/32") + assert last_command() =~ ~s("sourceType":"CIDR_BLOCK") + assert last_command() =~ ~s("min":22) + assert last_command() =~ ~s("max":22) + end + + # PIN: adding a rule preserves existing rules. add_rule_command must never read the + # current rule set and write a merged array back — it sends exactly the one new rule, which + # is what makes the underlying `oci network nsg rules add` call additive rather than a + # replace. A regression toward "read all, append, send the whole array" would still pass the + # command-shape assertions above but fail this one. + test "sends exactly one rule — it never reads or rewrites the existing set" do + OciSecurityGroup.authorize_ingress(@nsg_id, "203.0.113.5/32", stub("")) + + [_prefix, rules_json] = String.split(last_command(), "--security-rules ", parts: 2) + decoded = rules_json |> String.trim("'") |> Jason.decode!() + + assert length(decoded) === 1 + end + + test "a CLI failure surfaces as an ErrorMessage, not :ok" do + assert {:error, %ErrorMessage{}} = + OciSecurityGroup.authorize_ingress(@nsg_id, "203.0.113.5/32", stub(cli_failure("boom"))) + end + end + + describe "revoke_ingress/3" do + @two_rules ~s({"data": [ + {"id": "RULE1", "protocol": "6", "source": "203.0.113.5/32", "source-type": "CIDR_BLOCK", + "tcp-options": {"destination-port-range": {"min": 22, "max": 22}}}, + {"id": "RULE2", "protocol": "6", "source": "198.51.100.9/32", "source-type": "CIDR_BLOCK", + "tcp-options": {"destination-port-range": {"min": 22, "max": 22}}} + ]}) + + # PIN: revoking removes only the matching rule. Proves the removal targets RULE1's id and + # never touches RULE2, which sits in the same NSG for a different CIDR. + test "removes only the rule matching the cidr, by id" do + assert OciSecurityGroup.revoke_ingress(@nsg_id, "203.0.113.5/32", stub(@two_rules)) === :ok + + assert last_command() =~ "network nsg rules remove" + assert last_command() =~ ~s(["RULE1"]) + refute last_command() =~ "RULE2" + end + + # PIN: revoking a rule that is not present is not an error — required for + # `mix deploy_ex.ssh.authorize -r` to be safely re-runnable. + test "a cidr with no matching rule is a no-op :ok, not an error" do + rules = String.replace(@two_rules, "203.0.113.5/32", "192.0.2.1/32") + + assert OciSecurityGroup.revoke_ingress(@nsg_id, "203.0.113.5/32", stub(rules)) === :ok + refute last_command() =~ "remove" + end + + test "an empty rule list is a no-op :ok, not an error" do + assert OciSecurityGroup.revoke_ingress(@nsg_id, "203.0.113.5/32", stub("")) === :ok + end + + test "a rule on a different port is not a match even with the same source" do + rules = ~s({"data": [ + {"id": "RULE3", "protocol": "6", "source": "203.0.113.5/32", "source-type": "CIDR_BLOCK", + "tcp-options": {"destination-port-range": {"min": 80, "max": 80}}} + ]}) + + assert OciSecurityGroup.revoke_ingress(@nsg_id, "203.0.113.5/32", stub(rules)) === :ok + refute last_command() =~ "remove" + end + + test "listing fails loudly rather than silently no-op'ing" do + assert {:error, %ErrorMessage{}} = + OciSecurityGroup.revoke_ingress(@nsg_id, "203.0.113.5/32", stub(cli_failure("boom"))) + end + end + + describe "find_group/1 — explicit override" do + test "an explicit security_group_id is validated with a live get, then returned" do + opts = stub(~s({"data": {"id": "#{@nsg_id}"}})) ++ [security_group_id: @nsg_id] + + assert OciSecurityGroup.find_group(opts) === {:ok, @nsg_id} + assert last_command() =~ "network nsg get --nsg-id '#{@nsg_id}'" + end + + test "a bad override id surfaces the not_found the CLI reports, not a silent wrong id" do + output = ~s(ServiceError:\n{"status": 404, "message": "not found"}) + opts = stub(cli_failure(output)) ++ [security_group_id: @nsg_id] + + assert {:error, %ErrorMessage{code: :not_found}} = OciSecurityGroup.find_group(opts) + end + end + + describe "find_group/1 — prefix search" do + test "finds the nsg matching --nsg, ignoring others" do + payload = ~s({"data": [ + {"display-name": "old-myapp-dev-nsg", "id": "ocid1.nsg.decoy"}, + {"display-name": "myapp-dev-nsg", "id": "#{@nsg_id}"} + ]}) + + opts = stub(payload) ++ [oci_compartment_id: @compartment, project_name: "myapp", environment: "dev"] + + assert OciSecurityGroup.find_group(opts) === {:ok, @nsg_id} + end + + test "no match is a not_found naming the searched prefix" do + payload = ~s({"data": [{"display-name": "other-app-dev-nsg", "id": "ocid1.nsg.other"}]}) + opts = stub(payload) ++ [oci_compartment_id: @compartment, project_name: "myapp", environment: "dev"] + + assert {:error, %ErrorMessage{code: :not_found} = error} = OciSecurityGroup.find_group(opts) + assert error.message =~ "myapp-dev-nsg" + end + + test "a missing compartment_id is a bad_request naming the config key, not a crash" do + assert {:error, %ErrorMessage{code: :bad_request} = error} = + OciSecurityGroup.find_group(stub(~s({"data": []}))) + + assert error.message =~ "compartment_id" + end + end +end diff --git a/test/deploy_ex/cloud/pagination_test.exs b/test/deploy_ex/cloud/pagination_test.exs new file mode 100644 index 00000000..aa6ddf77 --- /dev/null +++ b/test/deploy_ex/cloud/pagination_test.exs @@ -0,0 +1,344 @@ +defmodule DeployEx.Cloud.PaginationTest do + @moduledoc """ + Behavioural tests for the paginators. + + Every fix here originally shipped with source-grep pins, which cannot fail when the loop is + deleted. Every test here does: replacing the recursion with a single request makes it red. + + Responses are queued in the process dictionary — per-PID isolated, no setup or teardown, and no + cache involved. + """ + + use ExUnit.Case, async: true + + alias DeployEx.Cloud.S3ObjectStore + + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + Process.put(:call_count, (Process.get(:call_count) || 0) + 1) + + head + end + + defp call_count, do: Process.get(:call_count) || 0 + + # The marker rides in the request struct's params, not in the config keyword list — config + # carries only region and credentials. + defp recording_request_fn do + fn request, _config -> + marker = request |> Map.get(:params, %{}) |> Map.get("marker") + Process.put(:markers, (Process.get(:markers) || []) ++ [marker]) + + next_response() + end + end + + defp s3_page(keys, truncated, next_marker) do + {:ok, + %{ + body: %{ + contents: Enum.map(keys, &%{key: &1}), + is_truncated: truncated, + next_marker: next_marker + } + }} + end + + defp ec2_page(instance_ids, next_token) do + items = + Enum.map_join(instance_ids, "", fn id -> + "" <> + "#{id}" <> + "running" <> + "" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, + %{ + body: + "#{items}#{token}" + }} + end + + defp ec2_subnets_page(subnets, next_token) do + items = + Enum.map_join(subnets, "", fn {subnet_id, az} -> + "#{subnet_id}#{az}" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, %{body: "#{items}#{token}"}} + end + + defp ec2_instances_with_subnet_page(subnet_ids, next_token) do + items = + Enum.map_join(subnet_ids, "", fn subnet_id -> + "#{subnet_id}" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, + %{body: "#{items}#{token}"}} + end + + defp ec2_images_page(images, next_token) do + items = + Enum.map_join(images, "", fn {image_id, creation_date} -> + "#{image_id}#{creation_date}" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, %{body: "#{items}#{token}"}} + end + + defp iam_profiles_page(names, truncated, marker) do + members = Enum.map_join(names, "", &"#{&1}") + marker_xml = if marker, do: "#{marker}", else: "" + + {:ok, + %{ + body: + "" <> + "#{members}" <> + "#{truncated}#{marker_xml}" <> + "" + }} + end + + describe "S3ObjectStore.list_objects/2 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([s3_page(["a", "b"], "true", "b"), s3_page(["c"], "false", "")]) + + assert S3ObjectStore.list_objects("bucket", + request_fn: recording_request_fn(), + prefix: "p/" + ) === {:ok, ["a", "b", "c"]} + end + + test "terminates when is_truncated is the STRING \"false\", which is truthy in Elixir" do + queue_responses([s3_page(["only"], "false", "")]) + + assert S3ObjectStore.list_objects("bucket", request_fn: recording_request_fn()) === + {:ok, ["only"]} + + assert call_count() === 1, "a truthy string must not drive a second request" + end + + test "advances the marker between pages instead of refetching page one" do + queue_responses([s3_page(["a"], "true", "a"), s3_page(["b"], "false", "")]) + + S3ObjectStore.list_objects("bucket", request_fn: recording_request_fn()) + + assert Process.get(:markers) === [nil, "a"] + end + + test "falls back to the last key when next_marker is empty" do + queue_responses([s3_page(["k1", "k2"], "true", ""), s3_page([], "false", "")]) + + S3ObjectStore.list_objects("bucket", request_fn: recording_request_fn()) + + assert Process.get(:markers) === [nil, "k2"] + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([s3_page(["a"], "true", "a"), {:error, {:http_error, 500, %{body: "boom"}}}]) + + assert {:error, %ErrorMessage{}} = + S3ObjectStore.list_objects("bucket", request_fn: recording_request_fn()) + end + end + + describe "AwsMachine.fetch_instances/2 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([ec2_page(["i-1", "i-2"], "TOKEN"), ec2_page(["i-3"], nil)]) + + assert {:ok, instances} = + DeployEx.AwsMachine.fetch_instances("us-east-1", request_fn: recording_request_fn()) + + assert Enum.map(instances, & &1["instanceId"]) === ["i-1", "i-2", "i-3"] + end + + test "terminates when no nextToken comes back" do + queue_responses([ec2_page(["i-1"], nil)]) + + assert {:ok, [_only]} = + DeployEx.AwsMachine.fetch_instances("us-east-1", request_fn: recording_request_fn()) + + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([ec2_page(["i-1"], "TOKEN"), {:error, {:http_error, 503, %{body: "nope"}}}]) + + assert {:error, %ErrorMessage{}} = + DeployEx.AwsMachine.fetch_instances("us-east-1", request_fn: recording_request_fn()) + end + end + + describe "AwsInfrastructure.find_subnet_ids/1 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([ + ec2_subnets_page([{"subnet-a", "us-east-1a"}], "TOKEN"), + ec2_subnets_page([{"subnet-b", "us-east-1b"}], nil) + ]) + + assert DeployEx.AwsInfrastructure.find_subnet_ids(vpc_id: "vpc-123", request_fn: recording_request_fn()) === + {:ok, ["subnet-a", "subnet-b"]} + end + + test "terminates when no nextToken comes back" do + queue_responses([ec2_subnets_page([{"subnet-a", "us-east-1a"}], nil)]) + + assert {:ok, ["subnet-a"]} = + DeployEx.AwsInfrastructure.find_subnet_ids(vpc_id: "vpc-123", request_fn: recording_request_fn()) + + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([ + ec2_subnets_page([{"subnet-a", "us-east-1a"}], "TOKEN"), + {:error, {:http_error, 500, %{body: "boom"}}} + ]) + + assert {:error, %ErrorMessage{}} = + DeployEx.AwsInfrastructure.find_subnet_ids(vpc_id: "vpc-123", request_fn: recording_request_fn()) + end + + test "a page that filters to zero items but still carries a token keeps paginating" do + queue_responses([ + ec2_subnets_page([], "TOKEN"), + ec2_subnets_page([{"subnet-b", "us-east-1b"}], nil) + ]) + + assert DeployEx.AwsInfrastructure.find_subnet_ids(vpc_id: "vpc-123", request_fn: recording_request_fn()) === + {:ok, ["subnet-b"]} + + assert call_count() === 2 + end + end + + describe "AwsInfrastructure.find_primary_subnet_id/2 pagination" do + test "a truncated ballot would flip the winner without full pagination" do + # page one alone would pick subnet-y (2 votes); the full ballot picks subnet-x (3 votes) + queue_responses([ + ec2_instances_with_subnet_page(["subnet-y", "subnet-y"], "TOKEN"), + ec2_instances_with_subnet_page(["subnet-x", "subnet-x", "subnet-x"], nil) + ]) + + assert DeployEx.AwsInfrastructure.find_primary_subnet_id("sg-123", request_fn: recording_request_fn()) === + {:ok, "subnet-x"} + end + + test "terminates when no nextToken comes back" do + queue_responses([ec2_instances_with_subnet_page(["subnet-x"], nil)]) + + assert {:ok, "subnet-x"} = + DeployEx.AwsInfrastructure.find_primary_subnet_id("sg-123", request_fn: recording_request_fn()) + + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial vote" do + queue_responses([ + ec2_instances_with_subnet_page(["subnet-x"], "TOKEN"), + {:error, {:http_error, 500, %{body: "boom"}}} + ]) + + assert {:error, %ErrorMessage{}} = + DeployEx.AwsInfrastructure.find_primary_subnet_id("sg-123", request_fn: recording_request_fn()) + end + + test "a page with no running instances on it but still carrying a token keeps paginating" do + queue_responses([ + ec2_instances_with_subnet_page([], "TOKEN"), + ec2_instances_with_subnet_page(["subnet-x"], nil) + ]) + + assert DeployEx.AwsInfrastructure.find_primary_subnet_id("sg-123", request_fn: recording_request_fn()) === + {:ok, "subnet-x"} + + assert call_count() === 2 + end + end + + describe "AwsInfrastructure.find_latest_ami/1 pagination" do + test "a truncated first page would pick a stale AMI without full pagination" do + # page one alone has only the older AMI; the newer one only shows up on page two + queue_responses([ + ec2_images_page([{"ami-old", "2024-06-01T00:00:00.000Z"}], "TOKEN"), + ec2_images_page([{"ami-newer", "2025-01-01T00:00:00.000Z"}], nil) + ]) + + assert DeployEx.AwsInfrastructure.find_latest_ami(request_fn: recording_request_fn()) === {:ok, "ami-newer"} + end + + test "terminates when no nextToken comes back" do + queue_responses([ec2_images_page([{"ami-only", "2024-01-01T00:00:00.000Z"}], nil)]) + + assert {:ok, "ami-only"} = DeployEx.AwsInfrastructure.find_latest_ami(request_fn: recording_request_fn()) + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial result" do + queue_responses([ + ec2_images_page([{"ami-old", "2024-01-01T00:00:00.000Z"}], "TOKEN"), + {:error, {:http_error, 500, %{body: "boom"}}} + ]) + + assert {:error, %ErrorMessage{}} = DeployEx.AwsInfrastructure.find_latest_ami(request_fn: recording_request_fn()) + end + + test "a page that filters to zero images but still carries a token keeps paginating" do + queue_responses([ + ec2_images_page([], "TOKEN"), + ec2_images_page([{"ami-only", "2024-01-01T00:00:00.000Z"}], nil) + ]) + + assert DeployEx.AwsInfrastructure.find_latest_ami(request_fn: recording_request_fn()) === {:ok, "ami-only"} + assert call_count() === 2 + end + end + + describe "AwsInfrastructure.find_iam_instance_profile/1 pagination" do + test "concatenates every page rather than returning the first" do + default_name = "deploy-ex-ec2-instance-profile-#{DeployEx.Config.env()}" + + queue_responses([ + iam_profiles_page(["other-profile"], "true", "TOKEN"), + iam_profiles_page([default_name], "false", nil) + ]) + + assert DeployEx.AwsInfrastructure.find_iam_instance_profile(request_fn: recording_request_fn()) === + {:ok, default_name} + end + + test "terminates when IsTruncated is the STRING \"false\", which is truthy in Elixir" do + default_name = "deploy-ex-ec2-instance-profile-#{DeployEx.Config.env()}" + queue_responses([iam_profiles_page([default_name], "false", nil)]) + + assert DeployEx.AwsInfrastructure.find_iam_instance_profile(request_fn: recording_request_fn()) === + {:ok, default_name} + + assert call_count() === 1, "a truthy string must not drive a second request" + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([ + iam_profiles_page(["other-profile"], "true", "TOKEN"), + {:error, {:http_error, 500, %{body: "boom"}}} + ]) + + assert {:error, %ErrorMessage{}} = + DeployEx.AwsInfrastructure.find_iam_instance_profile(request_fn: recording_request_fn()) + end + end +end diff --git a/test/deploy_ex/cloud/providers/aws_test.exs b/test/deploy_ex/cloud/providers/aws_test.exs new file mode 100644 index 00000000..ce0e5dbc --- /dev/null +++ b/test/deploy_ex/cloud/providers/aws_test.exs @@ -0,0 +1,72 @@ +defmodule DeployEx.Cloud.Providers.AwsTest do + use ExUnit.Case, async: true + + alias DeployEx.Cloud.Providers.Aws + + test "declares the descriptor behaviour" do + assert DeployEx.Cloud.Provider in (Aws.module_info(:attributes)[:behaviour] || []) + end + + test "capabilities point at the existing leaf modules" do + assert Aws.capabilities() === %{ + machine: DeployEx.AwsMachine, + object_store: DeployEx.Cloud.S3ObjectStore, + infrastructure: DeployEx.AwsInfrastructure, + security: DeployEx.AwsSecurityGroup + } + end + + test "every capability module actually exists" do + for {_name, module} <- Aws.capabilities() do + assert Code.ensure_loaded?(module), "#{inspect(module)} is a dangling reference" + end + end + + test "backend_template/0 is :s3" do + assert Aws.backend_template() === :s3 + end + + test "completion_marker/0 is :ci_tag" do + assert Aws.completion_marker() === :ci_tag + end + + test "inventory/0 owns both the template path and the rendered filename" do + assert Aws.inventory() === %{ + strategy: :aws_ec2_plugin, + template: "ansible/aws_ec2.yaml.eex", + filename: "aws_ec2.yaml" + } + end + + test "inventory template path resolves to a real file in priv" do + priv_path = :deploy_ex |> :code.priv_dir() |> Path.join(Aws.inventory().template) + + assert File.exists?(priv_path), "descriptor points at a nonexistent template: #{priv_path}" + end + + test "default_ssh_user/0 preserves today's ssh.ex hardcoded value" do + assert Aws.default_ssh_user() === "admin" + end + + test "cli_adapter/0 is nil — AWS goes through ExAws, not a CLI" do + assert is_nil(Aws.cli_adapter()) + end + + describe "config_schema/0" do + test "accepts today's full flat env plus an arbitrary unknown key" do + env = Application.get_all_env(:deploy_ex) ++ [some_unknown_key_xyz: %{nested: 1}] + + assert {:ok, _} = NimbleOptions.validate(env, Aws.config_schema()) + end + + test "accepts legitimately nil values" do + env = [aws_iam_instance_profile: nil, aws_security_group_id: nil, llm_provider: nil] + + assert {:ok, _} = NimbleOptions.validate(env, Aws.config_schema()) + end + + test "accepts an entirely empty env" do + assert {:ok, _} = NimbleOptions.validate([], Aws.config_schema()) + end + end +end diff --git a/test/deploy_ex/cloud/providers/oci_test.exs b/test/deploy_ex/cloud/providers/oci_test.exs new file mode 100644 index 00000000..cb1fb5a0 --- /dev/null +++ b/test/deploy_ex/cloud/providers/oci_test.exs @@ -0,0 +1,73 @@ +defmodule DeployEx.Cloud.Providers.OciTest do + use ExUnit.Case, async: true + + alias DeployEx.Cloud.Providers.Oci + + test "declares the descriptor behaviour" do + assert DeployEx.Cloud.Provider in (Oci.module_info(:attributes)[:behaviour] || []) + end + + test "capabilities/0 exposes the object store and security group and nothing else" do + assert Oci.capabilities() === %{ + object_store: DeployEx.Cloud.OciObjectStore, + security: DeployEx.Cloud.OciSecurityGroup + } + end + + test "backend_template/0 is :s3 — state rides the S3-compatibility endpoint" do + assert Oci.backend_template() === :s3 + end + + test "slots not yet filled are nil, not invented" do + assert is_nil(Oci.completion_marker()) + assert is_nil(Oci.cli_adapter()) + end + + test "inventory/0 declares the static oci CLI generator, not an ansible collection plugin" do + assert Oci.inventory() === %{ + strategy: :static_oci_cli, + template: "ansible/providers/oci/oci.yaml.eex", + filename: "oci.yaml" + } + end + + test "default_ssh_user/0 is ubuntu — OCI's Ubuntu images have no admin user" do + assert Oci.default_ssh_user() === "ubuntu" + end + + describe "config_schema/0" do + test "accepts the documented key set" do + config = [ + region: "us-phoenix-1", + profile: "DEFAULT", + compartment_id: "ocid1.compartment.oc1..aaaa", + namespace: "mynamespace", + shape: "VM.Standard.E5.Flex", + shape_ocpus: 1, + shape_memory_gbs: 8 + ] + + assert {:ok, _} = NimbleOptions.validate(config, Oci.config_schema()) + end + + test "accepts an empty config — the schema catches typos, it does not force config" do + assert {:ok, _} = NimbleOptions.validate([], Oci.config_schema()) + end + + test "EVERY key accepts nil, so an unset System.get_env/1 does not fail task start" do + nil_config = Enum.map(Oci.config_schema(), fn {key, _spec} -> {key, nil} end) + + assert {:ok, _} = NimbleOptions.validate(nil_config, Oci.config_schema()) + end + + test "REJECTS a typo'd key — this is what makes the strict schema non-vacuous" do + assert {:error, %NimbleOptions.ValidationError{}} = + NimbleOptions.validate([regionn: "typo"], Oci.config_schema()) + end + + test "rejects a wrong-typed value" do + assert {:error, %NimbleOptions.ValidationError{}} = + NimbleOptions.validate([shape_ocpus: "not-an-integer"], Oci.config_schema()) + end + end +end diff --git a/test/deploy_ex/cloud/s3_object_store_test.exs b/test/deploy_ex/cloud/s3_object_store_test.exs new file mode 100644 index 00000000..242eb05b --- /dev/null +++ b/test/deploy_ex/cloud/s3_object_store_test.exs @@ -0,0 +1,77 @@ +defmodule DeployEx.Cloud.S3ObjectStoreTest do + use ExUnit.Case, async: true + + alias DeployEx.Cloud.S3ObjectStore + + describe "Cloud.ObjectStore conformance" do + test "declares the behaviour" do + assert DeployEx.Cloud.ObjectStore in (S3ObjectStore.module_info(:attributes)[:behaviour] || []) + end + + test "exports every callback the behaviour declares" do + Code.ensure_loaded!(S3ObjectStore) + + missing = + DeployEx.Cloud.ObjectStore.behaviour_info(:callbacks) + |> Enum.reject(fn {name, arity} -> function_exported?(S3ObjectStore, name, arity) end) + + assert missing === [], "S3ObjectStore is missing callbacks: #{inspect(missing)}" + end + + test "the AWS descriptor now fills the object_store slot" do + assert DeployEx.Cloud.capability(:object_store) === {:ok, S3ObjectStore} + end + + test "every capability module in the AWS descriptor exists" do + for {_name, module} <- DeployEx.Cloud.Providers.Aws.capabilities() do + assert Code.ensure_loaded?(module), "#{inspect(module)} is a dangling reference" + end + end + end + + describe "classify_error/3" do + test "maps 409 to conflict" do + assert {:error, %ErrorMessage{code: :conflict}} = + S3ObjectStore.classify_error(409, "exists", %{container: "b"}) + end + + test "maps 404 to not_found" do + assert {:error, %ErrorMessage{code: :not_found}} = + S3ObjectStore.classify_error(404, "missing", %{container: "b"}) + end + + test "falls back to the http code reason for anything else" do + assert {:error, %ErrorMessage{code: :forbidden}} = + S3ObjectStore.classify_error(403, "denied", %{container: "b"}) + end + + test "carries details through unchanged" do + details = %{container: "my-bucket", key: "some/key.json"} + + assert {:error, %ErrorMessage{details: ^details}} = + S3ObjectStore.classify_error(500, "boom", details) + end + end + + describe "AwsBucket compatibility" do + test "keeps its public API so existing call sites are untouched" do + Code.ensure_loaded!(DeployEx.AwsBucket) + + for {name, arity} <- [ + create_bucket: 2, + list_buckets: 1, + list_objects: 2, + delete_all_objects: 3, + delete_bucket: 2 + ] do + assert function_exported?(DeployEx.AwsBucket, name, arity), + "AwsBucket.#{name}/#{arity} disappeared" + end + end + + test "holds no ExAws calls of its own — they moved into S3ObjectStore" do + refute File.read!("lib/deploy_ex/aws_bucket.ex") =~ "ExAws.", + "aws_bucket.ex must delegate its S3 calls to S3ObjectStore" + end + end +end diff --git a/test/deploy_ex/cloud_test.exs b/test/deploy_ex/cloud_test.exs new file mode 100644 index 00000000..4cb28818 --- /dev/null +++ b/test/deploy_ex/cloud_test.exs @@ -0,0 +1,288 @@ +defmodule DeployEx.CloudTest do + use ExUnit.Case, async: true + + alias DeployEx.Cloud + + describe "capability/2 — default provider" do + test "resolves AWS capabilities through the descriptor" do + assert Cloud.capability(:machine) === {:ok, DeployEx.AwsMachine} + assert Cloud.capability(:infrastructure) === {:ok, DeployEx.AwsInfrastructure} + assert Cloud.capability(:security) === {:ok, DeployEx.AwsSecurityGroup} + end + + test "object_store resolves now that P0.2 filled the slot" do + assert Cloud.capability(:object_store) === {:ok, DeployEx.Cloud.S3ObjectStore} + end + + test "a capability AWS does not implement returns :not_implemented rather than nil" do + assert {:error, %ErrorMessage{code: :not_implemented}} = Cloud.capability(:autoscaling) + end + end + + describe "capability/2 — explicit provider" do + test "an OCI capability is honestly not implemented" do + assert {:error, %ErrorMessage{code: :not_implemented}} = + Cloud.capability(:machine, provider: :oci) + end + + test "an unknown provider errors instead of raising KeyError" do + assert {:error, %ErrorMessage{code: :not_implemented}} = + Cloud.capability(:machine, provider: :gcp) + end + end + + describe "capability/2 — module injection seam (the put_env-free test seam)" do + test "accepts a descriptor MODULE and resolves through it" do + assert Cloud.capability(:machine, provider: DeployEx.Cloud.Providers.Aws) === + {:ok, DeployEx.AwsMachine} + end + + test "a module descriptor lacking the capability reports :not_implemented" do + assert {:error, %ErrorMessage{code: :not_implemented}} = + Cloud.capability(:machine, provider: DeployEx.Cloud.Providers.Oci) + end + + test "a module that is not a descriptor errors instead of raising" do + assert {:error, %ErrorMessage{}} = Cloud.capability(:machine, provider: Enum) + end + + test "a non-atom provider errors instead of raising FunctionClauseError" do + assert {:error, %ErrorMessage{}} = Cloud.capability(:machine, provider: "aws") + assert {:error, %ErrorMessage{}} = Cloud.capability(:machine, provider: 42) + end + end + + describe "inventory/1" do + test "resolves AWS's dynamic plugin descriptor" do + assert Cloud.inventory(:aws) === + {:ok, %{strategy: :aws_ec2_plugin, template: "ansible/aws_ec2.yaml.eex", filename: "aws_ec2.yaml"}} + end + + test "resolves OCI's static generator descriptor" do + assert Cloud.inventory(:oci) === + {:ok, + %{strategy: :static_oci_cli, template: "ansible/providers/oci/oci.yaml.eex", filename: "oci.yaml"}} + end + + test "an unknown provider errors instead of raising" do + assert {:error, %ErrorMessage{code: :not_implemented}} = Cloud.inventory(:gcp) + end + + test "accepts a descriptor module directly, same seam as capability/2" do + assert Cloud.inventory(DeployEx.Cloud.Providers.Aws) === + {:ok, %{strategy: :aws_ec2_plugin, template: "ansible/aws_ec2.yaml.eex", filename: "aws_ec2.yaml"}} + end + end + + describe "validate_config/2 — pure arity, the seam that makes the permissive pin testable" do + test ":aws accepts today's full flat env plus an arbitrary unknown key" do + env = Application.get_all_env(:deploy_ex) ++ [totally_unknown_key_xyz: %{a: 1}] + + assert Cloud.validate_config(:aws, env) === :ok + end + + test ":aws accepts legitimately nil values" do + env = [aws_iam_instance_profile: nil, aws_security_group_id: nil, llm_provider: nil] + + assert Cloud.validate_config(:aws, env) === :ok + end + + test ":aws accepts an empty env" do + assert Cloud.validate_config(:aws, []) === :ok + end + + test ":oci rejects a typo'd key with an ErrorMessage" do + assert {:error, %ErrorMessage{}} = Cloud.validate_config(:oci, regionn: "typo") + end + + test ":oci accepts a valid key" do + assert Cloud.validate_config(:oci, region: "us-phoenix-1") === :ok + end + + test "an unknown provider does not raise" do + assert {:error, %ErrorMessage{}} = Cloud.validate_config(:gcp, []) + end + end + + describe "validate_config/1 — convenience arity reading the real source" do + test "returns :ok under the default :aws provider" do + assert Cloud.validate_config() === :ok + end + + test "an absent provider namespace is [] and not an error" do + assert Cloud.validate_config(provider: :oci) === :ok + end + + test "accepts a bare provider atom, which reads naturally and must not crash" do + assert Cloud.validate_config(:oci) === :ok + assert Cloud.validate_config(:aws) === :ok + end + + test "a non-keyword config errors instead of raising" do + assert {:error, %ErrorMessage{}} = Cloud.validate_config(:oci, %{region: "us-phoenix-1"}) + end + + test "a non-atom provider errors instead of raising" do + assert {:error, %ErrorMessage{}} = Cloud.validate_config("aws", []) + end + end + + describe "dispatcher purity (plan section 3.1 executable pin)" do + test "cloud.ex references no capability or behaviour module literals" do + offenders = + "lib/deploy_ex/cloud.ex" + |> File.read!() + |> then(&Regex.scan(~r/DeployEx\.[A-Za-z0-9_.]+/, &1)) + |> List.flatten() + |> Enum.uniq() + |> Enum.reject(fn reference -> + # DeployEx.Cloud.Provider is the descriptor BEHAVIOUR, not a capability + # implementation — naming it in a typespec is correct and is the point of having it. + # Rejecting it forced a caller to weaken a spec to bare map(), which is this test + # degrading the code rather than guarding it. + reference in ["DeployEx.Config", "DeployEx.Cloud"] or + String.starts_with?(reference, "DeployEx.Cloud.Provider") or + String.starts_with?(reference, "DeployEx.Cloud.Providers.") + end) + + assert offenders === [], + "DeployEx.Cloud must hold no capability literals, found: #{inspect(offenders)}" + end + + test "provider resolution falls back to the configured provider" do + assert Cloud.active_provider([]) === DeployEx.Config.cloud_provider() + end + + test "an explicit provider opt wins over the configured one" do + assert Cloud.active_provider(provider: :oci) === :oci + end + + test "every dispatch path resolves the provider through active_provider/1" do + source = File.read!("lib/deploy_ex/cloud.ex") + + assert function_body(source, "capability") =~ "active_provider(", + "capability/2 must resolve the provider through active_provider/1; resolving it " <> + "inline lets a hardcoded provider ship while every other test stays green" + + assert source =~ ~r/def validate_config\(opts\) when is_list\(opts\) do\n\s+provider = active_provider\(opts\)/, + "validate_config/1 must resolve the provider through active_provider/1" + + assert function_body(source, "active_provider") =~ "Config.cloud_provider()", + "active_provider/1 is the single place the configured provider is read" + end + end + + describe "%Cloud.Instance{}" do + test "has the exact provider-neutral field set" do + keys = + %DeployEx.Cloud.Instance{} + |> Map.from_struct() + |> Map.keys() + |> Enum.sort() + + assert keys === [ + :id, + :ipv6, + :launched_at, + :name, + :private_ip, + :public_ip, + :qa_node?, + :state, + :tags, + :type + ] + end + + test "is not JSON-encodable — section 11 mandates explicit remapping at the output site" do + assert_raise Protocol.UndefinedError, fn -> + Jason.encode!(%DeployEx.Cloud.Instance{}) + end + end + end + + describe "behaviours" do + test "Machine declares its exact callback set" do + assert callback_set(DeployEx.Cloud.Machine) === [ + delete_tags: 3, + describe_instance: 2, + fetch_tags: 2, + find_app_instances: 3, + instance_address: 1, + list_instances: 2, + put_tags: 3, + run_instance: 2, + start_instance: 2, + stop_instance: 2, + terminate_instance: 2 + ] + end + + test "the Phase-5 callbacks are optional so AwsMachine conforms without them" do + assert DeployEx.Cloud.Machine.behaviour_info(:optional_callbacks) |> Enum.sort() === [ + delete_tags: 3, + put_tags: 3, + run_instance: 2, + terminate_instance: 2 + ] + end + + test "ObjectStore declares its exact callback set" do + assert callback_set(DeployEx.Cloud.ObjectStore) === [ + create_container: 2, + delete_container: 2, + delete_object: 3, + get_object: 3, + list_containers: 1, + list_objects: 2, + put_object: 4, + put_object_tags: 4, + upload_file: 4 + ] + end + + test "Infrastructure declares its exact callback set" do + assert callback_set(DeployEx.Cloud.Infrastructure) === [ + find_image: 1, + find_instance_identity: 1, + find_key_pair: 2, + find_network: 1, + find_subnet: 1 + ] + end + + test "Security declares its exact callback set" do + assert callback_set(DeployEx.Cloud.Security) === [ + authorize_ingress: 3, + find_group: 1, + revoke_ingress: 3 + ] + end + + test "the Provider descriptor declares its exact callback set" do + assert callback_set(DeployEx.Cloud.Provider) === [ + backend_template: 0, + capabilities: 0, + cli_adapter: 0, + completion_marker: 0, + config_schema: 0, + default_ssh_user: 0, + inventory: 0 + ] + end + end + + defp function_body(source, name) do + [_before, rest] = String.split(source, ~r/\n def #{name}\(/, parts: 2) + + rest + |> String.split(~r/\n (?:@doc|@spec|def|defp) /, parts: 2) + |> List.first() + end + + defp callback_set(module) do + module.behaviour_info(:callbacks) + |> Enum.sort() + |> Enum.map(fn {name, arity} -> {name, arity} end) + end +end diff --git a/test/deploy_ex/config_test.exs b/test/deploy_ex/config_test.exs new file mode 100644 index 00000000..6e7f31f6 --- /dev/null +++ b/test/deploy_ex/config_test.exs @@ -0,0 +1,11 @@ +defmodule DeployEx.ConfigTest do + use ExUnit.Case, async: true + + alias DeployEx.Config + + describe "cloud_provider/0" do + test "defaults to :aws when nothing is configured" do + assert Config.cloud_provider() === :aws + end + end +end diff --git a/test/deploy_ex/k6_runner_test.exs b/test/deploy_ex/k6_runner_test.exs index 2e64d6a6..61df3bd3 100644 --- a/test/deploy_ex/k6_runner_test.exs +++ b/test/deploy_ex/k6_runner_test.exs @@ -144,4 +144,101 @@ defmodule DeployEx.K6RunnerTest do assert K6Runner.verify_instance_exists(nil) === {:ok, nil} end end + + # Behavioural pagination tests. Responses are queued in the process dictionary — per-PID + # isolated, no setup/teardown. Mirrors test/deploy_ex/cloud/pagination_test.exs: replacing the + # recursion with a single request makes these red. + describe "find_runners_from_ec2/1 pagination" do + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + Process.put(:call_count, (Process.get(:call_count) || 0) + 1) + + head + end + + defp call_count, do: Process.get(:call_count) || 0 + + defp recording_request_fn do + fn _request, _config -> next_response() end + end + + defp ec2_runner_page(instance_ids, next_token) do + items = + Enum.map_join(instance_ids, "", fn id -> + "" <> + "#{id}" <> + "running" <> + "" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, + %{ + body: + "#{items}#{token}" + }} + end + + test "concatenates every page rather than returning the first" do + queue_responses([ec2_runner_page(["i-1", "i-2"], "TOKEN"), ec2_runner_page(["i-3"], nil)]) + + assert {:ok, runners} = K6Runner.find_runners_from_ec2(request_fn: recording_request_fn()) + assert Enum.map(runners, & &1.instance_id) === ["i-1", "i-2", "i-3"] + end + + test "terminates when no nextToken comes back" do + queue_responses([ec2_runner_page(["i-1"], nil)]) + + assert {:ok, [_only]} = K6Runner.find_runners_from_ec2(request_fn: recording_request_fn()) + assert call_count() === 1 + end + end + + describe "fetch_all_runners/1 pagination" do + defp s3_state_page(keys, truncated, next_marker) do + {:ok, + %{ + body: %{ + contents: Enum.map(keys, &%{key: &1}), + is_truncated: truncated, + next_marker: next_marker + } + }} + end + + defp s3_get_object(runner) do + {:ok, %{body: K6Runner.to_json(runner)}} + end + + test "concatenates runner states across S3 pages rather than returning the first" do + runner_one = %K6Runner{instance_id: "i-1"} + runner_two = %K6Runner{instance_id: "i-2"} + + queue_responses([ + s3_state_page(["k6-runners/i-1.json"], "true", "k6-runners/i-1.json"), + s3_state_page(["k6-runners/i-2.json"], "false", ""), + s3_get_object(runner_one), + s3_get_object(runner_two) + ]) + + assert {:ok, runners} = K6Runner.fetch_all_runners(request_fn: recording_request_fn()) + assert Enum.map(runners, & &1.instance_id) === ["i-1", "i-2"] + end + + test "terminates when is_truncated is the STRING \"false\", which is truthy in Elixir" do + runner = %K6Runner{instance_id: "i-only"} + + queue_responses([ + s3_state_page(["k6-runners/i-only.json"], "false", ""), + s3_get_object(runner) + ]) + + assert {:ok, [_only]} = K6Runner.fetch_all_runners(request_fn: recording_request_fn()) + assert call_count() === 2, "a truthy string must not drive a second list request" + end + end end diff --git a/test/deploy_ex/oci_backend_template_test.exs b/test/deploy_ex/oci_backend_template_test.exs new file mode 100644 index 00000000..108ba014 --- /dev/null +++ b/test/deploy_ex/oci_backend_template_test.exs @@ -0,0 +1,71 @@ +defmodule DeployEx.OciBackendTemplateTest do + use ExUnit.Case, async: true + + @template DeployExHelpers.priv_folder("terraform/providers/oci/providers.tf.eex") + + @assigns [ + terraform_backend: :s3, + oci_region: "ap-seoul-1", + oci_namespace: "axm8ic8kr5of", + oci_state_bucket: "opgg-terraform", + oci_state_key: "oracle/ap-seoul-1/opgg-umbrella-dev/terraform.tfstate", + oci_state_profile: "opgg-oci-compat" + ] + + defp render(overrides) do + EEx.eval_file(@template, assigns: Keyword.merge(@assigns, overrides)) + end + + test "renders the compat backend when a state bucket is configured" do + rendered = render([]) + + assert rendered =~ ~s(backend "s3") + assert rendered =~ ~s(bucket = "opgg-terraform") + assert rendered =~ ~s(key = "oracle/ap-seoul-1/opgg-umbrella-dev/terraform.tfstate") + assert rendered =~ ~s(profile = "opgg-oci-compat") + assert rendered =~ "https://axm8ic8kr5of.compat.objectstorage.ap-seoul-1.oraclecloud.com" + assert rendered =~ "skip_s3_checksum = true" + assert rendered =~ "use_path_style = true" + end + + test "omits the profile line when no state profile is configured" do + rendered = render(oci_state_profile: nil) + + assert rendered =~ ~s(backend "s3") + refute rendered =~ "profile" + end + + test "renders no backend at all when the state bucket is absent" do + rendered = render(oci_state_bucket: nil) + + refute rendered =~ "backend" + refute rendered =~ "compat.objectstorage" + end + + test "renders no backend when the terraform backend is :local" do + rendered = render(terraform_backend: :local) + + refute rendered =~ "backend" + end + + describe "managed postgres template" do + test "database.tf declares the psql system with durable storage and an ingress NSG" do + contents = "terraform/providers/oci/database.tf" |> DeployExHelpers.priv_folder() |> File.read!() + + assert contents =~ ~s(resource "oci_psql_db_system" "database") + assert contents =~ "for_each = var.resource_databases" + assert contents =~ "is_regionally_durable = try(each.value.regionally_durable, false)" + assert contents =~ ~s(resource "oci_core_network_security_group" "database") + assert contents =~ ~s(resource "oci_core_subnet" "database_private") + assert contents =~ "prohibit_public_ip_on_vnic = true" + assert contents =~ "min = 5432" + assert contents =~ ~s(password_type = "PLAIN_TEXT") + end + + test "the oci variables template declares resource_databases with an empty default" do + contents = "terraform/providers/oci/variables.tf.eex" |> DeployExHelpers.priv_folder() |> File.read!() + + assert contents =~ ~s(variable "resource_databases") + end + end +end diff --git a/test/deploy_ex/priv_renderer_determinism_test.exs b/test/deploy_ex/priv_renderer_determinism_test.exs new file mode 100644 index 00000000..aa6daef0 --- /dev/null +++ b/test/deploy_ex/priv_renderer_determinism_test.exs @@ -0,0 +1,46 @@ +defmodule DeployEx.PrivRendererDeterminismTest do + use ExUnit.Case, async: true + + alias DeployEx.PrivRenderer + + defp render(opts) do + {:ok, dir} = PrivRenderer.render_to_temp(opts) + on_exit(fn -> File.rm_rf!(dir) end) + + dir + end + + defp read_rendered(dir, relative_path), do: File.read!(Path.join(dir, relative_path)) + + test "pinned pem_app_name renders identical key-pair bytes" do + one = render(pem_app_name: "pinned-abc") + two = render(pem_app_name: "pinned-abc") + + assert read_rendered(one, "terraform/key-pair-main.tf") === + read_rendered(two, "terraform/key-pair-main.tf") + end + + test "default pem_app_name stays random per render" do + one = render([]) + two = render([]) + + assert read_rendered(one, "terraform/key-pair-main.tf") !== + read_rendered(two, "terraform/key-pair-main.tf") + end + + test "different pinned pem_app_names render different key-pair bytes" do + one = render(pem_app_name: "pinned-abc") + two = render(pem_app_name: "other-xyz") + + assert read_rendered(one, "terraform/key-pair-main.tf") !== + read_rendered(two, "terraform/key-pair-main.tf") + end + + test "pinning pem_app_name does not change the database template" do + pinned = render(pem_app_name: "pinned-abc") + default = render([]) + + assert read_rendered(pinned, "terraform/database.tf") === + read_rendered(default, "terraform/database.tf") + end +end diff --git a/test/deploy_ex/qa_node_pagination_test.exs b/test/deploy_ex/qa_node_pagination_test.exs new file mode 100644 index 00000000..d50c3f6f --- /dev/null +++ b/test/deploy_ex/qa_node_pagination_test.exs @@ -0,0 +1,132 @@ +defmodule DeployEx.QaNodePaginationTest do + @moduledoc """ + Behavioural tests for QaNode's EC2/S3 paginated lookups. + + Both DescribeInstances and ListObjects cap a single response and signal more via a + token/marker. A single request returns `{:ok, partial}` silently, never an error — these + tests replace the recursion (in the delegated paginators) with a single request to prove + each site here actually threads `:request_fn` through and consumes every page. + + Responses are queued in the process dictionary — per-PID isolated, no setup or teardown. + """ + + use ExUnit.Case, async: true + + alias DeployEx.QaNode + + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + head + end + + defp recording_request_fn do + fn _request, _config -> next_response() end + end + + defp s3_page(keys, truncated, next_marker) do + {:ok, + %{ + body: %{ + contents: Enum.map(keys, &%{key: &1}), + is_truncated: truncated, + next_marker: next_marker + } + }} + end + + defp ec2_page(instance_ids, next_token) do + items = + Enum.map_join(instance_ids, "", fn id -> + "" <> + "#{id}" <> + "running" <> + "InstanceGroupmy_app_qa" <> + "" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, + %{ + body: + "#{items}#{token}" + }} + end + + defp get_object_page(json) do + {:ok, %{body: json}} + end + + describe "find_qa_nodes_by_branch/2 pagination" do + test "concatenates every EC2 page rather than returning the first" do + queue_responses([ec2_page(["i-1", "i-2"], "TOKEN"), ec2_page(["i-3"], nil)]) + + assert {:ok, nodes} = + QaNode.find_qa_nodes_by_branch("main", request_fn: recording_request_fn()) + + assert Enum.map(nodes, & &1.instance_id) === ["i-1", "i-2", "i-3"] + end + end + + describe "find_qa_nodes_from_ec2/2 pagination" do + test "concatenates every EC2 page rather than returning the first" do + queue_responses([ec2_page(["i-1"], "TOKEN"), ec2_page(["i-2"], nil)]) + + assert {:ok, nodes} = + QaNode.find_qa_nodes_from_ec2("my_app", request_fn: recording_request_fn()) + + assert Enum.map(nodes, & &1.instance_id) === ["i-1", "i-2"] + end + end + + describe "find_qa_node_from_ec2/2 pagination" do + test "still returns the first match once every page has been consumed" do + queue_responses([ec2_page(["i-1"], "TOKEN"), ec2_page(["i-2"], nil)]) + + assert {:ok, node} = + QaNode.find_qa_node_from_ec2("my_app", request_fn: recording_request_fn()) + + assert node.instance_id === "i-1" + end + + test "returns nil, not an error, when no page has a match" do + queue_responses([ec2_page([], nil)]) + + assert {:ok, nil} = + QaNode.find_qa_node_from_ec2("my_app", request_fn: recording_request_fn()) + end + end + + describe "list_all_qa_states/1 pagination" do + test "concatenates every S3 page into a deduped app_name list" do + queue_responses([ + s3_page(["qa-nodes/app_a/i-1.json", "qa-nodes/app_b/i-2.json"], "true", "qa-nodes/app_b/i-2.json"), + s3_page(["qa-nodes/app_a/i-3.json"], "false", "") + ]) + + assert {:ok, app_names} = QaNode.list_all_qa_states(request_fn: recording_request_fn()) + + assert Enum.sort(app_names) === ["app_a", "app_b"] + end + end + + describe "fetch_all_qa_states_for_app/2 pagination" do + test "fetches every key across every S3 page, not just the first page" do + queue_responses([ + s3_page(["qa-nodes/my_app/i-1.json", "qa-nodes/my_app/i-2.json"], "true", "qa-nodes/my_app/i-2.json"), + s3_page(["qa-nodes/my_app/i-3.json"], "false", ""), + get_object_page(~s({"instance_id":"i-1","app_name":"my_app"})), + get_object_page(~s({"instance_id":"i-2","app_name":"my_app"})), + get_object_page(~s({"instance_id":"i-3","app_name":"my_app"})) + ]) + + assert {:ok, states} = + QaNode.fetch_all_qa_states_for_app("my_app", request_fn: recording_request_fn()) + + assert Enum.map(states, & &1.instance_id) === ["i-1", "i-2", "i-3"] + end + end +end diff --git a/test/deploy_ex/release_tracker_test.exs b/test/deploy_ex/release_tracker_test.exs new file mode 100644 index 00000000..3efc7543 --- /dev/null +++ b/test/deploy_ex/release_tracker_test.exs @@ -0,0 +1,119 @@ +defmodule DeployEx.ReleaseTrackerTest do + use ExUnit.Case, async: true + + alias DeployEx.ReleaseTracker + + @source "lib/deploy_ex/release_tracker.ex" + + describe "public API" do + test "keeps every function its call sites use" do + Code.ensure_loaded!(ReleaseTracker) + + for {name, arity} <- [ + current_release_key: 1, + current_release_key: 2, + release_history_key: 1, + release_history_key: 2, + fetch_current_release: 1, + fetch_current_release: 2, + fetch_release_history: 1, + fetch_release_history: 2, + set_current_release: 2, + set_current_release: 3, + append_to_release_history: 2, + append_to_release_history: 3, + list_release_history: 1, + list_release_history: 2, + list_release_history: 3 + ] do + assert function_exported?(ReleaseTracker, name, arity), + "ReleaseTracker.#{name}/#{arity} disappeared" + end + end + end + + describe "current_release_key/2" do + test "defaults to the release-state prefix" do + assert ReleaseTracker.current_release_key("cfx_web") === + "release-state/cfx_web/current_release.txt" + end + + test "qa_release nests under a qa segment" do + assert ReleaseTracker.current_release_key("cfx_web", qa_release: true) === + "release-state/qa/cfx_web/current_release.txt" + end + + test "release_prefix wins over qa_release" do + opts = [release_prefix: "branch-x", qa_release: true] + + assert ReleaseTracker.current_release_key("cfx_web", opts) === + "release-state/branch-x/cfx_web/current_release.txt" + end + + test "release_state_prefix replaces the whole prefix" do + assert ReleaseTracker.current_release_key("cfx_web", release_state_prefix: "custom/state") === + "custom/state/cfx_web/current_release.txt" + end + + test "accepts a map of opts" do + assert ReleaseTracker.current_release_key("cfx_web", %{qa_release: true}) === + "release-state/qa/cfx_web/current_release.txt" + end + + test "blank prefixes fall back to the default" do + assert ReleaseTracker.current_release_key("cfx_web", release_prefix: "") === + "release-state/cfx_web/current_release.txt" + + assert ReleaseTracker.current_release_key("cfx_web", release_state_prefix: "") === + "release-state/cfx_web/current_release.txt" + end + end + + describe "release_history_key/2" do + test "defaults to the release-state prefix" do + assert ReleaseTracker.release_history_key("cfx_web") === + "release-state/cfx_web/release_history.txt" + end + + test "honours the same prefix rules as current_release_key/2" do + assert ReleaseTracker.release_history_key("cfx_web", qa_release: true) === + "release-state/qa/cfx_web/release_history.txt" + end + end + + describe "object-store delegation" do + test "holds no ExAws calls of its own" do + refute File.read!(@source) =~ "ExAws.", + "release_tracker.ex must route its S3 calls through S3ObjectStore" + end + + test "routes through S3ObjectStore" do + assert File.read!(@source) =~ "S3ObjectStore", + "release_tracker.ex must call the provider-neutral object store" + end + + test "reads fixed keys only, so no listing can silently truncate" do + refute File.read!(@source) =~ "list_objects", + "a listing here would need pagination; release_tracker.ex must not grow one" + end + end + + describe "error mapping" do + test "a missing release state still reads as not_found" do + assert File.read!(@source) =~ ~S|ErrorMessage.not_found("release state not found")|, + "ansible.deploy prints this message when a release has never been recorded" + end + + test "other failures keep the aws-failure wording and a :reason detail" do + source = File.read!(@source) + + assert source =~ ~s("aws failure"), "non-404 failures kept the aws failure message" + assert source =~ ":reason", "non-404 failures carried the raw reason in details" + end + + test "writes still answer {:ok, :uploaded}" do + assert File.read!(@source) =~ "{:ok, :uploaded}", + "set_current_release/3 and append_to_release_history/3 return {:ok, :uploaded}" + end + end +end diff --git a/test/deploy_ex/terraform_state_test.exs b/test/deploy_ex/terraform_state_test.exs new file mode 100644 index 00000000..1f99231a --- /dev/null +++ b/test/deploy_ex/terraform_state_test.exs @@ -0,0 +1,155 @@ +defmodule DeployEx.TerraformStateTest do + use ExUnit.Case, async: true + + alias DeployEx.TerraformState + + @source "lib/deploy_ex/terraform_state.ex" + + @state %{ + "version" => 4, + "outputs" => %{"databases" => %{"general" => %{"endpoint" => "db.example.com:5432"}}}, + "resources" => [ + %{ + "type" => "aws_db_instance", + "name" => "rds_database", + "instances" => [ + %{"attributes" => %{"password" => "hunter2", "tags" => %{"Name" => "my-database"}}} + ] + } + ] + } + + setup do + directory = Path.join(System.tmp_dir!(), "deploy_ex_tfstate_#{System.unique_integer([:positive])}") + + File.mkdir_p!(directory) + on_exit(fn -> File.rm_rf!(directory) end) + + {:ok, directory: directory} + end + + describe "public API" do + test "keeps every function its call sites use" do + Code.ensure_loaded!(TerraformState) + + for {name, arity} <- [ + read_state: 1, + read_state: 2, + get_output: 2, + get_resource_attribute: 4, + get_resource_attribute_by_tag: 5, + get_app_display_name: 1, + get_app_display_name: 2 + ] do + assert function_exported?(TerraformState, name, arity), + "TerraformState.#{name}/#{arity} disappeared" + end + end + end + + describe "read_state/2 with the local backend" do + test "decodes the state file", %{directory: directory} do + File.write!(Path.join(directory, "terraform.tfstate"), Jason.encode!(@state)) + + assert TerraformState.read_state(directory, backend: :local) === {:ok, @state} + end + + test "returns a bare string error when the file is absent", %{directory: directory} do + assert TerraformState.read_state(directory, backend: :local) === + {:error, "Terraform state file not found: #{Path.join(directory, "terraform.tfstate")}"} + end + end + + describe "get_output/2" do + test "walks a dotted path into outputs" do + assert TerraformState.get_output(@state, "databases.general.endpoint") === + {:ok, "db.example.com:5432"} + end + + test "returns a bare string error for a missing key" do + assert TerraformState.get_output(@state, "databases.general.nope") === + {:error, "Output key not found: databases.general.nope"} + end + end + + describe "get_resource_attribute/4" do + test "pulls the attribute off the first instance" do + assert TerraformState.get_resource_attribute(@state, "aws_db_instance", "rds_database", "password") === + {:ok, "hunter2"} + end + + test "returns a bare string error when the resource is absent" do + assert TerraformState.get_resource_attribute(@state, "aws_db_instance", "nope", "password") === + {:error, "Resource not found"} + end + + test "returns a bare string error when the attribute is absent" do + assert TerraformState.get_resource_attribute(@state, "aws_db_instance", "rds_database", "nope") === + {:error, "Attribute not found: nope"} + end + end + + describe "get_resource_attribute_by_tag/5" do + test "matches the resource on a tag value" do + assert TerraformState.get_resource_attribute_by_tag( + @state, + "aws_db_instance", + "Name", + "my-database", + "password" + ) === {:ok, "hunter2"} + end + + test "returns a bare string error when no tag matches" do + assert TerraformState.get_resource_attribute_by_tag( + @state, + "aws_db_instance", + "Name", + "other-database", + "password" + ) === {:error, "Resource not found"} + end + end + + describe "object-store delegation" do + test "holds no ExAws calls of its own" do + refute File.read!(@source) =~ "ExAws.", + "terraform_state.ex must route its S3 read through S3ObjectStore" + end + + test "routes through S3ObjectStore" do + assert File.read!(@source) =~ "S3ObjectStore", + "terraform_state.ex must call the provider-neutral object store" + end + + test "reads one fixed key, so no listing can silently truncate" do + refute File.read!(@source) =~ "list_objects", + "a listing here would need pagination; terraform_state.ex must not grow one" + end + end + + describe "s3 error mapping" do + test "keeps the three bare-string messages its callers surface" do + source = File.read!(@source) + + assert source =~ "Terraform state not found in S3: s3://" + assert source =~ "Access denied to S3 bucket: " + assert source =~ "Failed to read Terraform state from S3: " + end + + test "discriminates on the ErrorMessage codes S3ObjectStore produces" do + source = File.read!(@source) + + assert source =~ ":not_found", "a missing object must still map to the not-found message" + assert source =~ ":forbidden", "a denied bucket must still map to the access-denied message" + end + + test "S3ObjectStore really produces those codes" do + assert {:error, %ErrorMessage{code: :not_found}} = + DeployEx.Cloud.S3ObjectStore.classify_error(404, "missing", %{container: "b"}) + + assert {:error, %ErrorMessage{code: :forbidden}} = + DeployEx.Cloud.S3ObjectStore.classify_error(403, "denied", %{container: "b"}) + end + end +end diff --git a/test/deploy_ex/terraform_test.exs b/test/deploy_ex/terraform_test.exs new file mode 100644 index 00000000..9e74a7dd --- /dev/null +++ b/test/deploy_ex/terraform_test.exs @@ -0,0 +1,85 @@ +defmodule DeployEx.TerraformTest do + use ExUnit.Case, async: true + + alias DeployEx.Terraform + + describe "parse_args/2 — the shared arg builder plan/apply/drop route through" do + test "passes an explicit --var-file through untouched" do + assert Terraform.parse_args(["--var-file", "terraform.tfvars"], :plan) === + "--var-file terraform.tfvars" + end + + test "returns an empty string for no args and no config default" do + assert Terraform.parse_args([], :plan) === "" + end + + test "accepts any var-file path verbatim, not just AWS-shaped ones" do + assert Terraform.parse_args(["--var-file", "providers/oci/terraform.tfvars"], :apply) === + "--var-file providers/oci/terraform.tfvars" + end + + test "leaves a fully-qualified target string unchanged" do + assert Terraform.parse_args(["--target", "module.ec2_instance[\\\"foo\\\"]"], :destroy) === + "--target module.ec2_instance[\\\"foo\\\"]" + end + + test "leaves a dotted (non-bare) target unchanged, regardless of provider resource naming" do + assert Terraform.parse_args(["--target", "oci_core_instance.main"], :plan) === + "--target oci_core_instance.main" + end + + test "expands multiple --target flags independently" do + result = Terraform.parse_args( + ["--target", "oci_core_vcn.main", "--target", "oci_core_subnet.public"], + :plan + ) + + assert result === "--target oci_core_vcn.main --target oci_core_subnet.public" + end + + # Pins build_target_string/1's current behavior: a bare, non-dotted target name is + # assumed to be an AWS app name and wrapped into the ec2_instance module path. This is + # the one AWS-module-shaped assumption left in the shared arg builder — a future + # provider-aware branch (e.g. for OCI's per-app modules) should change this test, not + # break it silently. + test "wraps a bare app-name target into the AWS ec2_instance module path (pre-provider-aware baseline)" do + assert Terraform.parse_args(["--target", "myapp"], :plan) === + "--target module.ec2_instance[\\\"myapp\\\"]" + end + end + + describe "plan/apply/drop tasks stay provider-neutral" do + @task_sources [ + "lib/mix/tasks/terraform.plan.ex", + "lib/mix/tasks/terraform.apply.ex", + "lib/mix/tasks/terraform.drop.ex" + ] + + test "hold no hardcoded AWS/S3-specific strings in their command path" do + Enum.each(@task_sources, fn source_path -> + source = File.read!(source_path) + + refute source =~ ~r/aws|s3|ec2/i, + "#{source_path} should stay provider-neutral; found an AWS/S3/EC2 reference" + end) + end + + test "each accepts --directory, so any provider's rendered set can be targeted" do + Enum.each(@task_sources, fn source_path -> + source = File.read!(source_path) + + assert source =~ "directory: :string", + "#{source_path} must accept --directory to point at a non-default provider tree" + end) + end + + test "each routes its command through DeployEx.Terraform.parse_args/2" do + Enum.each(@task_sources, fn source_path -> + source = File.read!(source_path) + + assert source =~ "DeployEx.Terraform.parse_args(args,", + "#{source_path} must build its tofu invocation through the shared, provider-neutral arg builder" + end) + end + end +end diff --git a/test/deploy_ex/terraform_variables_test.exs b/test/deploy_ex/terraform_variables_test.exs new file mode 100644 index 00000000..21dfa7c1 --- /dev/null +++ b/test/deploy_ex/terraform_variables_test.exs @@ -0,0 +1,84 @@ +defmodule DeployEx.TerraformVariablesTest do + use ExUnit.Case, async: true + + alias DeployEx.TerraformVariables + + describe "terraform_clickhouse_variables/2" do + test "renders nothing on oci unless --clickhouse is passed" do + assert TerraformVariables.terraform_clickhouse_variables([], :oci) === "" + end + + test "renders a DatabaseKey-tagged entry on oci when opted in" do + rendered = TerraformVariables.terraform_clickhouse_variables([clickhouse: true], :oci) + + assert rendered =~ "_clickhouse = {" + assert rendered =~ ~s(DatabaseKey = ") + assert rendered =~ "_clickhouse\"" + assert rendered =~ ~s(shape = "VM.Standard.E6.Flex") + end + + test "renders nothing on aws even when opted in — aws users hand-write the entry" do + assert TerraformVariables.terraform_clickhouse_variables([clickhouse: true], :aws) === "" + end + end + + describe "generate_terraform_release_variables/2" do + test "T18: oci clause discovers load_balancer via a commented example block" do + rendered = TerraformVariables.generate_terraform_release_variables("my_app", :oci) + + assert rendered =~ ~r/#\s*load_balancer\s*=\s*\{/ + assert rendered =~ ~r/#\s*enable\s*=\s*true/ + assert rendered =~ ~r/#\s*enable_https\s*=\s*false/ + assert rendered =~ ~r/#\s*reserved_ip_ocid\s*=\s*null/ + assert rendered =~ ~r/#\s*health_check\s*=\s*\{/ + assert rendered =~ ~r/#\s*path\s*=/ + assert rendered =~ ~r/#\s*return_code\s*=/ + assert rendered =~ ~r/#\s*https_return_code\s*=/ + assert rendered =~ ~r/#\s*unhealthy_threshold\s*=/ + assert rendered =~ ~r/#\s*timeout\s*=/ + assert rendered =~ ~r/#\s*interval\s*=/ + end + + test "T18: aws clause is byte-identical to the current pinned string" do + rendered = TerraformVariables.generate_terraform_release_variables("my_app", :aws) + + assert rendered === String.trim_trailing(""" + my_app = { + name = "My App" + tags = { + Vendor = "Self" + Type = "Self Made" + } + + # Autoscaling Configuration (optional) + # Uncomment and configure to enable AWS Auto Scaling Groups + # autoscaling = { + # enable = true + # min_size = 1 + # max_size = 5 + # desired_capacity = 2 + # cpu_target_percent = 60 + # } + } + """, "\n") + end + end + + describe "terraform_rabbitmq_variables/2" do + test "renders nothing on oci unless --rabbitmq is passed" do + assert TerraformVariables.terraform_rabbitmq_variables([], :oci) === "" + end + + test "renders a DatabaseKey-tagged single node on oci when opted in" do + rendered = TerraformVariables.terraform_rabbitmq_variables([rabbitmq: true], :oci) + + assert rendered =~ "_rabbitmq = {" + assert rendered =~ "_rabbitmq\"" + refute rendered =~ "instance_count" + end + + test "renders nothing on aws" do + assert TerraformVariables.terraform_rabbitmq_variables([rabbitmq: true], :aws) === "" + end + end +end diff --git a/test/deploy_ex/tui/deploy_progress_test.exs b/test/deploy_ex/tui/deploy_progress_test.exs index 01ed595a..a59384e2 100644 --- a/test/deploy_ex/tui/deploy_progress_test.exs +++ b/test/deploy_ex/tui/deploy_progress_test.exs @@ -3,6 +3,49 @@ defmodule DeployEx.TUI.DeployProgressTest do alias DeployEx.TUI.DeployProgress + describe "run/3 failure propagation" do + # Task.async_stream wraps every completed task as {:ok, result}, so a run_fn returning + # {:error, _} reaches the reducer as {:ok, {:error, _}} — which matches its SUCCESS clause. + # Without unwrapping, a failed ansible play is aggregated as a successful record, + # ansible.setup/ansible.deploy skip their Mix.raise, and the task exits 0. In CI that is a + # green deploy that never deployed. MEASURED against a live node: a play with failed=1 + # exited 0. + test "a run_fn error makes run/3 report an error, not a successful record" do + run_fn = fn _playbook, _callback -> + {:error, ErrorMessage.internal_server_error("command failed", %{code: 2})} + end + + assert {:error, [%ErrorMessage{code: :internal_server_error}]} = + DeployProgress.run(["setup/app.yaml"], run_fn) + end + + test "one failure among several still reports an error" do + run_fn = fn + "setup/bad.yaml", _callback -> {:error, ErrorMessage.internal_server_error("boom")} + _playbook, _callback -> :ok + end + + assert {:error, errors} = + DeployProgress.run(["setup/ok.yaml", "setup/bad.yaml", "setup/ok2.yaml"], run_fn) + + assert length(errors) === 1 + end + + test "all succeeding still reports ok" do + assert {:ok, _} = DeployProgress.run(["setup/a.yaml"], fn _playbook, _callback -> :ok end) + end + + test "a task that exceeds the timeout is an error, not a silent omission" do + # Real plays run for 20+ minutes (asdf compiles Erlang from source), so hitting the + # async_stream timeout is not hypothetical. Task.async_stream yields {:exit, :timeout} + # for it, which must surface rather than vanish from the aggregate. + run_fn = fn _playbook, _callback -> Process.sleep(200) end + + assert {:error, [%ErrorMessage{}]} = + DeployProgress.run(["setup/slow.yaml"], run_fn, timeout: 10) + end + end + describe "action_labels/1" do test "defaults to deploy wording" do assert DeployProgress.action_labels([]) === %{gerund: "Deploying", noun: "Deploy"} diff --git a/test/deploy_ex/tui/wizard/command_registry_test.exs b/test/deploy_ex/tui/wizard/command_registry_test.exs index 80a916fd..e7d011b0 100644 --- a/test/deploy_ex/tui/wizard/command_registry_test.exs +++ b/test/deploy_ex/tui/wizard/command_registry_test.exs @@ -8,6 +8,10 @@ defmodule DeployEx.TUI.Wizard.CommandRegistryTest do @hidden_per_task %{ # `--no-tui` is a tooling-internal flag set automatically when the wizard # invokes the task; users do not need to toggle it through the wizard UI. + # The render-harness flags (`--render-dir`, `--pem-app-name`, + # `--db-password`) are likewise tooling-internal: bin/render_harness.sh + # drives them to produce a deterministic render for cross-revision diffing. + "ansible.build" => [:render_dir], "ansible.deploy" => [:no_tui], "ansible.setup" => [:no_tui], "deploy_ex.autoscale.refresh" => [:no_tui], @@ -19,7 +23,8 @@ defmodule DeployEx.TUI.Wizard.CommandRegistryTest do "deploy_ex.restart_app" => [:no_tui], "deploy_ex.restart_machine" => [:no_tui], "deploy_ex.start_app" => [:no_tui], - "deploy_ex.stop_app" => [:no_tui] + "deploy_ex.stop_app" => [:no_tui], + "terraform.build" => [:render_dir, :pem_app_name, :db_password] } # Tasks that accept positional arguments — these are surfaced as diff --git a/test/mix/tasks/ansible_build_oci_inventory_test.exs b/test/mix/tasks/ansible_build_oci_inventory_test.exs new file mode 100644 index 00000000..340d6197 --- /dev/null +++ b/test/mix/tasks/ansible_build_oci_inventory_test.exs @@ -0,0 +1,130 @@ +defmodule Mix.Tasks.Ansible.Build.OciInventoryTest do + use ExUnit.Case, async: true + + alias Mix.Tasks.Ansible.Build + + # These exercise the PURE transform from hydrated OCI instances (already fetched from the + # oci CLI, see Build.fetch_oci_instances/1) to inventory host entries — no live oci CLI or + # network access, so this covers the group/hostvar contract hermetically. + + defp instance(overrides) do + Map.merge( + %{ + id: "ocid1.instance.oc1.ap-seoul-1.example", + name: "dx-ansible-test-01", + tags: %{"Group" => "Deploy Ex Backend", "InstanceGroup" => "deploy_ex_basic_dev"}, + public_ip: "64.110.76.187", + private_ip: "10.60.0.183", + ipv6: nil + }, + overrides + ) + end + + describe "oci_inventory_hosts/1" do + test "composes hostname as -" do + [host] = Build.oci_inventory_hosts([instance(%{})]) + + assert host.hostname === "ocid1.instance.oc1.ap-seoul-1.example-dx-ansible-test-01" + end + + test "ansible_host prefers ipv6, then public_ip, then private_ip" do + [ipv6_host] = Build.oci_inventory_hosts([instance(%{ipv6: "fe80::1"})]) + [public_host] = Build.oci_inventory_hosts([instance(%{})]) + [private_host] = Build.oci_inventory_hosts([instance(%{public_ip: nil})]) + + assert ipv6_host.vars.ansible_host === "fe80::1" + assert public_host.vars.ansible_host === "64.110.76.187" + assert private_host.vars.ansible_host === "10.60.0.183" + end + + test "composes the seven tag-derived hostvars for a non-qa host" do + [host] = Build.oci_inventory_hosts([instance(%{})]) + + assert host.vars.release_prefix === "" + assert host.vars.release_state_prefix === "release-state" + assert host.vars.git_branch === "" + assert host.vars.qa_node === false + assert host.vars.qa_node_suffix === "" + assert host.vars.instance_tag === "" + assert host.vars.letsencrypt_use_public_ip === false + end + + test "QaNode=true flips the qa-derived hostvars together" do + tags = %{"Group" => "Deploy Ex Backend", "QaNode" => "true", "GitBranch" => "feature/x"} + [host] = Build.oci_inventory_hosts([instance(%{tags: tags})]) + + assert host.vars.release_prefix === "qa" + assert host.vars.release_state_prefix === "release-state/qa" + assert host.vars.qa_node === true + assert host.vars.qa_node_suffix === "_qa" + assert host.vars.git_branch === "feature/x" + end + + test "UsePublicIpCert=true sets letsencrypt_use_public_ip" do + tags = %{"Group" => "Deploy Ex Backend", "UsePublicIpCert" => "true"} + [host] = Build.oci_inventory_hosts([instance(%{tags: tags})]) + + assert host.vars.letsencrypt_use_public_ip === true + end + + test "keyed groups come from MonitoringKey/InstanceGroup/DatabaseKey/QaNode tags only" do + tags = %{ + "Group" => "Deploy Ex Backend", + "InstanceGroup" => "deploy_ex_basic_dev", + "MonitoringKey" => "prometheus_db", + "DatabaseKey" => "deploy_ex_redis", + "QaNode" => "true", + "SomeOtherTag" => "ignored" + } + + [host] = Build.oci_inventory_hosts([instance(%{tags: tags})]) + + expected = ["group_deploy_ex_basic_dev", "monitoring_prometheus_db", "database_deploy_ex_redis", "qa_true"] + + assert Enum.sort(host.groups) === Enum.sort(expected) + end + + test "a tag with no value contributes no group" do + [host] = Build.oci_inventory_hosts([instance(%{tags: %{"Group" => "Deploy Ex Backend"}})]) + + assert host.groups === [] + end + end + + describe "render_oci_hosts_section/1" do + test "renders {} for an empty host list" do + assert Build.render_oci_hosts_section([]) === " hosts: {}" + end + + test "renders one indented block per host with every hostvar" do + [host] = Build.oci_inventory_hosts([instance(%{})]) + + rendered = Build.render_oci_hosts_section([host]) + + assert rendered =~ " hosts:\n" + assert rendered =~ " ocid1.instance.oc1.ap-seoul-1.example-dx-ansible-test-01:\n" + assert rendered =~ " ansible_host: \"64.110.76.187\"" + assert rendered =~ " qa_node: false" + end + end + + describe "render_oci_children_section/1" do + test "renders {} when no host belongs to any group" do + [host] = Build.oci_inventory_hosts([instance(%{tags: %{"Group" => "Deploy Ex Backend"}})]) + + assert Build.render_oci_children_section([host]) === " children: {}" + end + + test "groups hosts under their keyed group name" do + tags = %{"Group" => "Deploy Ex Backend", "InstanceGroup" => "deploy_ex_basic_dev"} + [host] = Build.oci_inventory_hosts([instance(%{tags: tags})]) + + rendered = Build.render_oci_children_section([host]) + + assert rendered =~ " children:\n" + assert rendered =~ " group_deploy_ex_basic_dev:\n hosts:\n" + assert rendered =~ " ocid1.instance.oc1.ap-seoul-1.example-dx-ansible-test-01: {}" + end + end +end diff --git a/test/mix/tasks/ansible_build_render_test.exs b/test/mix/tasks/ansible_build_render_test.exs new file mode 100644 index 00000000..15322212 --- /dev/null +++ b/test/mix/tasks/ansible_build_render_test.exs @@ -0,0 +1,195 @@ +defmodule Mix.Tasks.Ansible.BuildRenderTest do + # async: false — drives a real Mix task and writes to the filesystem + use ExUnit.Case, async: false + + import ExUnit.CaptureIO + + alias Mix.Tasks.Ansible.Build + + setup do + dirs = Enum.map(1..2, fn index -> + Path.join(System.tmp_dir!(), "p00_ans_#{System.unique_integer([:positive])}_#{index}") + end) + + on_exit(fn -> Enum.each(dirs, &File.rm_rf!/1) end) + + {:ok, dirs: dirs} + end + + defp render(args), do: capture_io(fn -> Build.run(args) end) + + defp file_tree(dir) do + dir + |> Path.join("**/*") + |> Path.wildcard(match_dot: true) + |> Enum.map(&Path.relative_to(&1, dir)) + |> Enum.sort() + end + + describe "--render-dir" do + test "renders into a fresh directory without raising", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + assert File.dir?(dir) + end + + test "renders ansible.cfg with the placeholder pem path", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + config_path = Path.join(dir, "ansible.cfg") + + assert File.exists?(config_path) + assert File.read!(config_path) =~ "../terraform/RENDER_DIR_PLACEHOLDER.pem" + end + + test "renders the hosts file and group vars", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + assert File.exists?(Path.join(dir, "aws_ec2.yaml")) + assert File.exists?(Path.join(dir, "group_vars/all.yaml")) + end + + test "seeds roles into the render dir", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + assert File.dir?(Path.join(dir, "roles")) + end + + test "writes playbooks into the render dir", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + assert File.exists?(Path.join(dir, "playbooks/deploy_ex.yaml")) + assert File.exists?(Path.join(dir, "setup/deploy_ex.yaml")) + end + + test "removes the copied root templates from the render dir", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + assert Path.wildcard(Path.join(dir, "*.eex")) === [] + end + + test "writes nothing into the live deploys tree", %{dirs: [dir | _]} do + deploys_before = Path.wildcard("./deploys/**", match_dot: true) + + render(["--render-dir", dir, "--quiet"]) + + assert Path.wildcard("./deploys/**", match_dot: true) === deploys_before + end + + test "two runs render identical trees", %{dirs: [one, two]} do + render(["--render-dir", one, "--quiet"]) + render(["--render-dir", two, "--quiet"]) + + assert file_tree(one) === file_tree(two) + + for relative_path <- file_tree(one), File.regular?(Path.join(one, relative_path)) do + assert File.read!(Path.join(one, relative_path)) === + File.read!(Path.join(two, relative_path)), + "#{relative_path} differed between runs" + end + end + + test "the default aws render carries zero provider-scoped files", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + refute File.dir?(Path.join(dir, "providers")) + refute File.exists?(Path.join(dir, "oci.yaml")) + end + end + + describe "--provider" do + test "an unknown provider raises a clear error", %{dirs: [dir | _]} do + assert_raise Mix.Error, ~r/unknown provider "unknown-cloud"/, fn -> + render(["--render-dir", dir, "--provider", "unknown-cloud", "--quiet"]) + end + end + + test "oci without a compartment_id raises before touching the oci CLI or the filesystem", %{dirs: [dir | _]} do + assert_raise Mix.Error, ~r/compartment_id is required/, fn -> + render(["--render-dir", dir, "--provider", "oci", "--quiet"]) + end + + refute File.exists?(dir), "validation must run before ensure_ansible_directory_exists/3 seeds anything" + end + + test "oci --auto-pull-aws raises instead of silently doing nothing", %{dirs: [dir | _]} do + assert_raise Mix.Error, ~r/--auto-pull-aws only supports the aws provider/, fn -> + render([ + "--render-dir", dir, + "--provider", "oci", + "--auto-pull-aws", + "--oci-compartment-id", "ocid1.compartment.oc1..fake", + "--quiet" + ]) + end + end + end + + describe "DeployEx.Cloud.PrivFileSet file selection" do + setup do + {:ok, priv_path: DeployExHelpers.priv_folder("ansible")} + end + + test "aws never resolves the oci-scoped templates", %{priv_path: priv_path} do + {:ok, files} = DeployEx.Cloud.PrivFileSet.files(:aws, priv_path) + dests = Enum.map(files, fn {_source, dest} -> dest end) + + refute "oci.yaml.eex" in dests + refute Enum.any?(dests, &String.starts_with?(&1, "providers/")) + end + + test "oci resolves ansible.cfg.eex and oci.yaml.eex flattened, and nothing aws-only", %{priv_path: priv_path} do + {:ok, files} = DeployEx.Cloud.PrivFileSet.files(:oci, priv_path) + + assert {"providers/oci/ansible.cfg.eex", "ansible.cfg.eex"} in files + assert {"providers/oci/oci.yaml.eex", "oci.yaml.eex"} in files + refute Enum.any?(files, fn {source, _dest} -> source === "aws_ec2.yaml.eex" end) + end + end + + describe "the oci ansible.cfg template" do + test "sets the ubuntu remote user, the oci.yaml inventory, and no [inventory] plugin section" do + contents = "ansible/providers/oci/ansible.cfg.eex" |> DeployExHelpers.priv_folder() |> File.read!() + + assert contents =~ "remote_user = ubuntu" + assert contents =~ "inventory = ./oci.yaml" + refute contents =~ "[inventory]" + refute contents =~ "enable_plugins" + end + end + + describe "remove_other_provider_inventories/2 — the provider-swap cleanup" do + setup %{dirs: [dir | _]} do + File.mkdir_p!(dir) + {:ok, dir: dir} + end + + test "removes a stale AWS inventory left over from a prior build for :oci", %{dir: dir} do + File.write!(Path.join(dir, "aws_ec2.yaml"), "stale plugin config") + + Build.remove_other_provider_inventories(:oci, directory: dir) + + refute File.exists?(Path.join(dir, "aws_ec2.yaml")) + end + + test "removes a stale OCI inventory left over from a prior build for :aws", %{dir: dir} do + File.write!(Path.join(dir, "oci.yaml"), "stale snapshot") + + Build.remove_other_provider_inventories(:aws, directory: dir) + + refute File.exists?(Path.join(dir, "oci.yaml")) + end + + test "never removes the current provider's own inventory file", %{dir: dir} do + File.write!(Path.join(dir, "oci.yaml"), "current snapshot") + + Build.remove_other_provider_inventories(:oci, directory: dir) + + assert File.exists?(Path.join(dir, "oci.yaml")) + end + + test "is a no-op when no other provider's file is present", %{dir: dir} do + assert :ok = Build.remove_other_provider_inventories(:oci, directory: dir) + end + end +end diff --git a/test/mix/tasks/ansible_deploy_test.exs b/test/mix/tasks/ansible_deploy_test.exs index 6e45f0cd..660e65c4 100644 --- a/test/mix/tasks/ansible_deploy_test.exs +++ b/test/mix/tasks/ansible_deploy_test.exs @@ -10,6 +10,7 @@ defmodule Mix.Tasks.Ansible.DeployTest do aliases: [f: :force, q: :quit, d: :directory, l: :only_local_release, t: :target_sha], switches: [ directory: :string, + provider: :string, quiet: :boolean, only: :keep, except: :keep, @@ -39,6 +40,16 @@ defmodule Mix.Tasks.Ansible.DeployTest do end describe "parse_args/1 option parsing" do + test "--provider parses to opts[:provider]" do + opts = parse_args(["--provider", "oci"]) + assert opts[:provider] === "oci" + end + + test "opts[:provider] is nil when --provider not passed" do + opts = parse_args([]) + assert is_nil(opts[:provider]) + end + test "--target-sha parses to opts[:target_sha]" do opts = parse_args(["--target-sha", "abc1234"]) assert opts[:target_sha] === "abc1234" diff --git a/test/mix/tasks/ansible_ping_test.exs b/test/mix/tasks/ansible_ping_test.exs new file mode 100644 index 00000000..aebf846f --- /dev/null +++ b/test/mix/tasks/ansible_ping_test.exs @@ -0,0 +1,28 @@ +defmodule Mix.Tasks.Ansible.PingTest do + use ExUnit.Case, async: true + + # parse_args/1 is private — mirror the OptionParser config here. + # Same pattern as ansible_setup_test.exs / ansible_deploy_test.exs. + + defp parse_args(args) do + {opts, _extra_args} = OptionParser.parse!(args, switches: [provider: :string]) + opts + end + + describe "parse_args/1 option parsing" do + test "--provider parses to opts[:provider]" do + opts = parse_args(["--provider", "oci"]) + assert opts[:provider] === "oci" + end + + test "opts[:provider] is nil when --provider not passed" do + opts = parse_args([]) + assert is_nil(opts[:provider]) + end + + test "unrelated ansible passthrough flags do not raise" do + opts = parse_args(["-i", "custom.yaml", "--limit", "webservers"]) + assert is_nil(opts[:provider]) + end + end +end diff --git a/test/mix/tasks/ansible_setup_test.exs b/test/mix/tasks/ansible_setup_test.exs index e786e6c3..1a83693e 100644 --- a/test/mix/tasks/ansible_setup_test.exs +++ b/test/mix/tasks/ansible_setup_test.exs @@ -12,6 +12,7 @@ defmodule Mix.Tasks.Ansible.SetupTest do aliases: [f: :force, q: :quit, d: :directory, i: :instance_id, b: :git_branch], switches: [ directory: :string, + provider: :string, only: :keep, except: :keep, force: :boolean, @@ -27,6 +28,16 @@ defmodule Mix.Tasks.Ansible.SetupTest do end describe "parse_args/1 option parsing" do + test "--provider parses to opts[:provider]" do + {opts, _extra} = parse_args(["--provider", "oci"]) + assert opts[:provider] === "oci" + end + + test "opts[:provider] is nil when --provider not passed" do + {opts, _extra} = parse_args(["--include-qa"]) + assert is_nil(opts[:provider]) + end + test "--git-branch parses to opts[:git_branch]" do {opts, _extra} = parse_args(["--git-branch", "qa/gamma_charts"]) assert opts[:git_branch] === "qa/gamma_charts" diff --git a/test/mix/tasks/terraform_build_oci_lb_test.exs b/test/mix/tasks/terraform_build_oci_lb_test.exs new file mode 100644 index 00000000..e3defaa3 --- /dev/null +++ b/test/mix/tasks/terraform_build_oci_lb_test.exs @@ -0,0 +1,303 @@ +defmodule Mix.Tasks.Terraform.BuildOciLbTest do + # async: false — matches ansible_build_render_test.exs; this file grows render-driven rows + # in later sprints (S2/S3) alongside these template-content rows + use ExUnit.Case, async: false + + import ExUnit.CaptureIO + + alias Mix.Tasks.Terraform.Build + + @load_balancer_tf "terraform/providers/oci/modules/oci-instance/load_balancer.tf" + @module_outputs_tf "terraform/providers/oci/modules/oci-instance/outputs.tf" + @module_main_tf "terraform/providers/oci/modules/oci-instance/main.tf" + @module_variables_tf "terraform/providers/oci/modules/oci-instance/variables.tf" + @root_variables_tf_eex "terraform/variables.tf.eex" + @aws_instance_main_tf "terraform/modules/aws-instance/main.tf" + @root_variables_tf_eex_oci "terraform/providers/oci/variables.tf.eex" + + defp load_balancer_tf, do: @load_balancer_tf |> DeployExHelpers.priv_folder() |> File.read!() + defp module_outputs_tf, do: @module_outputs_tf |> DeployExHelpers.priv_folder() |> File.read!() + defp module_main_tf, do: @module_main_tf |> DeployExHelpers.priv_folder() |> File.read!() + defp module_variables_tf, do: @module_variables_tf |> DeployExHelpers.priv_folder() |> File.read!() + defp root_variables_tf_eex_oci, do: @root_variables_tf_eex_oci |> DeployExHelpers.priv_folder() |> File.read!() + + defp render(args), do: capture_io(fn -> Build.run(args) end) + + defp render_dir do + Path.join(System.tmp_dir!(), "p00_tf_lb_#{System.unique_integer([:positive])}") + end + + defp file_tree(dir) do + dir + |> Path.join("**/*") + |> Path.wildcard(match_dot: true) + |> Enum.map(&Path.relative_to(&1, dir)) + |> Enum.sort() + end + + describe "load_balancer.tf — resource declarations" do + test "T1: declares all six load-balancer resource types" do + contents = load_balancer_tf() + + assert contents =~ ~r/resource\s+"oci_core_network_security_group"\s+"load_balancer"/ + assert contents =~ ~r/resource\s+"oci_core_network_security_group_security_rule"/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_network_load_balancer"\s+"main"/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_backend_set"/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_backend"/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_listener"/ + refute contents =~ ~r/"oci_load_balancer_/ + end + + test "T2: the NLB sets is_private = false" do + assert load_balancer_tf() =~ ~r/is_private\s*=\s*false/ + end + + test "T3: backend sets carry policy = FIVE_TUPLE and is_preserve_source = true" do + contents = load_balancer_tf() + + assert length(Regex.scan(~r/policy\s*=\s*"FIVE_TUPLE"/, contents)) === 2 + assert length(Regex.scan(~r/is_preserve_source\s*=\s*true/, contents)) === 2 + end + + test "T4: seconds->millis conversion uses * 1000 and no hardcoded millisecond literal" do + contents = load_balancer_tf() + + assert contents =~ ~r/timeout_in_millis\s*=.*\*\s*1000/ + assert contents =~ ~r/interval_in_millis\s*=.*\*\s*1000/ + refute contents =~ ~r/_in_millis\s*=\s*\d/ + assert length(Regex.scan(~r/timeout_in_millis\s*=\s*local\.lb_timeout_in_millis/, contents)) === 2 + assert length(Regex.scan(~r/interval_in_millis\s*=\s*local\.lb_interval_in_millis/, contents)) === 2 + assert contents =~ ~r/lb_timeout_in_millis\s*=\s*var\.load_balancer_health_check_timeout_seconds\s*==\s*null\s*\?\s*null\s*:/ + assert contents =~ ~r/lb_interval_in_millis\s*=\s*var\.load_balancer_health_check_interval_seconds\s*==\s*null\s*\?\s*null\s*:/ + assert length(Regex.scan(~r/retries\s*=\s*var\.load_balancer_health_check_retries/, contents)) === 2 + end + + test "T5: backend-set protocol reads the declared lb_has_path local, defined as != \"\", and falls back to TCP" do + contents = load_balancer_tf() + + assert contents =~ ~r/lb_has_path\s*=\s*var\.load_balancer_health_check_path\s*!=\s*""/ + assert contents =~ ~r/protocol\s*=\s*local\.lb_has_path\s*\?\s*"HTTP"/ + assert contents =~ ~r/protocol\s*=\s*local\.lb_has_path\s*\?\s*"HTTPS"/ + assert contents =~ ~r/\?\s*"HTTP"\s*:\s*"TCP"/ + assert contents =~ ~r/\?\s*"HTTPS"\s*:\s*"TCP"/ + assert length(Regex.scan(~r/url_path\s*=\s*local\.lb_has_path\s*\?\s*var\.load_balancer_health_check_path\s*:\s*null/, contents)) === 2 + assert contents =~ ~r/return_code\s*=\s*local\.lb_has_path\s*\?\s*coalesce\(var\.load_balancer_health_check_return_code,\s*200\)\s*:\s*null/ + assert contents =~ ~r/return_code\s*=\s*local\.lb_has_path\s*\?\s*coalesce\(var\.load_balancer_health_check_https_return_code,\s*200\)\s*:\s*null/ + end + + test "T6: backend-set and listener names are the unqualified literals http/https" do + contents = load_balancer_tf() + + assert contents =~ ~r/name\s*=\s*"http"/ + assert contents =~ ~r/name\s*=\s*"https"/ + refute contents =~ ~r/name\s*=\s*"\$\{local\.kebab_instance_name\}-https?"/ + end + + test "T7: NSG rules cover 80 and 443, and the 443 rule is conditional on the https flag" do + contents = load_balancer_tf() + + assert contents =~ ~r/direction\s*=\s*"INGRESS"/ + assert contents =~ ~r/protocol\s*=\s*"6"/ + assert contents =~ ~r/source\s*=\s*"0\.0\.0\.0\/0"/ + assert contents =~ ~r/min\s*=\s*80(?!\d)/ + assert contents =~ ~r/min\s*=\s*443(?!\d)/ + assert contents =~ ~r/network_security_group_ids\s*=\s*oci_core_network_security_group\.load_balancer\[\*\]\.id/ + + assert contents =~ ~r/lb_https_count\s*=\s*var\.enable_load_balancer\s*&&\s*var\.enable_load_balancer_https/ + + assert contents =~ ~r/resource\s+"oci_core_network_security_group"\s+"load_balancer"\s*\{\s*count\s*=\s*local\.lb_count\b/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_network_load_balancer"\s+"main"\s*\{\s*count\s*=\s*local\.lb_count\b/ + assert contents =~ ~r/resource\s+"oci_core_network_security_group_security_rule"\s+"load_balancer_http"\s*\{\s*count\s*=\s*local\.lb_count\b/ + assert contents =~ ~r/resource\s+"oci_core_network_security_group_security_rule"\s+"load_balancer_https"\s*\{\s*count\s*=\s*local\.lb_https_count\b/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_backend_set"\s+"http"\s*\{\s*count\s*=\s*local\.lb_count\b/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_backend_set"\s+"https"\s*\{\s*count\s*=\s*local\.lb_https_count\b/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_listener"\s+"http"\s*\{\s*count\s*=\s*local\.lb_count\b/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_listener"\s+"https"\s*\{\s*count\s*=\s*local\.lb_https_count\b/ + end + + test "T20: listener ports are literals 80 and 443, protocol TCP" do + contents = load_balancer_tf() + + assert contents =~ ~r/resource\s+"oci_network_load_balancer_listener"\s+"http"[\s\S]*?port\s*=\s*80(?!\d)/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_listener"\s+"https"[\s\S]*?port\s*=\s*443(?!\d)/ + assert length(Regex.scan(~r/protocol\s*=\s*"TCP"/, contents)) === 2 + end + + test "T21: backend port matches its set and backend count includes the instance_count factor" do + contents = load_balancer_tf() + + assert contents =~ ~r/resource\s+"oci_network_load_balancer_backend"\s+"http"\s*\{[^}]*port\s*=\s*80(?!\d)/ + assert contents =~ ~r/resource\s+"oci_network_load_balancer_backend"\s+"https"\s*\{[^}]*port\s*=\s*443(?!\d)/ + assert contents =~ ~r/count\s*=\s*local\.lb_count\s*\*\s*var\.instance_count/ + assert contents =~ ~r/count\s*=\s*local\.lb_https_count\s*\*\s*var\.instance_count/ + end + + test "R1: reserved_ip_ocid is wired as a dynamic reserved_ips block" do + contents = load_balancer_tf() + + assert contents =~ ~r/dynamic\s+"reserved_ips"/ + assert contents =~ ~r/var\.reserved_ip_ocid\s*==\s*null/ + assert contents =~ ~r/for_each\s*=\s*var\.reserved_ip_ocid\s*==\s*null\s*\?\s*\[\]\s*:\s*\[var\.reserved_ip_ocid\]/ + end + end + + describe "module outputs.tf — load_balancer_public_ips" do + test "T9: filters on ip.is_public and never indexes the NLB as [0]" do + contents = module_outputs_tf() + + assert contents =~ ~r/output\s+"load_balancer_public_ips"/ + assert contents =~ ~r/if\s+ip\.is_public/ + refute contents =~ ~r/network_load_balancer\.main\[0\]/ + end + end + + describe "module main.tf — nsg_ids never indexes the LB NSG as [0]" do + test "T8: uses concat + splat, not a bare [0] index" do + contents = module_main_tf() + + assert contents =~ ~r/nsg_ids\s*=\s*concat\(var\.nsg_ids,\s*oci_core_network_security_group\.load_balancer\[\*\]\.id\)/ + refute contents =~ ~r/oci_core_network_security_group\.load_balancer\[0\]\.id/ + end + end + + describe "module variables.tf — enable_load_balancer_https default" do + test "T22: defaults to true, matching AWS's enable_elb_https default" do + contents = module_variables_tf() + + assert contents =~ ~r/variable\s+"enable_load_balancer_https"\s*\{[\s\S]*?default\s*=\s*true/ + end + end + + describe "AWS byte-parity regression guard" do + test "T10: root variables.tf.eex load_balancer block is untouched and aws-instance/main.tf has no oci_ resources" do + variables_contents = @root_variables_tf_eex |> DeployExHelpers.priv_folder() |> File.read!() + aws_main_contents = @aws_instance_main_tf |> DeployExHelpers.priv_folder() |> File.read!() + + expected_load_balancer_block = """ + load_balancer = optional(object({ + enable = optional(bool) + enable_https = optional(bool) + + port = optional(number) + instance_port = optional(number) + + health_check = optional(object({ + path = optional(string) + protocol = optional(string) + matcher = optional(string) + https_matcher = optional(string) + + unhealthy_threshold = optional(number) + healthy_threshold = optional(number) + timeout = optional(number) + interval = optional(number) + })) + })) +""" + + assert variables_contents =~ expected_load_balancer_block + + refute variables_contents =~ ~r/return_code\s*=\s*optional\(number\)/ + refute aws_main_contents =~ ~r/oci_/ + end + end + + describe "PrivFileSet — load_balancer.tf resolves for oci, not aws" do + setup do + {:ok, priv_path: DeployExHelpers.priv_folder("terraform")} + end + + test "T12: oci file set includes load_balancer.tf flattened; aws set has nothing under providers/", %{priv_path: priv_path} do + {:ok, oci_files} = DeployEx.Cloud.PrivFileSet.files(:oci, priv_path) + {:ok, aws_files} = DeployEx.Cloud.PrivFileSet.files(:aws, priv_path) + + assert {"providers/oci/modules/oci-instance/load_balancer.tf", "modules/oci-instance/load_balancer.tf"} in oci_files + refute Enum.any?(aws_files, fn {source, _dest} -> String.starts_with?(source, "providers/") end) + end + end + + describe "render — instance.tf wiring and outputs" do + setup do + dir = render_dir() + on_exit(fn -> File.rm_rf!(dir) end) + render(["--provider", "oci", "--render-dir", dir, "--pem-app-name", "s2-render", "--quiet"]) + + {:ok, dir: dir} + end + + test "T13: instance.tf wires vcn_id and all 8 load_balancer_* keys via try() with the module's own defaults", %{dir: dir} do + contents = Path.join(dir, "instance.tf") |> File.read!() + + assert contents =~ ~r/^\s*vcn_id\s*=\s*oci_core_vcn\.main\.id/m + assert contents =~ ~r/^\s*enable_load_balancer\s*=\s*try\(each\.value\.load_balancer\.enable,\s*false\)/m + assert contents =~ ~r/^\s*enable_load_balancer_https\s*=\s*try\(each\.value\.load_balancer\.enable_https,\s*true\)/m + assert contents =~ ~r/^\s*load_balancer_health_check_path\s*=\s*try\(each\.value\.load_balancer\.health_check\.path,\s*""\)/m + assert contents =~ ~r/^\s*load_balancer_health_check_return_code\s*=\s*try\(each\.value\.load_balancer\.health_check\.return_code,\s*null\)/m + assert contents =~ ~r/^\s*load_balancer_health_check_https_return_code\s*=\s*try\(each\.value\.load_balancer\.health_check\.https_return_code,\s*null\)/m + assert contents =~ ~r/^\s*load_balancer_health_check_retries\s*=\s*try\(each\.value\.load_balancer\.health_check\.unhealthy_threshold,\s*null\)/m + assert contents =~ ~r/^\s*load_balancer_health_check_timeout_seconds\s*=\s*try\(each\.value\.load_balancer\.health_check\.timeout,\s*null\)/m + assert contents =~ ~r/^\s*load_balancer_health_check_interval_seconds\s*=\s*try\(each\.value\.load_balancer\.health_check\.interval,\s*null\)/m + assert contents =~ ~r/^\s*reserved_ip_ocid\s*=\s*try\(each\.value\.load_balancer\.reserved_ip_ocid,\s*null\)/m + end + + test "T14: instance.tf wires neither load_balancer.port nor load_balancer.instance_port", %{dir: dir} do + contents = Path.join(dir, "instance.tf") |> File.read!() + + refute contents =~ ~r/load_balancer\.port\b/ + refute contents =~ ~r/load_balancer\.instance_port\b/ + refute contents =~ ~r/load_balancer\[\s*"(instance_)?port"\s*\]/ + end + + test "T15: outputs.tf exposes load_balancer_public_ips keyed by app, with a description", %{dir: dir} do + contents = Path.join(dir, "outputs.tf") |> File.read!() + + assert contents =~ ~r/output\s+"load_balancer_public_ips"/ + assert contents =~ ~r/for\s+app,\s*mod\s+in\s+module\.oci_instance\s*:\s*app\s*=>\s*mod\.load_balancer_public_ips/ + assert contents =~ ~r/output\s+"load_balancer_public_ips"\s*\{\s*description\s*=/ + end + end + + describe "render — aws/oci render sets stay disjoint on load_balancer.tf" do + test "T11: oci render carries load_balancer.tf; aws render has neither providers/ nor load_balancer.tf" do + aws_dir = render_dir() + on_exit(fn -> File.rm_rf!(aws_dir) end) + render(["--render-dir", aws_dir, "--pem-app-name", "s2-render-aws", "--quiet"]) + + oci_dir = render_dir() + on_exit(fn -> File.rm_rf!(oci_dir) end) + render(["--provider", "oci", "--render-dir", oci_dir, "--pem-app-name", "s2-render-oci", "--quiet"]) + + assert File.exists?(Path.join(oci_dir, "modules/oci-instance/load_balancer.tf")) + refute File.dir?(Path.join(aws_dir, "providers")) + refute Enum.any?(file_tree(aws_dir), &(Path.basename(&1) === "load_balancer.tf")) + end + end + + describe "root variables.tf.eex (oci) — recognized keys" do + test "T16: the _project description lists load_balancer among recognized keys" do + contents = root_variables_tf_eex_oci() + + assert contents =~ ~r/variable\s+"<%= @app_name %>_project"\s*\{\s*description\s*=\s*"[^"]*Recognized keys:[^"]*\bload_balancer\b[^"]*"/ + end + end + + describe "render determinism" do + test "T17: two pinned oci renders are byte-identical across the whole tree" do + one = render_dir() + on_exit(fn -> File.rm_rf!(one) end) + two = render_dir() + on_exit(fn -> File.rm_rf!(two) end) + + render(["--provider", "oci", "--render-dir", one, "--pem-app-name", "s2-determinism", "--quiet"]) + render(["--provider", "oci", "--render-dir", two, "--pem-app-name", "s2-determinism", "--quiet"]) + + assert file_tree(one) === file_tree(two) + + for relative_path <- file_tree(one), File.regular?(Path.join(one, relative_path)) do + assert File.read!(Path.join(one, relative_path)) === + File.read!(Path.join(two, relative_path)), + "#{relative_path} differed between runs" + end + end + end +end diff --git a/test/mix/tasks/terraform_build_render_test.exs b/test/mix/tasks/terraform_build_render_test.exs new file mode 100644 index 00000000..9da2c975 --- /dev/null +++ b/test/mix/tasks/terraform_build_render_test.exs @@ -0,0 +1,85 @@ +defmodule Mix.Tasks.Terraform.BuildRenderTest do + # async: false — drives a real Mix task and writes to the filesystem + use ExUnit.Case, async: false + + import ExUnit.CaptureIO + + alias Mix.Tasks.Terraform.Build + + setup do + dirs = Enum.map(1..2, fn index -> + Path.join(System.tmp_dir!(), "p00_tf_#{System.unique_integer([:positive])}_#{index}") + end) + + on_exit(fn -> Enum.each(dirs, &File.rm_rf!/1) end) + + {:ok, dirs: dirs} + end + + defp render(args), do: capture_io(fn -> Build.run(args) end) + + defp key_pair_contents(dir), do: File.read!(Path.join(dir, "key-pair-main.tf")) + + describe "--render-dir" do + test "renders the terraform set into a fresh directory", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + assert File.dir?(dir) + + for file <- ~w(variables.tf ec2.tf providers.tf key-pair-main.tf outputs.tf database.tf) do + assert File.exists?(Path.join(dir, file)), "expected #{file} in render dir" + end + end + + test "leaves no .eex templates behind", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + assert Path.wildcard(Path.join(dir, "**/*.eex")) === [] + end + + test "skips terraform init", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--quiet"]) + + refute File.exists?(Path.join(dir, ".terraform")) + end + + test "writes nothing outside the render dir", %{dirs: [dir | _]} do + deploys_before = File.exists?("./deploys") + + render(["--render-dir", dir, "--quiet"]) + + assert File.exists?("./deploys") === deploys_before + end + end + + describe "--pem-app-name" do + test "two runs with the same pinned name render identical bytes", %{dirs: [one, two]} do + render(["--render-dir", one, "--pem-app-name", "pinned-abc", "--quiet"]) + render(["--render-dir", two, "--pem-app-name", "pinned-abc", "--quiet"]) + + assert key_pair_contents(one) === key_pair_contents(two) + end + + test "two runs with different pinned names render different bytes", %{dirs: [one, two]} do + render(["--render-dir", one, "--pem-app-name", "pinned-abc", "--quiet"]) + render(["--render-dir", two, "--pem-app-name", "other-xyz", "--quiet"]) + + assert key_pair_contents(one) !== key_pair_contents(two) + end + + test "without the flag the pem name stays random per run", %{dirs: [one, two]} do + render(["--render-dir", one, "--quiet"]) + render(["--render-dir", two, "--quiet"]) + + assert key_pair_contents(one) !== key_pair_contents(two) + end + end + + describe "--db-password" do + test "is accepted and the run completes", %{dirs: [dir | _]} do + render(["--render-dir", dir, "--db-password", "pinnedpw", "--quiet"]) + + assert File.exists?(Path.join(dir, "database.tf")) + end + end +end diff --git a/test/mix/tasks/terraform_ebs_snapshot_pagination_test.exs b/test/mix/tasks/terraform_ebs_snapshot_pagination_test.exs new file mode 100644 index 00000000..94a49a0d --- /dev/null +++ b/test/mix/tasks/terraform_ebs_snapshot_pagination_test.exs @@ -0,0 +1,227 @@ +defmodule Mix.Tasks.Terraform.EbsSnapshotPaginationTest do + @moduledoc """ + Behavioural tests for the EC2 DescribeVolumes/DescribeSnapshots paginators added to + terraform.delete_ebs_snapshot.ex and terraform.create_ebs_snapshot.ex. + + Same pattern as test/deploy_ex/cloud/pagination_test.exs: replacing the recursion with a + single request makes these red. Responses are queued in the process dictionary -- per-PID + isolated, no setup or teardown, and no mocking library involved. + """ + + use ExUnit.Case, async: true + + alias Mix.Tasks.Terraform.CreateEbsSnapshot + alias Mix.Tasks.Terraform.DeleteEbsSnapshot + + defp queue_responses(responses), do: Process.put(:responses, responses) + + defp next_response do + [head | rest] = Process.get(:responses) + Process.put(:responses, rest) + Process.put(:call_count, (Process.get(:call_count) || 0) + 1) + + head + end + + defp call_count, do: Process.get(:call_count) || 0 + + defp recording_request_fn do + fn request, _config -> + next_token = request |> Map.get(:params, %{}) |> Map.get("NextToken") + Process.put(:next_tokens, (Process.get(:next_tokens) || []) ++ [next_token]) + + next_response() + end + end + + defp volumes_page(volume_ids, next_token) do + items = + Enum.map_join(volume_ids, "", fn id -> + "#{id}100" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, %{body: "#{items}#{token}"}} + end + + defp snapshots_page(snapshot_ids, next_token) do + items = + Enum.map_join(snapshot_ids, "", fn id -> + "#{id}vol-1" <> + "d2024-01-01T00:00:00.000Z" + end) + + token = if next_token, do: "#{next_token}", else: "" + + {:ok, + %{body: "#{items}#{token}"}} + end + + defp instance(instance_id), do: %{"instanceId" => instance_id} + defp volume(volume_id), do: %{"volumeId" => volume_id} + + describe "DeleteEbsSnapshot.find_volumes_for_instances/3 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([volumes_page(["vol-1", "vol-2"], "TOKEN"), volumes_page(["vol-3"], nil)]) + + assert {:ok, volumes} = + DeleteEbsSnapshot.find_volumes_for_instances( + "us-east-1", + [instance("i-1")], + request_fn: recording_request_fn() + ) + + assert Enum.map(volumes, & &1["volumeId"]) === ["vol-1", "vol-2", "vol-3"] + end + + test "terminates when no nextToken comes back" do + queue_responses([volumes_page(["vol-1"], nil)]) + + assert {:ok, [_only]} = + DeleteEbsSnapshot.find_volumes_for_instances( + "us-east-1", + [instance("i-1")], + request_fn: recording_request_fn() + ) + + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([volumes_page(["vol-1"], "TOKEN"), {:error, {:http_error, 500, %{body: "boom"}}}]) + + assert {:error, %ErrorMessage{}} = + DeleteEbsSnapshot.find_volumes_for_instances( + "us-east-1", + [instance("i-1")], + request_fn: recording_request_fn() + ) + end + end + + describe "DeleteEbsSnapshot.find_snapshots_for_volumes/3 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([snapshots_page(["snap-1", "snap-2"], "TOKEN"), snapshots_page(["snap-3"], nil)]) + + assert {:ok, snapshots} = + DeleteEbsSnapshot.find_snapshots_for_volumes( + "us-east-1", + [volume("vol-1")], + request_fn: recording_request_fn() + ) + + assert Enum.map(snapshots, & &1.snapshot_id) === ["snap-1", "snap-2", "snap-3"] + end + + test "terminates when no nextToken comes back" do + queue_responses([snapshots_page(["snap-1"], nil)]) + + assert {:ok, [_only]} = + DeleteEbsSnapshot.find_snapshots_for_volumes( + "us-east-1", + [volume("vol-1")], + request_fn: recording_request_fn() + ) + + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([snapshots_page(["snap-1"], "TOKEN"), {:error, {:http_error, 500, %{body: "boom"}}}]) + + assert {:error, %ErrorMessage{}} = + DeleteEbsSnapshot.find_snapshots_for_volumes( + "us-east-1", + [volume("vol-1")], + request_fn: recording_request_fn() + ) + end + end + + describe "DeleteEbsSnapshot.get_snapshots_by_ids/3 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([snapshots_page(["snap-1"], "TOKEN"), snapshots_page(["snap-2"], nil)]) + + assert {:ok, snapshots} = + DeleteEbsSnapshot.get_snapshots_by_ids( + "us-east-1", + ["snap-1", "snap-2"], + request_fn: recording_request_fn() + ) + + assert Enum.map(snapshots, & &1.snapshot_id) === ["snap-1", "snap-2"] + end + + test "terminates when no nextToken comes back" do + queue_responses([snapshots_page(["snap-1"], nil)]) + + assert {:ok, [_only]} = + DeleteEbsSnapshot.get_snapshots_by_ids( + "us-east-1", + ["snap-1"], + request_fn: recording_request_fn() + ) + + assert call_count() === 1 + end + + test "drops max_results instead of sending it alongside snapshot_ids" do + # AWS rejects DescribeSnapshots when SnapshotIds and MaxResults are both present + # ("InvalidParameterCombination") -- confirmed against a live account. + queue_responses([snapshots_page(["snap-1"], nil)]) + + request_fn = fn request, _config -> + refute Map.has_key?(request.params, "MaxResults") + next_response() + end + + assert {:ok, [_only]} = + DeleteEbsSnapshot.get_snapshots_by_ids( + "us-east-1", + ["snap-1"], + max_results: 5, + request_fn: request_fn + ) + end + end + + describe "CreateEbsSnapshot.find_volumes_for_instances/3 pagination" do + test "concatenates every page rather than returning the first" do + queue_responses([volumes_page(["vol-1", "vol-2"], "TOKEN"), volumes_page(["vol-3"], nil)]) + + assert {:ok, volumes} = + CreateEbsSnapshot.find_volumes_for_instances( + "us-east-1", + [instance("i-1")], + request_fn: recording_request_fn() + ) + + assert Enum.map(volumes, & &1["volumeId"]) === ["vol-1", "vol-2", "vol-3"] + end + + test "terminates when no nextToken comes back" do + queue_responses([volumes_page(["vol-1"], nil)]) + + assert {:ok, [_only]} = + CreateEbsSnapshot.find_volumes_for_instances( + "us-east-1", + [instance("i-1")], + request_fn: recording_request_fn() + ) + + assert call_count() === 1 + end + + test "an error on a later page fails loudly instead of returning a partial list" do + queue_responses([volumes_page(["vol-1"], "TOKEN"), {:error, {:http_error, 500, %{body: "boom"}}}]) + + assert {:error, %ErrorMessage{}} = + CreateEbsSnapshot.find_volumes_for_instances( + "us-east-1", + [instance("i-1")], + request_fn: recording_request_fn() + ) + end + end +end