From 97487b43db7f11722f47dc69d88351eae8db3ab1 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 13 Jul 2026 16:44:26 -0700 Subject: [PATCH 01/38] Update .gitignore to include Docker-related files and directories --- .gitignore | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index adb8d45..7c3451e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,13 @@ Icon* *.sql wcoa/fixtures/initial_data_archive-*.json wars/ -tomcat-users.xml \ No newline at end of file +tomcat-users.xml + +# Docker +docker/backups/ +docker/media/ +docker/static/ +docker/entrypoint.sh +docker/docker-requirements.txt +docker/wars/ +.env \ No newline at end of file From cb8025b9805214e9ef6c57e6ee16243c95ddfde7 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 14 Jul 2026 10:46:29 -0700 Subject: [PATCH 02/38] Add Docker configuration and setup for WCOA development environment - Update .gitignore to exclude new configuration files and directories - Create Taskfile.yml for managing Docker tasks - Add .env.example for environment variable configuration - Modify Dockerfile to support new build context and dependencies - Introduce docker-compose.yml for orchestrating services - Add config.wcoa.docker.ini for application-specific settings - Implement geoportal-entrypoint.sh for application startup and configuration - Create authentication and security XML templates for user management --- .gitignore | 5 + Taskfile.yml | 46 +++++ docker/.env.example | 34 ++++ docker/Dockerfile | 28 +-- docker/compose.yml | 77 ++++++++ docker/config.wcoa.docker.ini | 75 ++++++++ docker/geoportal-entrypoint.sh | 196 ++++++++++++++++++++ docker/templates/authentication-simple.xml | 21 +++ docker/templates/catalog-app-security.xml | 80 ++++++++ docker/templates/harvester-app-security.xml | 42 +++++ 10 files changed, 586 insertions(+), 18 deletions(-) create mode 100644 Taskfile.yml create mode 100644 docker/.env.example create mode 100644 docker/compose.yml create mode 100644 docker/config.wcoa.docker.ini create mode 100755 docker/geoportal-entrypoint.sh create mode 100644 docker/templates/authentication-simple.xml create mode 100644 docker/templates/catalog-app-security.xml create mode 100644 docker/templates/harvester-app-security.xml diff --git a/.gitignore b/.gitignore index 7c3451e..eb614f4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,11 @@ wcoa/fixtures/initial_data_archive-*.json wars/ tomcat-users.xml +config.wcoa.ini +config.wcoa.docker.ini +env/ +marco/static/ + # Docker docker/backups/ docker/media/ diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..ffc9f61 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,46 @@ +version: "3" + +vars: + CORE: ../../madrona-portal + COMPOSE: docker compose -f {{.CORE}}/docker/compose.base.yml -f docker/compose.yml --env-file docker/.env + +tasks: + base: + desc: Build the core base image locally. + cmds: + - docker build -t ghcr.io/ecotrust/madrona-portal:latest -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../ + + build: + desc: Build the WCOA overlay image. + cmds: + - "{{.COMPOSE}} build" + + up: + desc: Start the stack. + cmds: + - "{{.COMPOSE}} up" + + init: + desc: Start the stack and run DB init on startup. + cmds: + - "DB_INIT=1 {{.COMPOSE}} up" + + down: + desc: Stop the stack. + cmds: + - "{{.COMPOSE}} down" + + logs: + desc: Tail app logs. + cmds: + - "{{.COMPOSE}} logs -f app" + + manage: + desc: Run a Django manage.py command, e.g. task manage -- migrate. + cmds: + - "{{.COMPOSE}} exec app python marco/manage.py {{.CLI_ARGS}}" + + shell: + desc: Open Django shell. + cmds: + - "{{.COMPOSE}} exec app python marco/manage.py shell" \ No newline at end of file diff --git a/docker/.env.example b/docker/.env.example new file mode 100644 index 0000000..0fd97f1 --- /dev/null +++ b/docker/.env.example @@ -0,0 +1,34 @@ +COMPOSE_PROJECT_NAME=wcoa +BASE_TAG=latest + +APP_PORT=8000 +DB_PORT=5432 + +SECRET_KEY= +DEBUG=True +ALLOWED_HOSTS=localhost,127.0.0.1,::1 +DB_ENGINE=django.contrib.gis.db.backends.postgis +DB_NAME=wcoa_docker_db +DB_USER=postgres +DB_PASSWORD= +REDIS_PASSWORD= + +DB_INIT=0 +DJANGO_SUPERUSER_USERNAME=admin +DJANGO_SUPERUSER_EMAIL=admin@example.com +DJANGO_SUPERUSER_PASSWORD= +DJANGO_ENV=development +GUNICORN_WORKERS=3 + +RECAPTCHA_PUBLIC_KEY= +RECAPTCHA_PRIVATE_KEY= +GA_ACCOUNT= +ARCGIS_API_KEY= + +gpt_catalog_war=./wars/geoportal.war +gpt_harvester_war=./wars/harvester.war +gpt_frame_options=SAMEORIGIN +gpt_allowed_origin='"self" http://localhost:8000' +CLUSTER_NAME=wcoa-geoportal +ELASTIC_PASSWORD= +ES_REINDEX_REMOTE_WHITELIST=*.ecotrust.org:443,*.ecotrust.org:9200 diff --git a/docker/Dockerfile b/docker/Dockerfile index 919dc84..ae888eb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,21 +1,13 @@ -FROM tomcat:9-jdk17 +ARG BASE_IMAGE=ghcr.io/ecotrust/madrona-portal +ARG BASE_TAG=dockerdecouple +FROM ${BASE_IMAGE}:${BASE_TAG} -# ARG GEOPORTAL_VERSION=v265 -# ARG GEOPORTAL_VERSION=v272 -ARG GEOPORTAL_VERSION=v302 +# Build context is the wcoa repository root. +COPY --chown=madrona_user:madrona_user . ./apps/wcoa -RUN rm -rf /usr/local/tomcat/webapps/* -COPY ./wars/geoportal.war $CATALINA_HOME/webapps/geoportal.war -COPY ./wars/harvester.war $CATALINA_HOME/webapps/harvester.war -# COPY ./wars/v265/geoportal.war $CATALINA_HOME/webapps/geoportal.war -# COPY ./wars/v265/harvester.war $CATALINA_HOME/webapps/harvester.war -# COPY ./wars/v272/geoportal.war $CATALINA_HOME/webapps/geoportal.war -# COPY ./wars/v272/harvester.war $CATALINA_HOME/webapps/harvester.war -# COPY ./wars/${GEOPORTAL_VERSION}/geoportal.war $CATALINA_HOME/webapps/geoportal.war -# COPY ./wars/gpcatalog/esri_geoportal_302.war $CATALINA_HOME/webapps/geoportal.war -# COPY ./wars/${GEOPORTAL_VERSION}/harvester.war $CATALINA_HOME/webapps/harvester.war +RUN pip install --no-deps -e ./apps/wcoa && \ + if [ -s ./apps/wcoa/docker/requirements.txt ]; then \ + pip install -r ./apps/wcoa/docker/requirements.txt; \ + fi -RUN echo es_node=elastic >> /usr/local/tomcat/conf/catalina.properties -RUN rm $CATALINA_HOME/conf/tomcat-users.xml -COPY ./files/tomcat-users.xml $CATALINA_HOME/conf/tomcat-users.xml -CMD ["catalina.sh", "run"] \ No newline at end of file +ENV MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.docker.ini \ No newline at end of file diff --git a/docker/compose.yml b/docker/compose.yml new file mode 100644 index 0000000..f016ab5 --- /dev/null +++ b/docker/compose.yml @@ -0,0 +1,77 @@ +services: + app: + image: wcoa-portal:dev + build: + context: .. + dockerfile: docker/Dockerfile + args: + BASE_TAG: ${BASE_TAG:-latest} + env_file: + - .env + environment: + - MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.docker.ini + volumes: + - ./static:/vol/web/static + - ./media:/usr/local/apps/madrona-portal/media + - ../../../madrona-portal/marco:/usr/local/apps/madrona-portal/marco + - ..:/usr/local/apps/madrona-portal/apps/wcoa + - ../../mp-data-manager:/usr/local/apps/madrona-portal/apps/mp-data-manager + - ../../mp-layers:/usr/local/apps/madrona-portal/apps/mp-layers + - ../../mp-accounts:/usr/local/apps/madrona-portal/apps/mp-accounts + - ../../mp-drawing:/usr/local/apps/madrona-portal/apps/mp-drawing + - ../../mp-visualize:/usr/local/apps/madrona-portal/apps/mp-visualize + - ../../madrona-features:/usr/local/apps/madrona-portal/apps/madrona-features + - ../../madrona-manipulators:/usr/local/apps/madrona-portal/apps/madrona-manipulators + - ../../madrona-scenarios:/usr/local/apps/madrona-portal/apps/madrona-scenarios + - ../../mp-map-groups:/usr/local/apps/madrona-portal/apps/mp-map-groups + - ../../mp-explore:/usr/local/apps/madrona-portal/apps/mp-explore + - ../../mp-proxy:/usr/local/apps/madrona-portal/apps/mp-proxy + - ../../p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery + - ../../django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener + - ../../madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools + - ../../mp-survey:/usr/local/apps/madrona-portal/apps/mp-survey + + geoportal: + image: tomcat:9-jdk11 + ports: + - "8080:8080" + volumes: + - gp-volume:/usr/local/tomcat/webapps/ + - ${gpt_catalog_war}:/usr/local/tomcat/webapps/geoportal.war + - ${gpt_harvester_war}:/usr/local/tomcat/webapps/harvester.war + - ./templates:/templates:ro + - ./geoportal-entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + entrypoint: + - /usr/local/bin/entrypoint.sh + env_file: + - .env + depends_on: + elastic: + condition: service_healthy + restart: always + + elastic: + image: elasticsearch:8.19.12 + volumes: + - es-volume:/usr/share/elasticsearch/data + environment: + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms512m -Xmx512m + - cluster.name=${CLUSTER_NAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} + - xpack.security.enabled=false + ports: + - "9200:9200" + - "9300:9300" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] + interval: 30s + timeout: 10s + retries: 10 + restart: always + +volumes: + gp-volume: + es-volume: diff --git a/docker/config.wcoa.docker.ini b/docker/config.wcoa.docker.ini new file mode 100644 index 0000000..d610472 --- /dev/null +++ b/docker/config.wcoa.docker.ini @@ -0,0 +1,75 @@ +# Docker-focused configuration for WCOA local development. +# Use with MP_PROJECT_CONFIG=config.wcoa.docker.ini + +[APP] +APP_NAME = WCOA Portal +APP_URL = +APP_TEAM_NAME = Marine Planner Team +PROJECT_APP = wcoa +PROJECT_SETTINGS_FILE = True +DEBUG = True +TEMPLATE_DEBUG = True +ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] +# SECRET_KEY is loaded from environment variable: SECRET_KEY +MEDIA_ROOT = /usr/local/apps/madrona-portal/media +MEDIA_URL = /media/ +TIME_ZONE = UTC +GA_ACCOUNT = +# ReCAPTCHA keys are loaded from env vars: RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY +STATIC_ROOT = /vol/web/static +EMAIL_SUBJECT_PREFIX = [WCOA] +MAP_LIBRARY = ol8 +COMPRESS_ENABLED = True +STATIC_CORE = /vol/web/static/ +ADDITIONAL_APPS = [] +ADDITIONAL_MIDDLEWARE = [] + +[REGION] +NAME = West Coast Ocean +INIT_ZOOM = 6 +INIT_LAT = 39 +INIT_LON = -120 +MAP = ocean + +[CACHES] +BACKEND = django_redis.cache.RedisCache +LOCATION = redis://tasks:6379/1 +CLIENT_CLASS = django_redis.client.DefaultClient + +[CELERY] +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +CELERY_ALWAYS_EAGER = False +CELERY_DISABLE_RATE_LIMITS = True + +[DATABASE] +ENGINE = django.contrib.gis.db.backends.postgis +NAME = wcoa_docker_db +HOST = db +PORT = 5432 +USER = postgres +# DB password is loaded from environment variable: DB_PASSWORD + +[EMAIL] +HOST = localhost +PORT = 25 +# SMTP credentials are loaded from env vars: EMAIL_HOST_USER, EMAIL_HOST_PASSWORD +DEFAULT_FROM_EMAIL = Mid-Atlantic Portal +SERVER_EMAIL = MidA Site Errors + +[AWS] +# AWS credentials are loaded from env vars: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +AWS_SES_REGION_NAME = us-east-1 +AWS_SES_REGION_ENDPOINT = email.us-east-1.amazonaws.com + +[SOCIAL_AUTH] +# Social OAuth credentials are loaded from env vars: +# FACEBOOK_KEY, FACEBOOK_SECRET, TWITTER_KEY, TWITTER_SECRET, GOOGLE_KEY, GOOGLE_SECRET + + +[CATALOG] +DATA_CATALOG_ENABLED = False +CATALOG_TECHNOLOGY = GeoPortal2 +CATALOG_PROXY = +CATALOG_SOURCE = http://192.168.0.40:9200 +CATALOG_QUERY_ENDPOINT = /geoportal/elastic/metadata/item/_search/ diff --git a/docker/geoportal-entrypoint.sh b/docker/geoportal-entrypoint.sh new file mode 100755 index 0000000..692dd88 --- /dev/null +++ b/docker/geoportal-entrypoint.sh @@ -0,0 +1,196 @@ +#!/bin/bash +################################### +# This file 100% written by Copilot +################################### +set -e + +echo "Starting Geoportal with configuration override..." + +# Install gettext for envsubst command +echo "Installing gettext package for environment variable substitution..." +apt-get update -qq && apt-get install -y gettext-base && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Cleanup function for graceful shutdown +cleanup() { + if [ ! -z "$TOMCAT_PID" ] && kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Cleaning up Tomcat process (PID: $TOMCAT_PID)..." + kill $TOMCAT_PID 2>/dev/null + sleep 2 + if kill -0 $TOMCAT_PID 2>/dev/null; then + kill -9 $TOMCAT_PID 2>/dev/null + fi + fi +} +trap cleanup EXIT INT TERM + +# Function to wait for WAR deployment +wait_for_deployment() { + local app_name=$1 + local max_wait=120 + local wait_time=0 + + echo "Waiting for $app_name to deploy..." + while [ ! -d "/usr/local/tomcat/webapps/$app_name" ] && [ $wait_time -lt $max_wait ]; do + sleep 2 + wait_time=$((wait_time + 2)) + echo "Waiting... ${wait_time}s" + done + + if [ $wait_time -ge $max_wait ]; then + echo "ERROR: $app_name failed to deploy within ${max_wait} seconds" + return 1 + fi + + echo "$app_name deployed successfully" + return 0 +} + +# Function to substitute environment variables in templates +substitute_variables() { + local template_file=$1 + local output_file=$2 + + echo "Processing template: $template_file -> $output_file" + + # Validate required environment variables + local missing_vars=() + if [ -z "$gpt_frame_options" ]; then + missing_vars+=("gpt_frame_options") + fi + if [ -z "$gpt_allowed_origin" ]; then + missing_vars+=("gpt_allowed_origin") + fi + + if [ ${#missing_vars[@]} -gt 0 ]; then + echo "WARNING: Missing required environment variables: ${missing_vars[*]}" + echo "Check your .env file and ensure these variables are set" + fi + + # Display current values for debugging + echo " gpt_frame_options = '$gpt_frame_options'" + echo " gpt_allowed_origin = '$gpt_allowed_origin'" + + # Validate CSP format (check for problematic characters) + if echo "$gpt_allowed_origin" | grep -q ":.*\*"; then + echo " WARNING: Port wildcards (*) in CSP frame-ancestors may not be supported by all browsers" + echo " Consider using specific ports or removing wildcards if you encounter issues" + fi + + # Use envsubst to replace environment variables + envsubst < "$template_file" > "$output_file" + + if [ $? -eq 0 ]; then + echo "Successfully processed $template_file" + + # Show a sample of the processed content for verification + echo "Sample of processed content:" + grep -E "(frame-options|Content-Security-Policy)" "$output_file" | head -2 | sed 's/^/ /' + else + echo "ERROR: Failed to process $template_file" + return 1 + fi +} + +# Start Tomcat in background to deploy WARs +echo "Starting Tomcat to deploy applications..." +catalina.sh run & +TOMCAT_PID=$! +echo "Tomcat started with PID: $TOMCAT_PID" + +# Wait for both applications to deploy +wait_for_deployment "geoportal" || exit 1 +wait_for_deployment "harvester" || exit 1 + +# Additional wait to ensure full extraction +echo "Waiting for full application extraction..." +sleep 10 + +# Create config directory if it doesn't exist +CATALOG_CONFIG_DIR="/usr/local/tomcat/webapps/geoportal/WEB-INF/classes/config" +mkdir -p "$CATALOG_CONFIG_DIR" +HARVESTER_CONFIG_DIR="/usr/local/tomcat/webapps/harvester/WEB-INF/classes/config" +mkdir -p "$HARVESTER_CONFIG_DIR" + +# Process and copy authentication configuration +if [ -f "/templates/authentication-simple.xml" ]; then + substitute_variables "/templates/authentication-simple.xml" "$CATALOG_CONFIG_DIR/authentication-simple.xml" + substitute_variables "/templates/authentication-simple.xml" "$HARVESTER_CONFIG_DIR/authentication-simple.xml" +else + echo "WARNING: authentication-simple.xml template not found" +fi + +# Process and copy security configuration +if [ -f "/templates/catalog-app-security.xml" ] && [ -f "/templates/harvester-app-security.xml" ]; then + substitute_variables "/templates/catalog-app-security.xml" "$CATALOG_CONFIG_DIR/app-security.xml" + substitute_variables "/templates/harvester-app-security.xml" "$HARVESTER_CONFIG_DIR/app-security.xml" +else + echo "WARNING: app-security.xml templates not found" + if [ ! -f "/templates/catalog-app-security.xml" ]; then + echo " Missing: /templates/catalog-app-security.xml" + fi + if [ ! -f "/templates/harvester-app-security.xml" ]; then + echo " Missing: /templates/harvester-app-security.xml" + fi +fi + +# Verify the configuration files were created +echo "Verifying configuration files..." +if [ -f "$CATALOG_CONFIG_DIR/authentication-simple.xml" ]; then + echo "✓ authentication-simple.xml configured" +else + echo "✗ authentication-simple.xml missing" +fi + +if [ -f "$CATALOG_CONFIG_DIR/app-security.xml" ]; then + echo "✓ CATALOG app-security.xml configured" +else + echo "✗ CATALOG app-security.xml missing" +fi + +if [ -f "$HARVESTER_CONFIG_DIR/authentication-simple.xml" ]; then + echo "✓ HARVESTER authentication-simple.xml configured" +else + echo "✗ HARVESTER authentication-simple.xml missing" +fi + +if [ -f "$HARVESTER_CONFIG_DIR/app-security.xml" ]; then + echo "✓ app-security.xml configured" +else + echo "✗ app-security.xml missing" +fi + +# Stop background Tomcat gracefully so H2 can flush and release its lock +echo "Stopping background Tomcat (PID: $TOMCAT_PID)..." + +if kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Requesting graceful Tomcat shutdown via catalina.sh stop..." + catalina.sh stop 30 -force + # Wait for the background process to exit + for i in {1..35}; do + if ! kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Tomcat stopped gracefully" + break + fi + echo "Waiting for shutdown... ${i}/35" + sleep 1 + done + # Final safety net + if kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Force stopping Tomcat..." + kill -9 $TOMCAT_PID + sleep 2 + fi +else + echo "Tomcat process was not running (PID $TOMCAT_PID)" +fi + +echo "Tomcat stopped successfully" + +# Remove any stale H2 lock/trace files left by the background Tomcat run. +# These persist in the named volume and prevent the DB from opening on restart. +echo "Cleaning up stale H2 artifacts in /root..." +rm -f /root/harvester.lock.db /root/harvester.trace.db + +# Start Tomcat in foreground +echo "Starting Tomcat with updated configuration..." +exec catalina.sh run \ No newline at end of file diff --git a/docker/templates/authentication-simple.xml b/docker/templates/authentication-simple.xml new file mode 100644 index 0000000..eada6db --- /dev/null +++ b/docker/templates/authentication-simple.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker/templates/catalog-app-security.xml b/docker/templates/catalog-app-security.xml new file mode 100644 index 0000000..4b09d8c --- /dev/null +++ b/docker/templates/catalog-app-security.xml @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/templates/harvester-app-security.xml b/docker/templates/harvester-app-security.xml new file mode 100644 index 0000000..b93047f --- /dev/null +++ b/docker/templates/harvester-app-security.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From f547f50b00dc39809017f007ef55dfb1c3d123c2 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 14 Jul 2026 11:43:42 -0700 Subject: [PATCH 03/38] Fix gpt_allowed_origin format in .env.example --- docker/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.env.example b/docker/.env.example index 0fd97f1..fd0c5b5 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -28,7 +28,7 @@ ARCGIS_API_KEY= gpt_catalog_war=./wars/geoportal.war gpt_harvester_war=./wars/harvester.war gpt_frame_options=SAMEORIGIN -gpt_allowed_origin='"self" http://localhost:8000' +gpt_allowed_origin=http://localhost:8000 CLUSTER_NAME=wcoa-geoportal ELASTIC_PASSWORD= ES_REINDEX_REMOTE_WHITELIST=*.ecotrust.org:443,*.ecotrust.org:9200 From 23e916f23b57890dddb7db227c73fc3d93c62cdc Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 15 Jul 2026 16:32:05 -0700 Subject: [PATCH 04/38] Add db-restore script for PostgreSQL dump restoration in WCOA Docker DB --- scripts/db-restore.sh | 183 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100755 scripts/db-restore.sh diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh new file mode 100755 index 0000000..22f9b39 --- /dev/null +++ b/scripts/db-restore.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# db-restore.sh — Restore a PostgreSQL dump into the decoupled WCOA Docker DB. +# +# Usage: +# ./scripts/db-restore.sh +# ./scripts/db-restore.sh --drop # drop & recreate DB first +# ./scripts/db-restore.sh --env-file +# ./scripts/db-restore.sh --core-compose +# ./scripts/db-restore.sh --project-compose +# +# Run from anywhere — this script always operates relative to madrona-apps/wcoa/. +# +# Prerequisites: +# 1. Docker Compose stack is running with both decoupled compose files: +# docker compose \ +# -f ../../madrona-portal/docker/compose.base.yml \ +# -f docker/compose.yml \ +# --env-file docker/.env up -d +# 2. wcoa/docker/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. +# +# Options: +# --drop Terminate all active connections, drop, and recreate the +# target database before restoring. Required for a clean +# import from prod. Without this flag the dump is applied +# on top of existing data. +# --env-file Path to the .env file (default: ./docker/.env). +# --core-compose +# Path to compose.base.yml +# (default: ../../madrona-portal/docker/compose.base.yml). +# --project-compose +# Path to wcoa compose overlay +# (default: ./docker/compose.yml). +# +# Notes: +# - The dump is streamed directly into the db container (no temp files). +# - psql warnings (e.g. "already exists") are normal when importing a dump +# produced on a different Postgres version (12 → 16) and are not fatal. +# - After a --drop restore, run migrations to pick up any schema drift: +# docker compose -f -f --env-file \ +# exec app python marco/manage.py migrate +# ----------------------------------------------------------------------------- +set -euo pipefail + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +die() { echo "[db-restore] ERROR: $*" >&2; exit 1; } +info() { echo "[db-restore] $*"; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +DEFAULT_ENV_FILE="$PROJECT_ROOT/docker/.env" +DEFAULT_CORE_COMPOSE="$PROJECT_ROOT/../../madrona-portal/docker/compose.base.yml" +DEFAULT_PROJECT_COMPOSE="$PROJECT_ROOT/docker/compose.yml" + +resolve_to_abs() { + local path="$1" + [[ -n "$path" ]] || return 1 + if [[ "$path" == ~* ]]; then + path="${path/#\~/$HOME}" + fi + echo "$(cd "$(dirname "$path")" && pwd)/$(basename "$path")" +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +DROP_FIRST=false +DUMP_FILE="" +ENV_FILE="$DEFAULT_ENV_FILE" +CORE_COMPOSE_FILE="$DEFAULT_CORE_COMPOSE" +PROJECT_COMPOSE_FILE="$DEFAULT_PROJECT_COMPOSE" + +while [[ $# -gt 0 ]]; do + case "$1" in + --drop) DROP_FIRST=true; shift ;; + --env-file) [[ -n "${2:-}" ]] || die "--env-file requires a path argument" + ENV_FILE="$2"; shift 2 ;; + --core-compose) + [[ -n "${2:-}" ]] || die "--core-compose requires a path argument" + CORE_COMPOSE_FILE="$2"; shift 2 ;; + --project-compose) + [[ -n "${2:-}" ]] || die "--project-compose requires a path argument" + PROJECT_COMPOSE_FILE="$2"; shift 2 ;; + -*) die "Unknown option: '$1'. Usage: $0 [--drop] [--env-file ] [--core-compose ] [--project-compose ] " ;; + *) [[ -z "$DUMP_FILE" ]] || die "Unexpected argument: '$1'" + DUMP_FILE="$1"; shift ;; + esac +done + +[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] [--env-file ] [--core-compose ] [--project-compose ] " + +# Resolve dump path before we cd away. +DUMP_ABS="$(resolve_to_abs "$DUMP_FILE")" +[[ -f "$DUMP_ABS" ]] || die "Dump file not found: $DUMP_FILE" + +ENV_FILE_ABS="$(resolve_to_abs "$ENV_FILE")" +[[ -f "$ENV_FILE_ABS" ]] || die "Env file not found: $ENV_FILE" + +CORE_COMPOSE_ABS="$(resolve_to_abs "$CORE_COMPOSE_FILE")" +[[ -f "$CORE_COMPOSE_ABS" ]] || die "Core compose file not found: $CORE_COMPOSE_FILE" + +PROJECT_COMPOSE_ABS="$(resolve_to_abs "$PROJECT_COMPOSE_FILE")" +[[ -f "$PROJECT_COMPOSE_ABS" ]] || die "Project compose file not found: $PROJECT_COMPOSE_FILE" + +# --------------------------------------------------------------------------- +# Always operate from the wcoa repo regardless of where the script is called +# --------------------------------------------------------------------------- +cd "$PROJECT_ROOT" + +# --------------------------------------------------------------------------- +# Load .env for DB credentials and DJANGO_ENV +# --------------------------------------------------------------------------- +set -a +# shellcheck source=/dev/null +source "$ENV_FILE_ABS" +set +a + +DB_NAME="${DB_NAME:-wcoa_docker_db}" +DB_USER="${DB_USER:-postgres}" +DB_PASSWORD="${DB_PASSWORD:?DB_PASSWORD must be set in .env}" + +# --------------------------------------------------------------------------- +# Compose command helpers +# --------------------------------------------------------------------------- +compose() { + docker compose -f "$CORE_COMPOSE_ABS" -f "$PROJECT_COMPOSE_ABS" --env-file "$ENV_FILE_ABS" "$@" +} + +psql_exec() { + compose exec -T -e "PGPASSWORD=$DB_PASSWORD" db psql -U "$DB_USER" "$@" +} + +info "Using core compose: $CORE_COMPOSE_ABS" +info "Using project compose: $PROJECT_COMPOSE_ABS" +info "Using env file: $ENV_FILE_ABS" + +# --------------------------------------------------------------------------- +# Verify the db container is healthy before doing anything +# --------------------------------------------------------------------------- +info "Checking db service health..." +compose ps db | grep -q "healthy" \ + || die "db container is not healthy. Is the stack running? Try: docker compose -f $CORE_COMPOSE_ABS -f $PROJECT_COMPOSE_ABS --env-file $ENV_FILE_ABS up -d" + +# --------------------------------------------------------------------------- +# Optional: terminate connections, drop, and recreate the database +# --------------------------------------------------------------------------- +if [[ "$DROP_FIRST" == true ]]; then + info "Terminating active connections to '$DB_NAME'..." + psql_exec -d postgres -c \ + "SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid();" \ + > /dev/null + + info "Dropping database '$DB_NAME'..." + psql_exec -d postgres -c "DROP DATABASE IF EXISTS \"$DB_NAME\";" + + info "Creating database '$DB_NAME'..." + psql_exec -d postgres -c "CREATE DATABASE \"$DB_NAME\";" + + info "Enabling PostGIS extension..." + psql_exec -d "$DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS postgis;" +fi + +# --------------------------------------------------------------------------- +# Stream the dump into the database +# --------------------------------------------------------------------------- +DUMP_SIZE="$(du -sh "$DUMP_ABS" | cut -f1)" +info "Restoring '$DUMP_FILE' (${DUMP_SIZE}) → '$DB_NAME'..." +info "psql warnings about existing objects are expected and non-fatal." + +psql_exec -d "$DB_NAME" \ + --set ON_ERROR_STOP=off \ + < "$DUMP_ABS" + +info "Restore complete." +info "" +info "Next steps:" +info " Apply any pending migrations:" +info " docker compose -f $CORE_COMPOSE_ABS -f $PROJECT_COMPOSE_ABS --env-file $ENV_FILE_ABS exec app python marco/manage.py migrate" From 669629c36fab89a9c231806b218a53abb1064561 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 16 Jul 2026 12:25:06 -0700 Subject: [PATCH 05/38] Update Taskfile to include environment variable loading for Docker build and add comment section in .env.example --- Taskfile.yml | 2 +- docker/.env.example | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index ffc9f61..750005c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -8,7 +8,7 @@ tasks: base: desc: Build the core base image locally. cmds: - - docker build -t ghcr.io/ecotrust/madrona-portal:latest -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../ + - sh -c 'if [ -f docker/.env ]; then set -a; . docker/.env; set +a; fi; docker build -t ghcr.io/ecotrust/madrona-portal:${BASE_TAG:-latest} -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../ build: desc: Build the WCOA overlay image. diff --git a/docker/.env.example b/docker/.env.example index fd0c5b5..439c658 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -20,6 +20,7 @@ DJANGO_SUPERUSER_PASSWORD= DJANGO_ENV=development GUNICORN_WORKERS=3 +# Optional integrations RECAPTCHA_PUBLIC_KEY= RECAPTCHA_PRIVATE_KEY= GA_ACCOUNT= From c55f85c701909a385910d3b736295a94a7b8b5d1 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 16 Jul 2026 17:10:35 -0700 Subject: [PATCH 06/38] Refactor Docker compose configuration and add requirements file for mida image --- Taskfile.yml | 2 +- docker/compose.yml | 106 +++++++++++++++++++++++++++++++++------- docker/requirements.txt | 2 + 3 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 docker/requirements.txt diff --git a/Taskfile.yml b/Taskfile.yml index 750005c..6eeef04 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,7 +2,7 @@ version: "3" vars: CORE: ../../madrona-portal - COMPOSE: docker compose -f {{.CORE}}/docker/compose.base.yml -f docker/compose.yml --env-file docker/.env + COMPOSE: docker compose -f docker/compose.yml -f {{.CORE}}/docker/compose.base.yml --env-file docker/.env tasks: base: diff --git a/docker/compose.yml b/docker/compose.yml index f016ab5..817caff 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -2,8 +2,8 @@ services: app: image: wcoa-portal:dev build: - context: .. - dockerfile: docker/Dockerfile + context: ../../ + dockerfile: madrona-apps/wcoa/docker/Dockerfile args: BASE_TAG: ${BASE_TAG:-latest} env_file: @@ -13,23 +13,91 @@ services: volumes: - ./static:/vol/web/static - ./media:/usr/local/apps/madrona-portal/media - - ../../../madrona-portal/marco:/usr/local/apps/madrona-portal/marco - - ..:/usr/local/apps/madrona-portal/apps/wcoa - - ../../mp-data-manager:/usr/local/apps/madrona-portal/apps/mp-data-manager - - ../../mp-layers:/usr/local/apps/madrona-portal/apps/mp-layers - - ../../mp-accounts:/usr/local/apps/madrona-portal/apps/mp-accounts - - ../../mp-drawing:/usr/local/apps/madrona-portal/apps/mp-drawing - - ../../mp-visualize:/usr/local/apps/madrona-portal/apps/mp-visualize - - ../../madrona-features:/usr/local/apps/madrona-portal/apps/madrona-features - - ../../madrona-manipulators:/usr/local/apps/madrona-portal/apps/madrona-manipulators - - ../../madrona-scenarios:/usr/local/apps/madrona-portal/apps/madrona-scenarios - - ../../mp-map-groups:/usr/local/apps/madrona-portal/apps/mp-map-groups - - ../../mp-explore:/usr/local/apps/madrona-portal/apps/mp-explore - - ../../mp-proxy:/usr/local/apps/madrona-portal/apps/mp-proxy - - ../../p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery - - ../../django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener - - ../../madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools - - ../../mp-survey:/usr/local/apps/madrona-portal/apps/mp-survey + - type: bind + source: ../../../madrona-portal/marco + target: /usr/local/apps/madrona-portal/marco + bind: + create_host_path: false + - type: bind + source: .. + target: /usr/local/apps/madrona-portal/apps/wcoa + bind: + create_host_path: false + - type: bind + source: ../../mp-data-manager + target: /usr/local/apps/madrona-portal/apps/mp-data-manager + bind: + create_host_path: false + - type: bind + source: ../../mp-layers + target: /usr/local/apps/madrona-portal/apps/mp-layers + bind: + create_host_path: false + - type: bind + source: ../../mp-accounts + target: /usr/local/apps/madrona-portal/apps/mp-accounts + bind: + create_host_path: false + - type: bind + source: ../../mp-drawing + target: /usr/local/apps/madrona-portal/apps/mp-drawing + bind: + create_host_path: false + - type: bind + source: ../../mp-visualize + target: /usr/local/apps/madrona-portal/apps/mp-visualize + bind: + create_host_path: false + - type: bind + source: ../../madrona-features + target: /usr/local/apps/madrona-portal/apps/madrona-features + bind: + create_host_path: false + - type: bind + source: ../../madrona-manipulators + target: /usr/local/apps/madrona-portal/apps/madrona-manipulators + bind: + create_host_path: false + - type: bind + source: ../../madrona-scenarios + target: /usr/local/apps/madrona-portal/apps/madrona-scenarios + bind: + create_host_path: false + - type: bind + source: ../../mp-map-groups + target: /usr/local/apps/madrona-portal/apps/mp-map-groups + bind: + create_host_path: false + - type: bind + source: ../../mp-explore + target: /usr/local/apps/madrona-portal/apps/mp-explore + bind: + create_host_path: false + - type: bind + source: ../../mp-proxy + target: /usr/local/apps/madrona-portal/apps/mp-proxy + bind: + create_host_path: false + - type: bind + source: ../../p97-nursery + target: /usr/local/apps/madrona-portal/apps/p97-nursery + bind: + create_host_path: false + - type: bind + source: ../../django_url_shortener + target: /usr/local/apps/madrona-portal/apps/django_url_shortener + bind: + create_host_path: false + - type: bind + source: ../../madrona-analysistools + target: /usr/local/apps/madrona-portal/apps/madrona-analysistools + bind: + create_host_path: false + - type: bind + source: ../../mp-survey + target: /usr/local/apps/madrona-portal/apps/mp-survey + bind: + create_host_path: false geoportal: image: tomcat:9-jdk11 diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000..045cd20 --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1,2 @@ +# Project-only Python dependencies for mida Docker image. +# Leave empty unless mida requires packages not already in the base image. \ No newline at end of file From 8d4e2974af30664c7eea71d39e4322d94086ab49 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 12:49:40 -0700 Subject: [PATCH 07/38] Add Docker quickstart section to README --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index d649af3..b28281d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,15 @@ West Coast Ocean Alliance (WCOA) Data Portal Django web application and Wagtail CMS for the West Coast Ocean Data Portal. +## Docker quickstart + +Use the split base/overlay workflow. + +```bash +cp docker/.env.example docker/.env +task base build init +``` + ## Local Vagrant Development Env requirements: From 3dc3468302dbe92b458de2b49a6898065d29dc99 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 12:51:26 -0700 Subject: [PATCH 08/38] Add GitHub Actions workflow for building and publishing wcoa image --- .github/workflows/build-and-publish-image.yml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/build-and-publish-image.yml diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml new file mode 100644 index 0000000..8306d7c --- /dev/null +++ b/.github/workflows/build-and-publish-image.yml @@ -0,0 +1,49 @@ +name: Build and publish wcoa image + +on: + push: + branches: [main, dockerdecouple] + workflow_dispatch: + +env: + BASE_TAG: 2026.07 + IMAGE_NAME: ghcr.io/ecotrust/wcoa + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract short SHA + id: meta + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + push: true + build-args: | + BASE_TAG=${{ env.BASE_TAG }} + tags: | + ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} + ${{ env.IMAGE_NAME }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max From 806002ffc80390fd4f2bdc4417413d7ab42cff27 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 12:51:35 -0700 Subject: [PATCH 09/38] Create production Docker Compose configuration --- docker/compose.prod.yml | 163 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 docker/compose.prod.yml diff --git a/docker/compose.prod.yml b/docker/compose.prod.yml new file mode 100644 index 0000000..d394be2 --- /dev/null +++ b/docker/compose.prod.yml @@ -0,0 +1,163 @@ +services: + app: + image: ghcr.io/ecotrust/wcoa:${IMAGE_TAG:-latest} + volumes: + - ./static:/vol/web/static + - ./media:/usr/local/apps/madrona-portal/media + - ./config.wcoa.docker.ini:/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.docker.ini:ro + env_file: + - ./.env + environment: + - DB_INIT=${DB_INIT:-0} + - MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.docker.ini + - SECRET_KEY=${SECRET_KEY} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} + - DEBUG=${DEBUG:-False} + - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} + - DB_NAME=${DB_NAME:-wcoa_docker_db} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=db + - DB_PORT=5432 + - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 + - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@example.com} + - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-} + - DJANGO_ENV=${DJANGO_ENV:-production} + - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} + depends_on: + db: + condition: service_healthy + tasks: + condition: service_healthy + ports: + - "${APP_PORT:-8008}:8008" + networks: + - madronanetwork + restart: unless-stopped + + db: + image: postgis/postgis:16-3.4 + volumes: + - postgis_data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=${DB_USER:-postgres} + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=${DB_NAME:-wcoa_docker_db} + ports: + - "${DB_PORT:-5432}:5432" + networks: + - madronanetwork + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-wcoa_docker_db}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + tasks: + image: redis:7-alpine + command: redis-server ${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}} + ports: + - "${REDIS_PORT:-6379}:6379" + volumes: + - redis_data:/data + networks: + - madronanetwork + healthcheck: + test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + geoportal: + image: tomcat:9-jdk11 + ports: + - 8080:8080 + volumes: + - gp-volume:/usr/local/tomcat/webapps/ + - harvester_data:/root + - ${gpt_catalog_war}:/usr/local/tomcat/webapps/geoportal.war + - ${gpt_harvester_war}:/usr/local/tomcat/webapps/harvester.war + - ./templates:/templates:ro + - ./geoportal-entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + entrypoint: ["/usr/local/bin/entrypoint.sh"] + networks: + - madronanetwork + restart: always + env_file: + - ./.env + depends_on: + elastic: + condition: service_healthy + + elastic: + image: elasticsearch:8.19.12 + volumes: + - es-volume:/usr/share/elasticsearch/data + - ./backups/elasticsearch:/usr/share/elasticsearch/backups + environment: + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms512m -Xmx512m + - cluster.name=${CLUSTER_NAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} + - xpack.security.enabled=false + - path.repo=/usr/share/elasticsearch/backups + ports: + - 9200:9200 + - 9300:9300 + networks: + - madronanetwork + group_add: + - "0" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] + interval: 30s + timeout: 10s + retries: 10 + restart: always + + kibana: + image: kibana:8.19.12 + profiles: ["dev"] + ports: + - 127.0.0.1:5601:5601 + environment: + ELASTICSEARCH_HOSTS: http://elastic:9200 + networks: + - madronanetwork + depends_on: + elastic: + condition: service_healthy + restart: no + + nginx: + image: nginx:alpine + profiles: ["dev"] + ports: + - "8081:80" + volumes: + - ./nginx-dev.conf:/etc/nginx/conf.d/default.conf:ro + - ./static:/vol/web/static:ro + - ./media:/usr/local/apps/madrona-portal/media:ro + networks: + - madronanetwork + depends_on: + - app + - geoportal + restart: no + +volumes: + postgis_data: + redis_data: + gp-volume: + es-volume: + harvester_data: + +networks: + madronanetwork: + driver: bridge From ae463ef61dd752fb628b731b0f91b5e51b36fbff Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 12:51:41 -0700 Subject: [PATCH 10/38] Fix syntax error in base task command in Taskfile.yml --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index 6eeef04..db97263 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -8,7 +8,7 @@ tasks: base: desc: Build the core base image locally. cmds: - - sh -c 'if [ -f docker/.env ]; then set -a; . docker/.env; set +a; fi; docker build -t ghcr.io/ecotrust/madrona-portal:${BASE_TAG:-latest} -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../ + - sh -c 'if [ -f docker/.env ]; then set -a; . docker/.env; set +a; fi; docker build -t ghcr.io/ecotrust/madrona-portal:${BASE_TAG:-latest} -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../' build: desc: Build the WCOA overlay image. From ea9a6d7bb790a70d6ca07aedd02be00a4c9bd318 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 15:33:51 -0700 Subject: [PATCH 11/38] Update actions/checkout to v6 in build-and-publish-image.yml --- .github/workflows/build-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 8306d7c..5639814 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From c9cd7c46529a29b08ffb80d6f74652727193a576 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 15:51:33 -0700 Subject: [PATCH 12/38] Update BASE_TAG to 'latest' in Dockerfile and fix quoting in Taskfile.yml --- Taskfile.yml | 2 +- docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index db97263..cb5bab5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -8,7 +8,7 @@ tasks: base: desc: Build the core base image locally. cmds: - - sh -c 'if [ -f docker/.env ]; then set -a; . docker/.env; set +a; fi; docker build -t ghcr.io/ecotrust/madrona-portal:${BASE_TAG:-latest} -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../' + - sh -c "if [ -f docker/.env ]; then set -a; . docker/.env; set +a; fi; docker build -t ghcr.io/ecotrust/madrona-portal:\${BASE_TAG:-latest} -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../" build: desc: Build the WCOA overlay image. diff --git a/docker/Dockerfile b/docker/Dockerfile index ae888eb..d06e8ad 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE=ghcr.io/ecotrust/madrona-portal -ARG BASE_TAG=dockerdecouple +ARG BASE_TAG=latest FROM ${BASE_IMAGE}:${BASE_TAG} # Build context is the wcoa repository root. From 2a95cc0f9e3dbece690bcf1cda0c7bd7f78ac968 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 17:16:13 -0700 Subject: [PATCH 13/38] Update BASE_TAG in build-and-publish-image.yml and improve environment variable handling in compose.prod.yml --- .github/workflows/build-and-publish-image.yml | 2 +- docker/compose.prod.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 5639814..1842a98 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: env: - BASE_TAG: 2026.07 + BASE_TAG: sha256:7b927cacf635dede11390524f4849450661f927636c15eae08979da9be551d89 IMAGE_NAME: ghcr.io/ecotrust/wcoa jobs: diff --git a/docker/compose.prod.yml b/docker/compose.prod.yml index d394be2..87111bb 100644 --- a/docker/compose.prod.yml +++ b/docker/compose.prod.yml @@ -10,13 +10,13 @@ services: environment: - DB_INIT=${DB_INIT:-0} - MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.docker.ini - - SECRET_KEY=${SECRET_KEY} + - SECRET_KEY=${SECRET_KEY:?SECRET_KEY must be set} - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} - DEBUG=${DEBUG:-False} - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} - DB_NAME=${DB_NAME:-wcoa_docker_db} - DB_USER=${DB_USER:-postgres} - - DB_PASSWORD=${DB_PASSWORD} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD must be set} - DB_HOST=db - DB_PORT=5432 - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 From 9b56487d83271a7f9902bbdb3700705a4bac4b6d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 17:20:35 -0700 Subject: [PATCH 14/38] Update BASE_TAG in build-and-publish-image.yml to a specific commit SHA --- .github/workflows/build-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 1842a98..974ef89 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: env: - BASE_TAG: sha256:7b927cacf635dede11390524f4849450661f927636c15eae08979da9be551d89 + BASE_TAG: bc0fe1d IMAGE_NAME: ghcr.io/ecotrust/wcoa jobs: From 5b46cd26696360ba80d2454de828aa4f9817d294 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 23 Jul 2026 13:59:02 -0700 Subject: [PATCH 15/38] Fix build context and dockerfile path in compose.yml for app service --- docker/compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/compose.yml b/docker/compose.yml index 817caff..5e2b186 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -2,8 +2,8 @@ services: app: image: wcoa-portal:dev build: - context: ../../ - dockerfile: madrona-apps/wcoa/docker/Dockerfile + context: .. + dockerfile: docker/Dockerfile args: BASE_TAG: ${BASE_TAG:-latest} env_file: From d73101dc8ef71efa7fdeb32cf2fa4348896a17d7 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 27 Jul 2026 16:02:28 -0700 Subject: [PATCH 16/38] Update environment variables in .env.example using local .env --- docker/.env.example | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 439c658..ee90b92 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -28,8 +28,20 @@ ARCGIS_API_KEY= gpt_catalog_war=./wars/geoportal.war gpt_harvester_war=./wars/harvester.war -gpt_frame_options=SAMEORIGIN -gpt_allowed_origin=http://localhost:8000 -CLUSTER_NAME=wcoa-geoportal +gpt_frame_options=DENY +gpt_allowed_origin="localhost localhost:* *.westcoastoceans.org *.ecotrust.org 52.33.200.130" +gpt_admin_username= +gpt_admin_password= +gpt_publisher_username= +gpt_publisher_password= +gpt_user_username= +gpt_user_password= +gpt_esri_username= +gpt_esri_password= + +CLUSTER_NAME=elasticsearch ELASTIC_PASSWORD= -ES_REINDEX_REMOTE_WHITELIST=*.ecotrust.org:443,*.ecotrust.org:9200 +ES_REINDEX_REMOTE_WHITELIST="elastic.prod.wcoa.ecotrust.org:80" +ES_PORT=9200 +ES_MEM_LIMIT=1073741824 +ES_NODE=elastic \ No newline at end of file From 86939956ff67ded9ea9c22e52d507663cc7617e8 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 30 Jul 2026 15:38:25 -0700 Subject: [PATCH 17/38] Refactor README.md to streamline setup instructions and remove legacy Vagrant development steps --- README.md | 572 ++++++++++++++++++------------------------------------ 1 file changed, 184 insertions(+), 388 deletions(-) diff --git a/README.md b/README.md index b28281d..860e1f3 100644 --- a/README.md +++ b/README.md @@ -3,408 +3,204 @@ West Coast Ocean Alliance (WCOA) Data Portal Django web application and Wagtail CMS for the West Coast Ocean Data Portal. -## Docker quickstart +This repository is the WCOA portal application for the [Madrona Portal](https://github.com/Ecotrust/madrona-portal) platform. + +## Prerequisites + +- Docker Desktop or Docker Engine with Compose v2 +- [go-task](https://taskfile.dev) (`brew install go-task`) +- Workspace with side-by-side [madrona-portal](https://github.com/Ecotrust/madrona-portal) repo and madrona-apps folder containing this repo and other portal repos: + +```text +madrona/ +├── madrona-portal/ +└── madrona-apps/ + └── wcoa/ + └── django_url_shortener/ + └── madrona-analysistools/ + └── etc. +``` + +```bash +mkdir madrona +cd madrona +git clone https://github.com/Ecotrust/madrona-portal.git +mkdir madrona-apps +cd madrona-apps +git clone https://github.com/Ecotrust/wcoa.git +git clone https://github.com/Ecotrust/django_url_shortener.git +git clone https://github.com/Ecotrust/madrona-analysistools.git +git clone https://github.com/Ecotrust/madrona-features.git +git clone https://github.com/Ecotrust/madrona-manipulators.git +git clone https://github.com/Ecotrust/madrona-scenarios.git +git clone https://github.com/Ecotrust/mp-accounts.git +git clone https://github.com/Ecotrust/mp-data-manager.git +git clone https://github.com/Ecotrust/mp-drawing.git +git clone https://github.com/Ecotrust/mp-explore.git +git clone https://github.com/Ecotrust/mp-layers.git +git clone https://github.com/Ecotrust/mp-map-groups.git +git clone https://github.com/Ecotrust/mp-proxy.git +git clone https://github.com/Ecotrust/mp-survey.git +git clone https://github.com/Ecotrust/mp-visualize.git +git clone https://github.com/Ecotrust/p97-nursery.git +``` + +## Quickstart (local development) + +Run all commands from this repository root (`madrona-apps/wcoa`). + +```bash +cd wcoa +``` -Use the split base/overlay workflow. +Create your local environment file: ```bash cp docker/.env.example docker/.env -task base build init ``` -## Local Vagrant Development Env - -requirements: - - git - - vagrant - -1. Choose a working project directory that will be referred to from here has PROJDIR (i.e. /home/username/src/ ). It is best if it’s a directory you use to hold all of your dev projects. - ``` - cd PROJDIR - git clone https://github.com/Ecotrust/madrona-portal.git - mv madrona-portal ocean_portal - cd ocean_portal - vagrant up - vagrant ssh - ``` - -2. Install dependencies - ``` - sudo apt update - sudo apt upgrade -y - sudo apt install git python3 python3-dev python3-virtualenv python3-pip postgresql postgresql-contrib postgis postgresql-server-dev-14 libjpeg-dev gdal-bin python3-gdal libgdal-dev -y - ``` - -3. Edit requirements.txt - ``` - vim /usr/local/apps/ocean_portal/requirements.txt - ``` - -4. Add WCOA app to requirements.txt - ``` - -e git+https://github.com/Ecotrust/wcoa.git@master#egg=wcoa-master - ``` - -5. Set up virtualenv - ``` - python3 -m pip install --user virtualenv - cd /usr/local/apps/ - sudo chown ${USER}:${USER} ./ - python3 -m virtualenv env - source /usr/local/apps/env/bin/activate - pip install -r /usr/local/apps/ocean_portal/requirements.txt - pip uninstall numpy - gdal-config --version - pip install "pygdal<'REPLACE with gdal-config version'" - ``` - if any of your packages were copied locally rather than pulled via requirements.txt, use pip to install them now: - ``` - pip install -e /usr/local/apps/ocean_portal/apps/... - ``` - -6. Install database - ``` - sudo -u postgres createdb -O postgres ocean_portal - sudo -u postgres psql -c "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;" ocean_portal - sudo vim /etc/postgresql/12/main/pg_hba.conf - #-------- - - #--------- - sudo service postgresql restart - ``` - -7. Configure project - ``` - cd /usr/local/apps/ocean_portal/marco - mkdir media - mkdir static - mkdir /usr/local/apps/marco_portal_static/ - cp config.ini.template config.ini - vim config.ini - ``` - -8. Edit config.ini - - Add the following lines under `[App]`: - ``` - PROJECT_APP = wcoa - PROJECT_SETTINGS_FILE = True - MEDIA_ROOT = /usr/local/apps/ocean_portal/marco/media - STATIC_ROOT = /usr/local/apps/ocean_portal/marco/static - ``` - - Add the following under [DATABASE]: - ``` - USER = postgres - NAME = ocean_portal - ``` - -9. Add shortcuts - ``` - vim ~/.bashrc - #---------- - alias dj="/usr/local/apps/env/bin/python3 /usr/local/apps/ocean_portal/marco/manage.py" - - alias djrun="dj runserver 0:8000" - #---------- - ``` - -10. Exit ssh session and re-ssh in - ``` - crtl+d - vagrant ssh - ``` - -11. Set up Django - ``` - dj makemigrations - dj migrate - dj compress - dj collectstatic - dj loaddata /usr/local/apps/ocean_portal/marco/marco_site/fixtures/content.json - djrun - ``` - -12. Open http://localhost:8000 in your browser - -13. Create super user - ``` - dj createsuperuser - ``` - -14. Open http://localhost:8000/django-admin and http://localhost:8000/admin in your browser to administer the site - - - - - -Notes - -Forked Marco-portal-2 from MidAtlanticPortal GitHub account to Ecotrust: -https://github.com/Ecotrust/madrona-portal - -Cloned it locally, updated the vagrant file to give me an Ubuntu 18.04 LTS box - -vagrant up -vagrant ssh - -sudo apt-get update -sudo apt-get upgrade - -The Compass install docs were a nice guideline for setting up MP, perhaps they will be useful for the portal as well…. -https://github.com/Ecotrust/COMPASS/wiki/install - -## Upgrading Wagtail -Upgrading to Wagtail 2.0+: https://wagtail.io/blog/upgrading-to-wagtail-2/ - -## Functional Testing -Functional testing is done with Selenium and the Chrome WebDriver. The tests are located in the `functional_tests.py` file. To run the tests, you will need to have Selenium, Google Chrome, and the Chrome WebDriver installed and available in your PATH. Here is how: - -1. Install Selenium - ``` - pip install selenium - ``` -2. Install Google Chrome - ``` - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt install ./google-chrome-stable_current_amd64.deb - ``` -3. Install Chrome WebDriver -You will need to know the current stable version of Chrome you installed in step 2. You can go to [https://chromedriver.com/download](https://chromedriver.com/download) to find the url for the latest stable version of the Chrome WebDriver. Then use the URL in the following command: - ``` - wget - ``` -unzip and install the driver - ``` - unzip chromedriver_linux64.zip - sudo mv chromedriver-linux64/chromedriver /usr/local/apps/env/bin/ - ``` - -4. Run Tests -You can now run functional tests just like you would other Django tests. -**Note:** *You will need to have the server running to run the functional tests.* - ``` - python manage.py test - ``` - - - - - - ## Production Installation (Ubuntu 18.04 LTS) - #### set up new server - ``` - sudo apt update - sudo apt upgrade -y - sudo apt install git python3 python3-dev python3-virtualenv python3-pip postgresql postgresql-contrib postgis postgresql-server-dev-12 libjpeg-dev gdal-bin python-gdal python3-gdal libgdal-dev redis -y - sudo mkdir /usr/local/apps - ``` - change ownership of /usr/local/apps to be your primary sudo user: - `sudo chown {USERNAME} /usr/local/apps` - - ``` - cd /usr/local/apps/ - git clone https://github.com/Ecotrust/madrona-portal.git - mv madrona-portal ocean_portal - cd ocean_portal - git checkout wcoa - git pull - ``` - - #### Set up virtualenv - ``` - python3 -m pip install --user virtualenv - cd /usr/local/apps/ocean_portal/ - python3 -m virtualenv env - source /usr/local/apps/ocean_portal/env/bin/activate - pip install -r /usr/local/apps/ocean_portal/requirements.txt - pip install -e git+https://github.com/Ecotrust/wcoa.git@master#egg=wcoa-master - ``` - -#### Install PyGDAL - ``` - pip uninstall numpy - gdal-config --version - ``` - Note what version is printed. You will want to intall the correct pygdal for your system's GDAL. - - For example, if the printed version is '2.2.3', then you will want the latest pyGDAL in the 2.2.3 family: - `pip install "pygdal<2.2.4"` - - You should see a new version of numpy installed as well. - -#### Install database - Create a Database user. Come up with a meaningful username and a secure password. You will use the username in place of `{DBUSER}` below and you will be prompted to create the new user's password immediately. - ``` - sudo -u postgres createuser -s -P {DBUSER} - sudo -u postgres createdb -O {DBUSER} ocean_portal - sudo -u postgres psql -c "CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology;" ocean_portal - sudo vim /etc/postgresql/10/main/pg_hba.conf - ``` - Add the following line to the bottom of the pg_hba.conf file, replacing `{DBUSER}` with the username you created: - ``` - local ocean_portal {DBUSER} md5 - ``` - Finally, restart postgres so your updated configuration can be implemented. - ``` - sudo service postgresql restart - ``` - -#### Configure project - ``` - cd /usr/local/apps/ocean_portal/marco - mkdir media - mkdir static - cp config.ini.template config.ini - vim config.ini - ``` - -#### Edit config.ini - - Add the following lines under `[App]`: - ``` - PROJECT_APP = wcoa - PROJECT_SETTINGS_FILE = True - MEDIA_ROOT = /usr/local/apps/ocean_portal/env/src/wcoa-master/media - STATIC_ROOT = /usr/local/apps/ocean_portal/marco/static - EMAIL_SUBJECT_PREFIX = [WCOA] - ``` - - If you already know your URL, you can put that in for `ALLOWED_HOSTS` - - Add the following under [DATABASE] (replacing `{DBUSER}` and `{DBPASSWORD}` with the database user and password you created above): - ``` - NAME = ocean_portal - USER = {DBUSER} - PASSWORD = {DBPASSWORD} - ``` - -#### Add Django shortcuts - ``` - vim ~/.bashrc - #---------- - alias dj="/usr/local/apps/ocean_portal/env/bin/python3 /usr/local/apps/ocean_portal/marco/manage.py" - #---------- - ``` - Exit your terminal session and re-SSH in to the server to load your updates - -#### Django Initialization - ``` - dj migrate - dj compress - dj collectstatic - ``` - -#### Load in initial data - There is no prescribed method for this. If you have access to existing servers, you have the following two options. - If you don't have access to existing servers, you'll need to just try to build from scratch. - ##### With pg_dump - You can use pg_dump to generate a .sql file representing the database. You can use scp to copy that onto your new server then do the following: - ``` - sudo -u postgres dropdb ocean_portal - sudo -u postgres createdb -O {DBUSER} ocean_portal - sudo -u postgres psql ocean_portal < {YOUR_DUMP_FILE} - dj migrate - ``` - ##### With fixtures - * on the source (old) server - ``` - dj dumpdata --indent=2 {your app_models} > /usr/local/apps/ocean_portal/marco/marco/fixtures/initial_data.json - ``` - * Use scp to copy that to your new server - * on the target (new) server - ``` - dj loaddata /usr/local/apps/ocean_portal/marco/marco/fixtures/initial_data.json - ``` - -#### Configure and enable webapplication server stack: Nginx + uWSGI -For reference [go here](http://uwsgi-docs.readthedocs.org/en/latest/tutorials/Django_and_nginx.html) - -From inside your virualenv: -* `sudo apt-get install nginx uwsgi uwsgi-plugin-python3 -y` -* `pip install uwsgi` -* `sudo cp /usr/local/apps/ocean_portal/env/src/wcoa-master/deploy/nginx_config /etc/nginx/sites-available/wcoa` -* `sudo rm /etc/nginx/sites-enabled/default` -* `sudo ln -s /etc/nginx/sites-available/wcoa /etc/nginx/sites-enabled/wcoa` -* `sudo cp /usr/local/apps/ocean_portal/env/src/wcoa-master/deploy/emperor.ini /etc/uwsgi/` -* `sudo cp /usr/local/apps/ocean_portal/env/src/wcoa-master/deploy/uwsgi.service /etc/systemd/system/` -* `sudo systemctl enable uwsgi.service` -* `sudo cp /usr/local/apps/ocean_portal/env/src/wcoa-master/deploy/wcoa.ini /etc/uwsgi/apps-enabled/wcoa.ini` -* `sudo service nginx start` - * If this fails, apache2 may already be running and hogging port 80. - * you can stop apache2 with `sudo service apache2 stop` - but it will restart on reboot. - * prevent it from launching on reboot with `sudo update-rc.d apache2 disable` OR - * update your apache2 configuration to run on another port -* `sudo service nginx restart` -* `sudo reboot` -* In a few minutes, test your URL in a browser to see that everything came up as expected - -#### Install munin -`sudo apt install munin munin-node -y` - -#### Configure unattended upgrades -* `sudo apt install unattended-upgrades` -* `sudo vim /etc/apt/apt.conf.d/50unattended-upgrades` - * Uncomment the "...-updates" line - * Uncomment and configure: - * Mail - * MailOnlyOnError - * Remove-Unused-Kernel-Packages "true"; - * Remove-Unused-Dependencies "true"; - * Automatic-Reboot "true"; - * Automatic-Reboot-Time "8:00"; - * The above assumes a UTC server with assumed 1 or 2 AM Pacific time downtime -* `sudo vim /etc/apt/apt.conf.d/20auto-upgrades` +Edit `docker/.env` and set at minimum: +- `SECRET_KEY` +- `DB_PASSWORD` +- `REDIS_PASSWORD` +- `DJANGO_SUPERUSER_PASSWORD` (recommended for first init) + +*A full `.env` example can be found in 1Password.* + +Add directory named wars: + +```bash +mkdir wars +``` + +Find the WAR files for Geoportal and Harvester in 1Password and then copy them into the `wars` directory: + +```text +└── wcoa/ + └── wars/ + └── geoportal.war + └── harvester.war +``` + +Copy media files into `docker/` from a backup: +*backup is available in 1Password* + +```text +└── wcoa/ + └── docker/ + └── media/ + └── documents/ + └── group_images/ + └── images/ + └── original_images/ +``` + +Build the core base image (first time or after core dependency changes): + +```bash +task base +``` + +Build the WCOA overlay image: + +```bash +task build +``` + +First boot with migrations, fixtures, and optional superuser creation: + +```bash +task init +``` + +Open the portal: +- App: http://localhost:8000 +- Geoportal: http://localhost:8080 +- Elasticsearch: http://localhost:9200 + +*After the first boot, use:* + +```bash +task up ``` -APT::Periodic::Update-Package-Lists "1"; -APT::Periodic::Download-Upgradeable-Packages "1"; -APT::Periodic::AutocleanInterval "7"; -APT::Periodic::Unattended-Upgrade "1"; + +Import database into the portal from a dump file: +*dump file is available in 1Password* + +```bash +./scripts/db-restore.sh ``` -* `sudo unattended-upgrades --dry-run --debug` - -#### Install Certbox and configure SSL Certs -Requirements: - * URL for your site with DNS configured - * external access to port 80 - * external access to port 443 - * access to a preferred email address to receive any alerts about your SSL certificates - -If you have not already done so, edit /usr/local/apps/ocean_portal/marco/config.ini in the [APP] section: - * `ALLOWED_HOSTS = {SITE_URL}` where `{SITE_URL}` is your site's intended address -Then restart uWSGI: `sudo service uwsgi restart` - -Install certbot: -* `sudo apt install software-properties-common -y` -* `sudo add-apt-repository ppa:certbot/certbot` - * press Enter to continue -* `sudo apt update` -* `sudo apt install python-certbot-nginx -y` - -Configure NGINX: + +--- + +## Day-to-day commands + +```bash +task up # Start stack +task down # Stop stack +task logs # Tail app logs +task shell # Django shell +task manage -- migrate +task manage -- createsuperuser ``` -sudo cp /etc/nginx/sites-available/wcoa /etc/nginx/sites-available/wcoa.http_only -sudo vim /etc/nginx/sites-available/wcoa + +## Docker structure in this repo + +- `docker/Dockerfile` builds the WCOA overlay from `ghcr.io/ecotrust/madrona-portal` +- `docker/compose.yml` defines WCOA app plus `geoportal` and `elastic` +- `docker/config.wcoa.docker.ini` holds portal-level non-secret config +- `docker/.env` holds local secrets and environment-specific values + +The app service uses: +- WCOA compose overlay (`docker/compose.yml`) +- Core compose base from `madrona-portal/docker/compose.base.yml` + +The `Taskfile.yml` intentionally composes with the WCOA file first and core base second. +Keep that order to avoid path-resolution issues with bind mounts. + +## Configuration notes + +- `MP_PROJECT_CONFIG` points to `docker/config.wcoa.docker.ini` in-container +- Secrets should come from environment variables in `docker/.env` +- `docker/.env` is gitignored +- `DB_INIT=1` should be used only when intentionally initializing data + +## Running the production-oriented compose file locally + +This repo also includes `docker/compose.prod.yml` for image-based runs. + +Example: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env up -d ``` -* Replace the line `server_name _;` with `server_name {SITE_URL}` where `{SITE_URL}` is your site's URL address -* Save -* Test your NGINX configuration: `sudo nginx -t` -* Restart NGINX: `sudo service nginx restart` -* Test your website out in a browser to be sure your DNS is resolving correctly. -Get your SSL Certificate, replace `{SITE_URL}` with your URL address: +Optional dev profile services in that file: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env --profile dev up -d ``` -sudo certbot --nginx -d {SITE_URL} + +## Troubleshooting + +- If app startup fails, inspect logs: + +```bash +task logs ``` -* provide your email address -* agree to their terms: https://letsencrypt.org/documents/LE-SA-v1.2-November-15-2017.pdf -* choose whether or not to have your email address shared with eff.org -* select if you want users hitting the site using HTTP to be automatically redirected to HTTPS -Test your SSL Cert installation: -* hit your site using HTTPS (or HTTP if you chose to have automatic redirection) -* Inspect your URL using https://www.ssllabs.com/ssltest/ -* Check that Certbot auto-renew is properly configured: - * `sudo certbot renew --dry-run` +- If Django commands fail due to schema state, re-run init once: + +```bash +task down +task init +``` +- If you changed only WCOA code/templates, `task up` is usually enough. +- If you changed Python dependencies in `docker/requirements.txt`, run `task build` again. +- If you changed shared core dependencies, rebuild base with `task base` and then `task build`. -#### Set up external uptime monitoring -recommended: https://uptimerobot.com +## Legacy notes -#### Restart your server and test +Older Vagrant and manual server setup steps have been removed from this README. +Use Docker-based workflows described above and onboarding references for current development. \ No newline at end of file From 5af33ede5304f94cca8db0784b01c50ee1c9c393 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 30 Jul 2026 16:06:28 -0700 Subject: [PATCH 18/38] Add support for multi-platform builds in GitHub Actions and update README --- .github/workflows/build-and-publish-image.yml | 2 ++ README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 974ef89..5beb98e 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -8,6 +8,7 @@ on: env: BASE_TAG: bc0fe1d IMAGE_NAME: ghcr.io/ecotrust/wcoa + IMAGE_PLATFORMS: linux/amd64,linux/arm64 jobs: build-and-push: @@ -40,6 +41,7 @@ jobs: context: . file: docker/Dockerfile push: true + platforms: ${{ env.IMAGE_PLATFORMS }} build-args: | BASE_TAG=${{ env.BASE_TAG }} tags: | diff --git a/README.md b/README.md index 860e1f3..64bcf9e 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,8 @@ Keep that order to avoid path-resolution issues with bind mounts. ## Running the production-oriented compose file locally This repo also includes `docker/compose.prod.yml` for image-based runs. +Published image tags include both `linux/amd64` and `linux/arm64`, so Docker will +pull the correct architecture automatically on Intel and Apple Silicon hosts. Example: From 82d0018a1177a01be331ec5f5d02472876932d81 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 3 Aug 2026 13:59:34 -0700 Subject: [PATCH 19/38] Add newline for better readability in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 64bcf9e..5ee3fe1 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ Example: ```bash docker compose -f docker/compose.prod.yml --env-file docker/.env up -d + ``` Optional dev profile services in that file: From a71c7a876fea9b2a935fbed2c4b23833c286cb80 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 3 Aug 2026 14:40:30 -0700 Subject: [PATCH 20/38] Remove IMAGE_PLATFORMS variable from build-and-publish-image.yml and clean up README.md formatting --- .github/workflows/build-and-publish-image.yml | 1 - README.md | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 5beb98e..b32c4de 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -8,7 +8,6 @@ on: env: BASE_TAG: bc0fe1d IMAGE_NAME: ghcr.io/ecotrust/wcoa - IMAGE_PLATFORMS: linux/amd64,linux/arm64 jobs: build-and-push: diff --git a/README.md b/README.md index 5ee3fe1..64bcf9e 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,6 @@ Example: ```bash docker compose -f docker/compose.prod.yml --env-file docker/.env up -d - ``` Optional dev profile services in that file: From 813c7e09989e43b097ca79b85a7c9c98be505ec7 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Mon, 3 Aug 2026 15:39:48 -0700 Subject: [PATCH 21/38] adding default nginx config file for testing production deploy on workstation --- docker/nginx-dev.conf | 141 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 docker/nginx-dev.conf diff --git a/docker/nginx-dev.conf b/docker/nginx-dev.conf new file mode 100644 index 0000000..981d789 --- /dev/null +++ b/docker/nginx-dev.conf @@ -0,0 +1,141 @@ +server { + listen 80; + server_name _; + + # Use Docker's internal DNS so upstream hostnames are resolved at + # request time, not at nginx startup (avoids "host not found" errors + # when a backend container hasn't started yet). + resolver 127.0.0.11 valid=30s; + # TODO: consider writing logs to specific file + # access_log /var/log/nginx/app.access.log; + # error_log /var/log/nginx/app.error.log; + + # Increase client body size for file uploads + client_max_body_size 100M; + + # For reference: SCCWRP server also hosted a WAF - we may need to restore this on AWS. + # location /geospatial/ { + # alias /var/www/html/geospatial/; + # autoindex on; + # } + + # For reference only: add Munin and its static files + # location /munin/static/ { + # alias /etc/munin/static/; + # } + + # location /munin { + # alias /var/cache/munin/www; + # } + + location /static { + # Static files served from Docker volume + alias /vol/web/static/; + # prevent caching + add_header Last-Modified $date_gmt; + add_header Cache-Control 'no-store, no-cache, must-revalidate'; + add_header Pragma 'no-cache'; + add_header Expires 0; + if_modified_since off; + expires off; + etag off; + } + + location /media { + # Media files served from Docker volume + alias /usr/local/apps/madrona-portal/media/; + } + + location /favicon.ico { + # Favicon served from static volume + alias /vol/web/favicon.ico; + } + + # Elasticsearch routing - route search/doc/metadata requests to Elasticsearch + location ~ ^(/_search/|/_doc/|/metadata).*$ { + set $elastic_backend http://elastic:9200; + proxy_pass $elastic_backend; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # ENABLE CORS + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + # Custom headers and headers various browsers *should* be OK with but are not + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + if ($request_method = 'POST') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + } + + # Geoportal management interfaces routing + location ~ ^/(manager|host-manager|semantix|solr|gc|geoportal|harvester).*$ { + set $geoportal_backend http://geoportal:8080; + proxy_pass $geoportal_backend; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # ENABLE CORS + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + # Custom headers and headers various browsers *should* be OK with but are not + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + if ($request_method = 'POST') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + } + + # Default routing - everything else goes to Django app + location / { + set $app_backend http://app:8008; + proxy_pass $app_backend; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + From 38d224520f04a6ed5ed241ab145776d01cfbca78 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 3 Aug 2026 15:52:55 -0700 Subject: [PATCH 22/38] Add TODOs for production compose instructions and media file permissions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 64bcf9e..3e37411 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,10 @@ Optional dev profile services in that file: docker compose -f docker/compose.prod.yml --env-file docker/.env --profile dev up -d ``` +TODO: Add instructions for running production compose file in a cloud environment. +TODO: Media files dir might permissions changes. + + ## Troubleshooting - If app startup fails, inspect logs: From 983ab630dbd384b271efcfcefc1ced6b9ea253f3 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 3 Aug 2026 15:55:58 -0700 Subject: [PATCH 23/38] Update BASE_TAG to latest in build-and-publish-image.yml --- .github/workflows/build-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index b32c4de..a1dd779 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: env: - BASE_TAG: bc0fe1d + BASE_TAG: latest IMAGE_NAME: ghcr.io/ecotrust/wcoa jobs: From c56d0c162f09b8c8ffbc0ded9fef0d58b450f53b Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 14:17:39 -0700 Subject: [PATCH 24/38] Add production configuration file for WCOA deployments --- docker/config.wcoa.prod.ini | 74 +++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 docker/config.wcoa.prod.ini diff --git a/docker/config.wcoa.prod.ini b/docker/config.wcoa.prod.ini new file mode 100644 index 0000000..eeff094 --- /dev/null +++ b/docker/config.wcoa.prod.ini @@ -0,0 +1,74 @@ +# Production-focused configuration for WCOA deployments. +# Use with MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.prod.ini + +[APP] +APP_NAME = WCOA Portal +APP_URL = https://portal.westcoastoceans.org +APP_TEAM_NAME = Marine Planner Team +PROJECT_APP = wcoa +PROJECT_SETTINGS_FILE = True +DEBUG = False +TEMPLATE_DEBUG = False +ALLOWED_HOSTS = ["portal.westcoastoceans.org", "www.westcoastoceans.org"] +# SECRET_KEY is loaded from environment variable: SECRET_KEY +MEDIA_ROOT = /usr/local/apps/madrona-portal/media +MEDIA_URL = /media/ +TIME_ZONE = UTC +GA_ACCOUNT = +# ReCAPTCHA keys are loaded from env vars: RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY +STATIC_ROOT = /vol/web/static +EMAIL_SUBJECT_PREFIX = [WCOA] +MAP_LIBRARY = ol8 +COMPRESS_ENABLED = True +STATIC_CORE = /vol/web/static/ +ADDITIONAL_APPS = [] +ADDITIONAL_MIDDLEWARE = [] + +[REGION] +NAME = West Coast Ocean +INIT_ZOOM = 6 +INIT_LAT = 39 +INIT_LON = -120 +MAP = ocean + +[CACHES] +BACKEND = django_redis.cache.RedisCache +LOCATION = redis://tasks:6379/1 +CLIENT_CLASS = django_redis.client.DefaultClient + +[CELERY] +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +CELERY_ALWAYS_EAGER = False +CELERY_DISABLE_RATE_LIMITS = True + +[DATABASE] +ENGINE = django.contrib.gis.db.backends.postgis +NAME = wcoa_docker_db +HOST = db +PORT = 5432 +USER = postgres +# DB password is loaded from environment variable: DB_PASSWORD + +[EMAIL] +HOST = localhost +PORT = 25 +# SMTP credentials are loaded from env vars: EMAIL_HOST_USER, EMAIL_HOST_PASSWORD +DEFAULT_FROM_EMAIL = WCOA Portal +SERVER_EMAIL = WCOA Site Errors + +[AWS] +# AWS credentials are loaded from env vars: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +AWS_SES_REGION_NAME = us-east-1 +AWS_SES_REGION_ENDPOINT = email.us-east-1.amazonaws.com + +[SOCIAL_AUTH] +# Social OAuth credentials are loaded from env vars: +# FACEBOOK_KEY, FACEBOOK_SECRET, TWITTER_KEY, TWITTER_SECRET, GOOGLE_KEY, GOOGLE_SECRET + +[CATALOG] +DATA_CATALOG_ENABLED = True +CATALOG_TECHNOLOGY = GeoPortal2 +CATALOG_PROXY = +CATALOG_SOURCE = http://elastic:9200 +CATALOG_QUERY_ENDPOINT = /geoportal/elastic/metadata/item/_search/ From 2cc51d1f9b3ac7163dafbb062f28d224d4141496 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 14:19:24 -0700 Subject: [PATCH 25/38] Update .gitignore to include elasticsearch backups and add .gitkeep file --- .gitignore | 6 ++++-- docker/backups/elasticsearch/.gitkeep | 0 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 docker/backups/elasticsearch/.gitkeep diff --git a/.gitignore b/.gitignore index eb614f4..2bbf016 100644 --- a/.gitignore +++ b/.gitignore @@ -10,12 +10,14 @@ wars/ tomcat-users.xml config.wcoa.ini -config.wcoa.docker.ini env/ marco/static/ # Docker -docker/backups/ +docker/backups/* +!docker/backups/elasticsearch/ +docker/backups/elasticsearch/* +!docker/backups/elasticsearch/.gitkeep docker/media/ docker/static/ docker/entrypoint.sh diff --git a/docker/backups/elasticsearch/.gitkeep b/docker/backups/elasticsearch/.gitkeep new file mode 100644 index 0000000..e69de29 From baf6c7b2a7538560fe45ca746a6af80da6473cc3 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 14:20:10 -0700 Subject: [PATCH 26/38] Update README and production compose files for Docker decoupling rollout - Clarify WAR directory structure in README - Update production compose file to use the correct config file and port mapping - Add detailed runbook for WCOA production rollout process --- README.md | 16 +- docker/compose.prod.yml | 6 +- docs/production-rollout-docker-decoupling.md | 178 +++++++++++++++++++ 3 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 docs/production-rollout-docker-decoupling.md diff --git a/README.md b/README.md index 3e37411..6036cec 100644 --- a/README.md +++ b/README.md @@ -67,17 +67,18 @@ Edit `docker/.env` and set at minimum: *A full `.env` example can be found in 1Password.* -Add directory named wars: +Add the WAR directory under `docker/`: ```bash -mkdir wars +mkdir -p docker/wars ``` -Find the WAR files for Geoportal and Harvester in 1Password and then copy them into the `wars` directory: +Find the WAR files for Geoportal and Harvester in 1Password and then copy them into `docker/wars`: ```text └── wcoa/ - └── wars/ + └── docker/ + └── wars/ └── geoportal.war └── harvester.war ``` @@ -148,7 +149,8 @@ task manage -- createsuperuser - `docker/Dockerfile` builds the WCOA overlay from `ghcr.io/ecotrust/madrona-portal` - `docker/compose.yml` defines WCOA app plus `geoportal` and `elastic` -- `docker/config.wcoa.docker.ini` holds portal-level non-secret config +- `docker/config.wcoa.docker.ini` holds local-development portal config +- `docker/config.wcoa.prod.ini` holds production-focused portal config - `docker/.env` holds local secrets and environment-specific values The app service uses: @@ -165,12 +167,16 @@ Keep that order to avoid path-resolution issues with bind mounts. - `docker/.env` is gitignored - `DB_INIT=1` should be used only when intentionally initializing data +For image-based production runs via `docker/compose.prod.yml`, `MP_PROJECT_CONFIG` is set to `docker/config.wcoa.prod.ini`. + ## Running the production-oriented compose file locally This repo also includes `docker/compose.prod.yml` for image-based runs. Published image tags include both `linux/amd64` and `linux/arm64`, so Docker will pull the correct architecture automatically on Intel and Apple Silicon hosts. +In production compose, Gunicorn listens on container port `8008`. `APP_PORT` controls only the host-side port mapping (`${APP_PORT}:8008`). + Example: ```bash diff --git a/docker/compose.prod.yml b/docker/compose.prod.yml index 87111bb..1d78363 100644 --- a/docker/compose.prod.yml +++ b/docker/compose.prod.yml @@ -4,12 +4,12 @@ services: volumes: - ./static:/vol/web/static - ./media:/usr/local/apps/madrona-portal/media - - ./config.wcoa.docker.ini:/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.docker.ini:ro + - ./config.wcoa.prod.ini:/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.prod.ini:ro env_file: - ./.env environment: - DB_INIT=${DB_INIT:-0} - - MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.docker.ini + - MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/wcoa/docker/config.wcoa.prod.ini - SECRET_KEY=${SECRET_KEY:?SECRET_KEY must be set} - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} - DEBUG=${DEBUG:-False} @@ -32,7 +32,7 @@ services: tasks: condition: service_healthy ports: - - "${APP_PORT:-8008}:8008" + - "${APP_PORT:-8000}:8008" networks: - madronanetwork restart: unless-stopped diff --git a/docs/production-rollout-docker-decoupling.md b/docs/production-rollout-docker-decoupling.md new file mode 100644 index 0000000..8930009 --- /dev/null +++ b/docs/production-rollout-docker-decoupling.md @@ -0,0 +1,178 @@ +# WCOA production rollout: Docker decoupling + +This runbook covers the production rollout for the WCOA Docker decoupling work. It assumes the application is deployed from the WCOA repo using the image-based compose file in [docker/compose.prod.yml](../docker/compose.prod.yml). + +## Goal + +Ship the decoupled WCOA deployment so the portal runs from its own image and config while still using the shared Madrona base image. + +## Recommended rollout order + +1. Confirm the shared base image is ready. +2. Build and publish the WCOA overlay image. +3. Deploy the new image to production. +4. Verify the app, DB, and supporting services. +5. Roll back quickly if anything fails. + +--- + +## 1. Prepare the release + +### Confirm the base image + +The WCOA overlay image depends on the shared Madrona base image published from the core repo. + +Before rollout: + +- Confirm the core image build completed successfully. +- Note the base image tag you want to use, preferably a pinned SHA or release tag rather than `latest`. +- Confirm the WCOA image build will use that base tag in the Docker build arguments. + +### Confirm release content + +Verify the following are ready: + +- The WCOA repo branch or PR is merged or otherwise approved for production. +- The image workflow completed successfully. +- The production environment has the required secrets in its `.env` file. +- Any required WAR files and media assets are present in the deployment host. + +--- + +## 2. Build and publish the WCOA image + +The WCOA workflow publishes images to GHCR from [/.github/workflows/build-and-publish-image.yml](../.github/workflows/build-and-publish-image.yml). + +### What to expect + +The workflow publishes: + +- `ghcr.io/ecotrust/wcoa:` +- `ghcr.io/ecotrust/wcoa:latest` + +### Production recommendation + +For production, prefer deploying a pinned SHA rather than `latest`. + +Example: + +```bash +# Example only: use the SHA from the successful workflow run +export IMAGE_TAG= +``` + +--- + +## 3. Prepare the production host + +On the deployment host, make sure the production environment is ready. + +### Required files + +Ensure the host has: + +- The production `.env` file with secrets and runtime settings +- The compose file at [docker/compose.prod.yml](../docker/compose.prod.yml) +- The production WCOA config file at `docker/config.wcoa.prod.ini` mounted by the container +- Static and media directories with correct permissions + +Before first startup, create the Elasticsearch snapshot path expected by compose: + +```bash +mkdir -p docker/backups/elasticsearch +chown -R "$(id -u)":"$(id -g)" docker/backups/elasticsearch +``` + +### Minimum environment values + +At minimum confirm these values are present in the host `.env`: + +```env +SECRET_KEY=... +DB_PASSWORD=... +REDIS_PASSWORD=... +DB_NAME=wcoa_docker_db +IMAGE_TAG= +DJANGO_SUPERUSER_PASSWORD=... +``` + +If this is the first production deployment after the decoupled rollout, set `DB_INIT=1` once for initial boot and data setup. After that, switch it back to `0`. + +--- + +## 4. Deploy the new image + +From the WCOA deployment directory, pull and recreate the stack. + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env pull +docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +``` + +If this is the first boot or the database needs initialization: + +```bash +DB_INIT=1 docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +``` + +After initialization, set `DB_INIT=0` in the environment and restart the app service if needed. + +--- + +## 5. Verify the deployment + +### Container health + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env ps +docker compose -f docker/compose.prod.yml --env-file docker/.env logs -f app +``` + +### Functional checks + +Verify: + +- The app responds on the expected host port. +- The portal loads without obvious errors. +- The database connection is healthy. +- Static and media files are served correctly. +- Geoportal and Elasticsearch are healthy if those services are part of the deployment. + +If you have a smoke-test endpoint or a browser-based check, use it at this stage. + +--- + +## 6. Roll back if needed + +If the deployment shows problems, revert to the previous known-good image tag. + +```bash +export IMAGE_TAG= +docker compose -f docker/compose.prod.yml --env-file docker/.env pull +docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +``` + +Keep the previous image tag recorded so rollback is fast. + +--- + +## 7. Post-deploy notes + +After the rollout is confirmed: + +- Record the deployed image tag in the deployment notes. +- Capture the date, release SHA, and any config changes. +- Keep the previous image tag available for rollback until the next deployment is stable. +- If the rollout included a new database migration or fixture load, verify the data shape before removing the temporary initialization step. + +--- + +## Quick checklist + +- [ ] Core base image build succeeded +- [ ] WCOA image workflow succeeded +- [ ] Production `.env` is ready +- [ ] Image tag is pinned for production +- [ ] Containers pulled and recreated +- [ ] App, DB, and supporting services are healthy +- [ ] Rollback target is recorded From 8ea26d930afc71f2acbed209430863d14b7b3f4b Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 14:28:31 -0700 Subject: [PATCH 27/38] Add IMAGE_PLATFORMS environment variable for multi-platform builds --- .github/workflows/build-and-publish-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index a1dd779..9f28f68 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -8,6 +8,7 @@ on: env: BASE_TAG: latest IMAGE_NAME: ghcr.io/ecotrust/wcoa + IMAGE_PLATFORMS: linux/amd64,linux/arm64 jobs: build-and-push: From bc520349f97120b88615940bda70446563feeb36 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 14:28:37 -0700 Subject: [PATCH 28/38] Add script to create Elasticsearch snapshots with configurable repository and URL --- scripts/create_elastic_snapshot.sh | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 scripts/create_elastic_snapshot.sh diff --git a/scripts/create_elastic_snapshot.sh b/scripts/create_elastic_snapshot.sh new file mode 100755 index 0000000..f67a608 --- /dev/null +++ b/scripts/create_elastic_snapshot.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +REPOSITORY="" +ELASTIC_URL="${ELASTIC_URL:-http://localhost:9200}" + +usage() { + cat < [-u ] + +Options: + -r Elasticsearch snapshot repository name (required) + -u Elasticsearch base URL (default: http://localhost:9200) + -h Show this help +EOF +} + +while getopts ":r:u:h" opt; do + case "$opt" in + r) REPOSITORY="$OPTARG" ;; + u) ELASTIC_URL="$OPTARG" ;; + h) + usage + exit 0 + ;; + :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; + \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; + esac +done + +if [[ -z "$REPOSITORY" ]]; then + echo "Error: -r is required" >&2 + exit 1 +fi + +DATETIME_VAR="$(date +%Y%m%d_%H%M)" +SNAPSHOT_NAME="snapshot_${DATETIME_VAR}" + +curl --fail --silent --show-error \ + -X PUT "${ELASTIC_URL}/_snapshot/${REPOSITORY}/${SNAPSHOT_NAME}" \ + -H 'Content-Type: application/json' \ + -d '{"indices": "metadata_v1", "ignore_unavailable": true, "include_global_state": false}' + +echo "Created Elasticsearch snapshot ${SNAPSHOT_NAME} in repository ${REPOSITORY}" From 63d44cb8f3511ab4f92aa9ba3bc0bbd9c0c03e0d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 14:28:43 -0700 Subject: [PATCH 29/38] Add db_dump script for automated database backups with Docker --- scripts/db_dump.sh | 100 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100755 scripts/db_dump.sh diff --git a/scripts/db_dump.sh b/scripts/db_dump.sh new file mode 100755 index 0000000..8d902ed --- /dev/null +++ b/scripts/db_dump.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT_DIR="$(cd "$DIR/.." >/dev/null 2>&1 && pwd)" + +COMPOSE_FILE="$ROOT_DIR/docker/compose.prod.yml" +ENV_FILE="$ROOT_DIR/docker/.env" +SERVICE_NAME="db" + +DBNAME="" +DBOWNER="" +DBPASSWORD="" +OUTDIR="$ROOT_DIR/docker/backups/sql" +OUTFILE="" + +usage() { + cat < Database name (default: DB_NAME from env file) + -o Database user (default: DB_USER from env file) + -p Database password (default: DB_PASSWORD from env file) + -d Output directory (default: docker/backups/sql) + -f Output filename (default: _dump_.sql) + -c Docker compose file path + -e Environment file path + -s Docker service name (default: db) + -h Show this help +EOF +} + +while getopts ":n:o:p:d:f:c:e:s:h" flag; do + case "$flag" in + n) DBNAME="$OPTARG" ;; + o) DBOWNER="$OPTARG" ;; + p) DBPASSWORD="$OPTARG" ;; + d) OUTDIR="$OPTARG" ;; + f) OUTFILE="$OPTARG" ;; + c) COMPOSE_FILE="$OPTARG" ;; + e) ENV_FILE="$OPTARG" ;; + s) SERVICE_NAME="$OPTARG" ;; + h) + usage + exit 0 + ;; + :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; + \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; + esac +done + +if [[ ! -f "$COMPOSE_FILE" ]]; then + echo "Error: Docker compose file not found at $COMPOSE_FILE" >&2 + exit 1 +fi + +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +DBNAME="${DBNAME:-${DB_NAME:-}}" +DBOWNER="${DBOWNER:-${DB_USER:-}}" +DBPASSWORD="${DBPASSWORD:-${DB_PASSWORD:-}}" + +if [[ -z "$DBNAME" || -z "$DBOWNER" || -z "$DBPASSWORD" ]]; then + echo "Error: DB credentials are incomplete. Provide -n/-o/-p or set DB_NAME/DB_USER/DB_PASSWORD in env file." >&2 + exit 1 +fi + +mkdir -p "$OUTDIR" + +if [[ -z "$OUTFILE" ]]; then + OUTFILE="${DBNAME}_dump_$(date +%F_%H-%M-%S).sql" +fi + +OUTPATH="$OUTDIR/$OUTFILE" + +compose_cmd=(docker compose -f "$COMPOSE_FILE") +if [[ -f "$ENV_FILE" ]]; then + compose_cmd+=(--env-file "$ENV_FILE") +fi + +CONTAINER_ID="$(${compose_cmd[@]} ps -q "$SERVICE_NAME")" + +if [[ -z "$CONTAINER_ID" ]]; then + echo "Error: Service '$SERVICE_NAME' is not running." >&2 + exit 1 +fi + +${compose_cmd[@]} exec -T \ + -e PGPASSWORD="$DBPASSWORD" \ + "$SERVICE_NAME" \ + pg_dump -b -c -n public -O --quote-all-identifiers --no-acl -w -U "$DBOWNER" -d "$DBNAME" > "$OUTPATH" + +echo "Database dump created: $OUTPATH" From 8bfdc4a563839998bbba9ccb622d5a75ccce4b2f Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 15:08:33 -0700 Subject: [PATCH 30/38] Reindent db_dump and create_elastic_snapshot scripts for consistency --- scripts/create_elastic_snapshot.sh | 38 +++++++------- scripts/db_dump.sh | 82 +++++++++++++++--------------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/scripts/create_elastic_snapshot.sh b/scripts/create_elastic_snapshot.sh index f67a608..8b77e21 100755 --- a/scripts/create_elastic_snapshot.sh +++ b/scripts/create_elastic_snapshot.sh @@ -6,40 +6,40 @@ REPOSITORY="" ELASTIC_URL="${ELASTIC_URL:-http://localhost:9200}" usage() { - cat < [-u ] Options: - -r Elasticsearch snapshot repository name (required) - -u Elasticsearch base URL (default: http://localhost:9200) - -h Show this help + -r Elasticsearch snapshot repository name (required) + -u Elasticsearch base URL (default: http://localhost:9200) + -h Show this help EOF } while getopts ":r:u:h" opt; do - case "$opt" in - r) REPOSITORY="$OPTARG" ;; - u) ELASTIC_URL="$OPTARG" ;; - h) - usage - exit 0 - ;; - :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; - \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; - esac + case "$opt" in + r) REPOSITORY="$OPTARG" ;; + u) ELASTIC_URL="$OPTARG" ;; + h) + usage + exit 0 + ;; + :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; + \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; + esac done if [[ -z "$REPOSITORY" ]]; then - echo "Error: -r is required" >&2 - exit 1 + echo "Error: -r is required" >&2 + exit 1 fi DATETIME_VAR="$(date +%Y%m%d_%H%M)" SNAPSHOT_NAME="snapshot_${DATETIME_VAR}" curl --fail --silent --show-error \ - -X PUT "${ELASTIC_URL}/_snapshot/${REPOSITORY}/${SNAPSHOT_NAME}" \ - -H 'Content-Type: application/json' \ - -d '{"indices": "metadata_v1", "ignore_unavailable": true, "include_global_state": false}' + -X PUT "${ELASTIC_URL}/_snapshot/${REPOSITORY}/${SNAPSHOT_NAME}" \ + -H 'Content-Type: application/json' \ + -d '{"indices": "metadata_v1", "ignore_unavailable": true, "include_global_state": false}' echo "Created Elasticsearch snapshot ${SNAPSHOT_NAME} in repository ${REPOSITORY}" diff --git a/scripts/db_dump.sh b/scripts/db_dump.sh index 8d902ed..ec7ec76 100755 --- a/scripts/db_dump.sh +++ b/scripts/db_dump.sh @@ -16,51 +16,51 @@ OUTDIR="$ROOT_DIR/docker/backups/sql" OUTFILE="" usage() { - cat < Database name (default: DB_NAME from env file) - -o Database user (default: DB_USER from env file) - -p Database password (default: DB_PASSWORD from env file) - -d Output directory (default: docker/backups/sql) - -f Output filename (default: _dump_.sql) - -c Docker compose file path - -e Environment file path - -s Docker service name (default: db) - -h Show this help + -n Database name (default: DB_NAME from env file) + -o Database user (default: DB_USER from env file) + -p Database password (default: DB_PASSWORD from env file) + -d Output directory (default: docker/backups/sql) + -f Output filename (default: _dump_.sql) + -c Docker compose file path + -e Environment file path + -s Docker service name (default: db) + -h Show this help EOF } while getopts ":n:o:p:d:f:c:e:s:h" flag; do - case "$flag" in - n) DBNAME="$OPTARG" ;; - o) DBOWNER="$OPTARG" ;; - p) DBPASSWORD="$OPTARG" ;; - d) OUTDIR="$OPTARG" ;; - f) OUTFILE="$OPTARG" ;; - c) COMPOSE_FILE="$OPTARG" ;; - e) ENV_FILE="$OPTARG" ;; - s) SERVICE_NAME="$OPTARG" ;; - h) - usage - exit 0 - ;; - :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; - \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; - esac + case "$flag" in + n) DBNAME="$OPTARG" ;; + o) DBOWNER="$OPTARG" ;; + p) DBPASSWORD="$OPTARG" ;; + d) OUTDIR="$OPTARG" ;; + f) OUTFILE="$OPTARG" ;; + c) COMPOSE_FILE="$OPTARG" ;; + e) ENV_FILE="$OPTARG" ;; + s) SERVICE_NAME="$OPTARG" ;; + h) + usage + exit 0 + ;; + :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; + \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; + esac done if [[ ! -f "$COMPOSE_FILE" ]]; then - echo "Error: Docker compose file not found at $COMPOSE_FILE" >&2 - exit 1 + echo "Error: Docker compose file not found at $COMPOSE_FILE" >&2 + exit 1 fi if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a fi DBNAME="${DBNAME:-${DB_NAME:-}}" @@ -68,33 +68,33 @@ DBOWNER="${DBOWNER:-${DB_USER:-}}" DBPASSWORD="${DBPASSWORD:-${DB_PASSWORD:-}}" if [[ -z "$DBNAME" || -z "$DBOWNER" || -z "$DBPASSWORD" ]]; then - echo "Error: DB credentials are incomplete. Provide -n/-o/-p or set DB_NAME/DB_USER/DB_PASSWORD in env file." >&2 - exit 1 + echo "Error: DB credentials are incomplete. Provide -n/-o/-p or set DB_NAME/DB_USER/DB_PASSWORD in env file." >&2 + exit 1 fi mkdir -p "$OUTDIR" if [[ -z "$OUTFILE" ]]; then - OUTFILE="${DBNAME}_dump_$(date +%F_%H-%M-%S).sql" + OUTFILE="${DBNAME}_dump_$(date +%F_%H-%M-%S).sql" fi OUTPATH="$OUTDIR/$OUTFILE" compose_cmd=(docker compose -f "$COMPOSE_FILE") if [[ -f "$ENV_FILE" ]]; then - compose_cmd+=(--env-file "$ENV_FILE") + compose_cmd+=(--env-file "$ENV_FILE") fi CONTAINER_ID="$(${compose_cmd[@]} ps -q "$SERVICE_NAME")" if [[ -z "$CONTAINER_ID" ]]; then - echo "Error: Service '$SERVICE_NAME' is not running." >&2 - exit 1 + echo "Error: Service '$SERVICE_NAME' is not running." >&2 + exit 1 fi ${compose_cmd[@]} exec -T \ - -e PGPASSWORD="$DBPASSWORD" \ - "$SERVICE_NAME" \ - pg_dump -b -c -n public -O --quote-all-identifiers --no-acl -w -U "$DBOWNER" -d "$DBNAME" > "$OUTPATH" + -e PGPASSWORD="$DBPASSWORD" \ + "$SERVICE_NAME" \ + pg_dump -b -c -n public -O --quote-all-identifiers --no-acl -w -U "$DBOWNER" -d "$DBNAME" > "$OUTPATH" echo "Database dump created: $OUTPATH" From c7cb01dd42151df90c0b16612f0234bf3fcf9430 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 15:46:54 -0700 Subject: [PATCH 31/38] Add deployment runbooks for WCOA and remove deprecated rollout document --- README.md | 8 +- docs/AWS_DEPLOY_WCOA.md | 311 +++++++++++++++++++ docs/PRODUCTION_CUTOVER.md | 210 +++++++++++++ docs/production-rollout-docker-decoupling.md | 178 ----------- 4 files changed, 528 insertions(+), 179 deletions(-) create mode 100644 docs/AWS_DEPLOY_WCOA.md create mode 100644 docs/PRODUCTION_CUTOVER.md delete mode 100644 docs/production-rollout-docker-decoupling.md diff --git a/README.md b/README.md index 6036cec..fb11de9 100644 --- a/README.md +++ b/README.md @@ -215,4 +215,10 @@ task init ## Legacy notes Older Vagrant and manual server setup steps have been removed from this README. -Use Docker-based workflows described above and onboarding references for current development. \ No newline at end of file +Use Docker-based workflows described above and onboarding references for current development. + +## Deployment runbooks + +- Host baseline setup: madrona-portal `docs/AWS_DEPLOY.md` +- WCOA greenfield deploy: `docs/AWS_DEPLOY_WCOA.md` +- WCOA production cutover: `docs/PRODUCTION_CUTOVER.md` \ No newline at end of file diff --git a/docs/AWS_DEPLOY_WCOA.md b/docs/AWS_DEPLOY_WCOA.md new file mode 100644 index 0000000..ee0a2ac --- /dev/null +++ b/docs/AWS_DEPLOY_WCOA.md @@ -0,0 +1,311 @@ +# AWS Deployment Runbook - WCOA Portal + +This runbook is for deploying WCOA on a host that is already prepared using the core platform guide in [madrona-portal/docs/AWS_DEPLOY.md](../../../madrona-portal/docs/AWS_DEPLOY.md). + +Use this document for greenfield WCOA deployment and repeatable release operations. + +## 1. What WCOA adds + +Compared to the platform baseline, WCOA adds: + +- Geoportal and Harvester on Tomcat. +- Elasticsearch for metadata indexing/search. +- Geoportal WAR artifacts. +- WCOA-specific proxy routes and cron jobs. + +Sizing guidance: + +- Start at t3.large minimum for production. +- Use at least 60 GB root volume. +- Monitor memory and disk pressure during indexing jobs. + +## 2. Prerequisites to collect + +From secure credential storage, collect: + +- WCOA deployment host SSH key. +- GHCR read token for image pulls. +- WCOA production environment values for docker/.env. +- Geoportal WAR files: + - geoportal.war + - harvester.war +- Current database dump for initial load. +- Media backup archive. + +## 3. Clone repository and place artifacts + +On the host: + +```bash +mkdir -p /home/ubuntu/portals +cd /home/ubuntu/portals +git clone https://github.com/Ecotrust/wcoa.git +cd wcoa +``` + +Create required directories and place artifacts: + +```bash +mkdir -p docker/wars +mkdir -p docker/media +mkdir -p docker/backups/elasticsearch +mkdir -p docker/backups/sql +``` + +Copy WAR files into docker/wars and restore media into docker/media. + +Set directory ownership for Elasticsearch snapshots: + +```bash +chown -R "$(id -u)":"$(id -g)" docker/backups/elasticsearch +``` + +## 4. Configure WCOA + +### 4.1 Create docker environment file + +```bash +cp docker/.env.example docker/.env +``` + +Required production settings in docker/.env: + +```env +COMPOSE_PROJECT_NAME=wcoa +IMAGE_TAG= +BASE_TAG= + +APP_PORT=8000 +DB_PORT=5432 + +SECRET_KEY= +DEBUG=False +ALLOWED_HOSTS=portal.westcoastoceans.org,www.westcoastoceans.org + +DB_NAME=wcoa_docker_db +DB_USER=postgres +DB_PASSWORD= +REDIS_PASSWORD= + +DB_INIT=0 +DJANGO_ENV=production +GUNICORN_WORKERS=3 + +gpt_catalog_war=./wars/geoportal.war +gpt_harvester_war=./wars/harvester.war + +ELASTIC_PASSWORD= +CLUSTER_NAME=elasticsearch +ES_REINDEX_REMOTE_WHITELIST=elastic.prod.wcoa.ecotrust.org:80 +``` + +Notes: + +- In production compose, APP_PORT controls host mapping only. Gunicorn binds container port 8008. + +### 4.2 Confirm production ini selection + +WCOA production compose mounts and uses: + +- docker/config.wcoa.prod.ini + +No change is required unless you need environment-specific non-secret overrides. + +## 5. Authenticate and first boot + +Log in to GHCR on host: + +```bash +echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin +``` + +Pull and start services: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env pull +docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +``` + +If this is first boot and you need Django init tasks: + +```bash +DB_INIT=1 docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +``` + +After init, ensure DB_INIT is set back to 0 in docker/.env. + +## 6. Load data + +Restore database dump with production compose: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env exec -T \ + -e PGPASSWORD="$DB_PASSWORD" db psql -U "$DB_USER" -d "$DB_NAME" \ + < /path/to/dump.sql +``` + +Apply migrations: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migrate +``` + +If required for legacy layers migration: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migration_to_layers +``` + +Rebuild static assets if needed: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py collectstatic --noinput +docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py compress --force +``` + +## 7. Nginx routes for WCOA + +Use core Nginx/TLS setup pattern, then add WCOA upstream routes. + +Core app route should proxy to host loopback APP_PORT. + +Example WCOA-specific upstreams: + +- /geoportal and /harvester -> 127.0.0.1:8080 +- /_search, /_doc, /metadata -> 127.0.0.1:9200 +- /geospatial -> filesystem path expected by Geoportal +- /nativeland -> app route as configured + +Do not proxy internal services via public IP. Use loopback targets. + +## 8. Elasticsearch and Geoportal checks + +Verify service health: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env ps +docker compose -f docker/compose.prod.yml --env-file docker/.env logs -f elastic +docker compose -f docker/compose.prod.yml --env-file docker/.env logs -f geoportal +``` + +Register snapshot repository if not yet configured: + +```bash +curl -X PUT "http://127.0.0.1:9200/_snapshot/gp_es_snap" \ + -H 'Content-Type: application/json' \ + -d '{"type":"fs","settings":{"location":"/usr/share/elasticsearch/backups"}}' +``` + +Create a test snapshot: + +```bash +./scripts/create_elastic_snapshot.sh -r gp_es_snap +``` + +## 9. systemd service for WCOA + +Create unit file such as /etc/systemd/system/wcoa.service: + +```ini +[Unit] +Description=WCOA Docker Stack +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/home/ubuntu/portals/madrona-apps/wcoa/docker +ExecStart=/usr/bin/docker compose -f compose.prod.yml --env-file .env up -d +ExecStop=/usr/bin/docker compose -f compose.prod.yml --env-file .env down +TimeoutStartSec=0 + +[Install] +WantedBy=multi-user.target +``` + +Enable and start: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable wcoa.service +sudo systemctl start wcoa.service +``` + +## 10. Cron jobs for WCOA + +Install cron entries for DB dump, ES snapshots, and nativeland refresh. + +Recommended entries: + +```cron +# DB dump retention +15 2 * * * cd /home/ubuntu/portals/madrona-apps/wcoa && /bin/bash -lc './scripts/db_dump.sh -c ./docker/compose.prod.yml -e ./docker/.env -d ./docker/backups/sql && find ./docker/backups/sql -type f -name "*.sql" -mtime +10 -delete' >> /home/ubuntu/portals/madrona-apps/wcoa/docker/backups/db_dump.log 2>&1 + +# Elasticsearch snapshot +15 3 * * * /usr/bin/bash /home/ubuntu/portals/madrona-apps/wcoa/scripts/create_elastic_snapshot.sh -r gp_es_snap + +# NativeLand refresh +31 5 * * * cd /home/ubuntu/portals/madrona-apps/wcoa/docker && docker compose -f compose.prod.yml --env-file .env exec app python marco/manage.py import_nativeland +``` + +## 11. Release and rollback + +### 11.1 Deploy a new release + +1. Set IMAGE_TAG to a pinned new SHA in docker/.env. +2. Pull and recreate: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env pull +docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +``` + +3. Verify app, db, elastic, and geoportal health. + +### 11.2 Rollback + +1. Set IMAGE_TAG back to prior known-good SHA. +2. Pull and recreate using same commands. +3. Re-verify health and core routes. + +## 12. Services and ports reference + +Container services: + +- app (Gunicorn inside container on 8008) +- db (PostGIS on 5432) +- tasks (Redis on 6379) +- geoportal (Tomcat on 8080) +- elastic (Elasticsearch on 9200/9300) +- kibana (dev profile only) +- nginx (dev profile only) + +Host-facing defaults: + +- APP_PORT default 8000 mapped to container 8008 +- DB_PORT default 5432 mapped to container 5432 +- Geoportal and Elasticsearch are mapped directly in compose and should remain security-group restricted + +## 13. Troubleshooting + +Useful checks: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env ps +docker compose -f docker/compose.prod.yml --env-file docker/.env logs -f app +``` + +Run a Django command: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py +``` + +Database connectivity smoke test: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env exec db pg_isready -U "$DB_USER" -d "$DB_NAME" +``` + +If APP_PORT changes, confirm Nginx proxy_pass target is updated to match host-side port. diff --git a/docs/PRODUCTION_CUTOVER.md b/docs/PRODUCTION_CUTOVER.md new file mode 100644 index 0000000..5eca0b1 --- /dev/null +++ b/docs/PRODUCTION_CUTOVER.md @@ -0,0 +1,210 @@ +# WCOA Production Cutover Runbook + +This runbook covers migration of the currently running production WCOA stack from the old coupled deployment to the decoupled WCOA stack. + +Use this document for production cutover planning and execution. For baseline host setup and greenfield deployment, use: + +- Core host guide: ../../../madrona-portal/docs/AWS_DEPLOY.md +- WCOA deploy runbook: ./AWS_DEPLOY_WCOA.md + +## 0. Preconditions + +Do not begin cutover until all preconditions are true. + +- Core base image and WCOA overlay image are published and pinned by tag. +- The target WCOA image tag was validated in staging. +- A full database restore test from the latest production dump has succeeded in staging. +- Latest media backup has been verified readable. +- Maintenance window is approved and communicated. +- Rollback owner and decision authority are explicitly assigned. + +## 1. Capture current production state + +Before changing anything, capture all current state artifacts and copy them off-instance. + +Capture checklist: + +- Running app image tag and digest. +- Current docker environment file values. +- Current nginx site config. +- Current cron configuration. +- Fresh PostgreSQL dump. +- Fresh Elasticsearch snapshot. +- Media tarball backup. + +Example commands: + +```bash +# Record running image references +docker ps --format '{{.Names}} {{.Image}}' > /tmp/cutover-running-images.txt + +# Save current crontab and nginx +crontab -l > /tmp/cutover-crontab.txt +sudo cp /etc/nginx/sites-available/default /tmp/cutover-nginx-default.conf + +# DB dump and ES snapshot from existing workflow paths (adjust if needed) +cd /home/ubuntu/portals/madrona-apps/wcoa +./scripts/db_dump.sh -c ./docker/compose.prod.yml -e ./docker/.env -d ./docker/backups/sql +./scripts/create_elastic_snapshot.sh -r gp_es_snap + +# Media backup +cd /home/ubuntu/portals/madrona-apps/wcoa +tar -czf /tmp/cutover-media-$(date +%F_%H-%M-%S).tgz docker/media +``` + +Copy all /tmp/cutover-* artifacts and backup files to durable external storage. + +## 2. Choose cutover strategy + +### Strategy A: In-place cutover + +Use the same instance, stop old services, deploy decoupled stack in-place. + +- Pros: fastest, no DNS/EIP move. +- Cons: highest blast radius. +- Rollback model: restart old stack with prior configs and image tags. + +### Strategy B: Side-by-side cutover (recommended) + +Provision a new instance, restore data, verify with staging hostname, then move Elastic IP or DNS. + +- Pros: safest, clean rollback by switching traffic back. +- Cons: requires temporary duplicate infrastructure. +- Rollback model: move Elastic IP/DNS back to old instance. + +Recommendation: use side-by-side in production. + +## 3. Data continuity warning + +Important: Docker named volumes are namespaced by compose project name. + +If compose project names differ between old and new stacks, the new stack will not see old volumes automatically. + +Required practice: + +- Always perform explicit DB restore and media restore into the new stack. +- Always validate Elasticsearch snapshot repository and restore path explicitly. +- Never assume volume reuse as a migration method. + +## 4. Cutover steps + +### 4.1 Prepare target stack + +On the target host: + +- Clone and configure WCOA repo. +- Place WAR files and media. +- Prepare docker env and production config. +- Pull and boot decoupled stack. + +```bash +cd /home/ubuntu/portals/madrona-apps/wcoa +docker compose -f docker/compose.prod.yml --env-file docker/.env pull +docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +``` + +### 4.2 Restore data and validate services + +```bash +# Restore DB from a known-good dump +docker compose -f docker/compose.prod.yml --env-file docker/.env exec -T \ + -e PGPASSWORD="$DB_PASSWORD" db psql -U "$DB_USER" -d "$DB_NAME" \ + < /path/to/production_dump.sql + +# Run migrations +docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migrate +``` + +If needed for legacy path alignment: + +```bash +docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migration_to_layers +``` + +### 4.3 Traffic move + +For side-by-side: + +- Move Elastic IP to new instance or switch DNS A record. +- Verify nginx and certbot on new host. +- Confirm security group does not expose internal ports. + +For in-place: + +- Stop old stack. +- Start decoupled stack. +- Validate ingress and app health immediately. + +## 5. Verification checklist + +Complete all checks before declaring success. + +Application: + +- Homepage loads. +- Admin login works. +- Representative map/layer pages load. +- Static and media assets load. + +Data/catalog: + +- Elasticsearch responds and metadata search works. +- Geoportal and harvester routes respond. +- Geospatial-related routes required by WCOA respond. + +Background jobs: + +- Celery task execution round-trip works. +- Cron entries are installed and dry-run successfully. + +Operational: + +- systemd unit status is healthy. +- No recurring critical errors in app/db/elastic/geoportal logs. + +## 6. Rollback triggers and commands + +Rollback immediately if any of these are true after remediation attempts during the window: + +- Core page flows remain unavailable for more than 10 minutes. +- Data integrity checks fail (missing critical records, broken catalog indices). +- Login/admin workflows remain broken. +- Unexpected high error rate persists in app logs. + +### Strategy A rollback (in-place) + +Use previously captured old-stack compose/config files and prior image tags. + +```bash +# Stop decoupled stack +cd /home/ubuntu/portals/madrona-apps/wcoa/docker +docker compose -f compose.prod.yml --env-file .env down + +# Start prior stack using its original compose/env paths +cd /home/ubuntu/portals/ +docker compose -f --env-file up -d +``` + +### Strategy B rollback (side-by-side) + +```bash +# Move traffic back to old instance +# Option 1: re-associate Elastic IP to old instance +# Option 2: revert DNS A/ALIAS to old instance endpoint +``` + +Then verify old stack health and keep the failed new stack online but isolated for diagnosis. + +## 7. Decommission and retention + +After stable operation period (recommended minimum 7 days): + +- Capture final post-cutover DB dump. +- Capture final Elasticsearch snapshot. +- Archive old instance logs and configs. +- Remove old instance only after backups are verified. + +Retention recommendation: + +- Keep pre-cutover and immediate post-cutover backup sets for at least 30 days. +- Record cutover date, deployed image tag, rollback point, and final acceptance sign-off. diff --git a/docs/production-rollout-docker-decoupling.md b/docs/production-rollout-docker-decoupling.md deleted file mode 100644 index 8930009..0000000 --- a/docs/production-rollout-docker-decoupling.md +++ /dev/null @@ -1,178 +0,0 @@ -# WCOA production rollout: Docker decoupling - -This runbook covers the production rollout for the WCOA Docker decoupling work. It assumes the application is deployed from the WCOA repo using the image-based compose file in [docker/compose.prod.yml](../docker/compose.prod.yml). - -## Goal - -Ship the decoupled WCOA deployment so the portal runs from its own image and config while still using the shared Madrona base image. - -## Recommended rollout order - -1. Confirm the shared base image is ready. -2. Build and publish the WCOA overlay image. -3. Deploy the new image to production. -4. Verify the app, DB, and supporting services. -5. Roll back quickly if anything fails. - ---- - -## 1. Prepare the release - -### Confirm the base image - -The WCOA overlay image depends on the shared Madrona base image published from the core repo. - -Before rollout: - -- Confirm the core image build completed successfully. -- Note the base image tag you want to use, preferably a pinned SHA or release tag rather than `latest`. -- Confirm the WCOA image build will use that base tag in the Docker build arguments. - -### Confirm release content - -Verify the following are ready: - -- The WCOA repo branch or PR is merged or otherwise approved for production. -- The image workflow completed successfully. -- The production environment has the required secrets in its `.env` file. -- Any required WAR files and media assets are present in the deployment host. - ---- - -## 2. Build and publish the WCOA image - -The WCOA workflow publishes images to GHCR from [/.github/workflows/build-and-publish-image.yml](../.github/workflows/build-and-publish-image.yml). - -### What to expect - -The workflow publishes: - -- `ghcr.io/ecotrust/wcoa:` -- `ghcr.io/ecotrust/wcoa:latest` - -### Production recommendation - -For production, prefer deploying a pinned SHA rather than `latest`. - -Example: - -```bash -# Example only: use the SHA from the successful workflow run -export IMAGE_TAG= -``` - ---- - -## 3. Prepare the production host - -On the deployment host, make sure the production environment is ready. - -### Required files - -Ensure the host has: - -- The production `.env` file with secrets and runtime settings -- The compose file at [docker/compose.prod.yml](../docker/compose.prod.yml) -- The production WCOA config file at `docker/config.wcoa.prod.ini` mounted by the container -- Static and media directories with correct permissions - -Before first startup, create the Elasticsearch snapshot path expected by compose: - -```bash -mkdir -p docker/backups/elasticsearch -chown -R "$(id -u)":"$(id -g)" docker/backups/elasticsearch -``` - -### Minimum environment values - -At minimum confirm these values are present in the host `.env`: - -```env -SECRET_KEY=... -DB_PASSWORD=... -REDIS_PASSWORD=... -DB_NAME=wcoa_docker_db -IMAGE_TAG= -DJANGO_SUPERUSER_PASSWORD=... -``` - -If this is the first production deployment after the decoupled rollout, set `DB_INIT=1` once for initial boot and data setup. After that, switch it back to `0`. - ---- - -## 4. Deploy the new image - -From the WCOA deployment directory, pull and recreate the stack. - -```bash -docker compose -f docker/compose.prod.yml --env-file docker/.env pull -docker compose -f docker/compose.prod.yml --env-file docker/.env up -d -``` - -If this is the first boot or the database needs initialization: - -```bash -DB_INIT=1 docker compose -f docker/compose.prod.yml --env-file docker/.env up -d -``` - -After initialization, set `DB_INIT=0` in the environment and restart the app service if needed. - ---- - -## 5. Verify the deployment - -### Container health - -```bash -docker compose -f docker/compose.prod.yml --env-file docker/.env ps -docker compose -f docker/compose.prod.yml --env-file docker/.env logs -f app -``` - -### Functional checks - -Verify: - -- The app responds on the expected host port. -- The portal loads without obvious errors. -- The database connection is healthy. -- Static and media files are served correctly. -- Geoportal and Elasticsearch are healthy if those services are part of the deployment. - -If you have a smoke-test endpoint or a browser-based check, use it at this stage. - ---- - -## 6. Roll back if needed - -If the deployment shows problems, revert to the previous known-good image tag. - -```bash -export IMAGE_TAG= -docker compose -f docker/compose.prod.yml --env-file docker/.env pull -docker compose -f docker/compose.prod.yml --env-file docker/.env up -d -``` - -Keep the previous image tag recorded so rollback is fast. - ---- - -## 7. Post-deploy notes - -After the rollout is confirmed: - -- Record the deployed image tag in the deployment notes. -- Capture the date, release SHA, and any config changes. -- Keep the previous image tag available for rollback until the next deployment is stable. -- If the rollout included a new database migration or fixture load, verify the data shape before removing the temporary initialization step. - ---- - -## Quick checklist - -- [ ] Core base image build succeeded -- [ ] WCOA image workflow succeeded -- [ ] Production `.env` is ready -- [ ] Image tag is pinned for production -- [ ] Containers pulled and recreated -- [ ] App, DB, and supporting services are healthy -- [ ] Rollback target is recorded From 44e3e0b77fa91b7c546082ddc8f647f921c358c6 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 16:45:06 -0700 Subject: [PATCH 32/38] Refactor AWS Deployment and Production Cutover runbooks for clarity and consistency --- docs/AWS_DEPLOY_WCOA.md | 36 ++++++++++++++-------------- docs/PRODUCTION_CUTOVER.md | 49 ++++++++++++++++---------------------- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/docs/AWS_DEPLOY_WCOA.md b/docs/AWS_DEPLOY_WCOA.md index ee0a2ac..4bbe1e3 100644 --- a/docs/AWS_DEPLOY_WCOA.md +++ b/docs/AWS_DEPLOY_WCOA.md @@ -2,9 +2,9 @@ This runbook is for deploying WCOA on a host that is already prepared using the core platform guide in [madrona-portal/docs/AWS_DEPLOY.md](../../../madrona-portal/docs/AWS_DEPLOY.md). -Use this document for greenfield WCOA deployment and repeatable release operations. +Use this document when starting from scratch with WCOA deployment and repeatable release operations. -## 1. What WCOA adds +## What WCOA adds Compared to the platform baseline, WCOA adds: @@ -19,7 +19,7 @@ Sizing guidance: - Use at least 60 GB root volume. - Monitor memory and disk pressure during indexing jobs. -## 2. Prerequisites to collect +## Prerequisites to collect From secure credential storage, collect: @@ -32,7 +32,7 @@ From secure credential storage, collect: - Current database dump for initial load. - Media backup archive. -## 3. Clone repository and place artifacts +## Clone repository and place artifacts On the host: @@ -60,9 +60,9 @@ Set directory ownership for Elasticsearch snapshots: chown -R "$(id -u)":"$(id -g)" docker/backups/elasticsearch ``` -## 4. Configure WCOA +## Configure WCOA -### 4.1 Create docker environment file +### Create docker environment file ```bash cp docker/.env.example docker/.env @@ -103,7 +103,7 @@ Notes: - In production compose, APP_PORT controls host mapping only. Gunicorn binds container port 8008. -### 4.2 Confirm production ini selection +### Confirm production ini selection WCOA production compose mounts and uses: @@ -111,7 +111,7 @@ WCOA production compose mounts and uses: No change is required unless you need environment-specific non-secret overrides. -## 5. Authenticate and first boot +## Authenticate and first boot Log in to GHCR on host: @@ -134,7 +134,7 @@ DB_INIT=1 docker compose -f docker/compose.prod.yml --env-file docker/.env up -d After init, ensure DB_INIT is set back to 0 in docker/.env. -## 6. Load data +## Load data Restore database dump with production compose: @@ -163,7 +163,7 @@ docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py compress --force ``` -## 7. Nginx routes for WCOA +## Nginx routes for WCOA Use core Nginx/TLS setup pattern, then add WCOA upstream routes. @@ -178,7 +178,7 @@ Example WCOA-specific upstreams: Do not proxy internal services via public IP. Use loopback targets. -## 8. Elasticsearch and Geoportal checks +## Elasticsearch and Geoportal checks Verify service health: @@ -202,7 +202,7 @@ Create a test snapshot: ./scripts/create_elastic_snapshot.sh -r gp_es_snap ``` -## 9. systemd service for WCOA +## systemd service for WCOA Create unit file such as /etc/systemd/system/wcoa.service: @@ -232,7 +232,7 @@ sudo systemctl enable wcoa.service sudo systemctl start wcoa.service ``` -## 10. Cron jobs for WCOA +## Cron jobs for WCOA Install cron entries for DB dump, ES snapshots, and nativeland refresh. @@ -249,9 +249,9 @@ Recommended entries: 31 5 * * * cd /home/ubuntu/portals/madrona-apps/wcoa/docker && docker compose -f compose.prod.yml --env-file .env exec app python marco/manage.py import_nativeland ``` -## 11. Release and rollback +## Release and rollback -### 11.1 Deploy a new release +### Deploy a new release 1. Set IMAGE_TAG to a pinned new SHA in docker/.env. 2. Pull and recreate: @@ -263,13 +263,13 @@ docker compose -f docker/compose.prod.yml --env-file docker/.env up -d 3. Verify app, db, elastic, and geoportal health. -### 11.2 Rollback +### Rollback 1. Set IMAGE_TAG back to prior known-good SHA. 2. Pull and recreate using same commands. 3. Re-verify health and core routes. -## 12. Services and ports reference +## Services and ports reference Container services: @@ -287,7 +287,7 @@ Host-facing defaults: - DB_PORT default 5432 mapped to container 5432 - Geoportal and Elasticsearch are mapped directly in compose and should remain security-group restricted -## 13. Troubleshooting +## Troubleshooting Useful checks: diff --git a/docs/PRODUCTION_CUTOVER.md b/docs/PRODUCTION_CUTOVER.md index 5eca0b1..b0d9c83 100644 --- a/docs/PRODUCTION_CUTOVER.md +++ b/docs/PRODUCTION_CUTOVER.md @@ -7,7 +7,7 @@ Use this document for production cutover planning and execution. For baseline ho - Core host guide: ../../../madrona-portal/docs/AWS_DEPLOY.md - WCOA deploy runbook: ./AWS_DEPLOY_WCOA.md -## 0. Preconditions +## Preconditions Do not begin cutover until all preconditions are true. @@ -18,7 +18,7 @@ Do not begin cutover until all preconditions are true. - Maintenance window is approved and communicated. - Rollback owner and decision authority are explicitly assigned. -## 1. Capture current production state +## Capture current production state Before changing anything, capture all current state artifacts and copy them off-instance. @@ -54,7 +54,7 @@ tar -czf /tmp/cutover-media-$(date +%F_%H-%M-%S).tgz docker/media Copy all /tmp/cutover-* artifacts and backup files to durable external storage. -## 2. Choose cutover strategy +## Choose cutover strategy ### Strategy A: In-place cutover @@ -64,7 +64,7 @@ Use the same instance, stop old services, deploy decoupled stack in-place. - Cons: highest blast radius. - Rollback model: restart old stack with prior configs and image tags. -### Strategy B: Side-by-side cutover (recommended) +### Strategy B: Side-by-side cutover Provision a new instance, restore data, verify with staging hostname, then move Elastic IP or DNS. @@ -72,23 +72,21 @@ Provision a new instance, restore data, verify with staging hostname, then move - Cons: requires temporary duplicate infrastructure. - Rollback model: move Elastic IP/DNS back to old instance. -Recommendation: use side-by-side in production. - ## 3. Data continuity warning -Important: Docker named volumes are namespaced by compose project name. +Docker named volumes are namespaced by compose project name. If compose project names differ between old and new stacks, the new stack will not see old volumes automatically. -Required practice: +Best practice: -- Always perform explicit DB restore and media restore into the new stack. -- Always validate Elasticsearch snapshot repository and restore path explicitly. -- Never assume volume reuse as a migration method. +- perform explicit DB restore and media restore into the new stack. +- validate Elasticsearch snapshot repository and restore path explicitly. +- assume volume reuse as a migration method. -## 4. Cutover steps +## Cutover steps -### 4.1 Prepare target stack +### Prepare target stack On the target host: @@ -103,13 +101,11 @@ docker compose -f docker/compose.prod.yml --env-file docker/.env pull docker compose -f docker/compose.prod.yml --env-file docker/.env up -d ``` -### 4.2 Restore data and validate services +### Restore data and validate services ```bash # Restore DB from a known-good dump -docker compose -f docker/compose.prod.yml --env-file docker/.env exec -T \ - -e PGPASSWORD="$DB_PASSWORD" db psql -U "$DB_USER" -d "$DB_NAME" \ - < /path/to/production_dump.sql +scripts/db_restore.sh -c ./docker/compose.prod.yml -e ./docker/.env -d ./docker/backups/sql/.sql # Run migrations docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migrate @@ -121,7 +117,7 @@ If needed for legacy path alignment: docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migration_to_layers ``` -### 4.3 Traffic move +### Traffic move For side-by-side: @@ -135,7 +131,7 @@ For in-place: - Start decoupled stack. - Validate ingress and app health immediately. -## 5. Verification checklist +## Verification checklist Complete all checks before declaring success. @@ -162,9 +158,9 @@ Operational: - systemd unit status is healthy. - No recurring critical errors in app/db/elastic/geoportal logs. -## 6. Rollback triggers and commands +## Rollback triggers and commands -Rollback immediately if any of these are true after remediation attempts during the window: +Rollback if any of these are true after remediation attempts during the window: - Core page flows remain unavailable for more than 10 minutes. - Data integrity checks fail (missing critical records, broken catalog indices). @@ -195,16 +191,11 @@ docker compose -f --env-file up -d Then verify old stack health and keep the failed new stack online but isolated for diagnosis. -## 7. Decommission and retention +## Decommission and retention -After stable operation period (recommended minimum 7 days): +After stable operation: - Capture final post-cutover DB dump. - Capture final Elasticsearch snapshot. - Archive old instance logs and configs. -- Remove old instance only after backups are verified. - -Retention recommendation: - -- Keep pre-cutover and immediate post-cutover backup sets for at least 30 days. -- Record cutover date, deployed image tag, rollback point, and final acceptance sign-off. +- Remove old instance. From 9b02c574b79b1d865bcc7b85ce8d753f2568782c Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 5 Aug 2026 16:48:04 -0700 Subject: [PATCH 33/38] Update WCOA Production Cutover documentation for clarity and consistency --- docs/PRODUCTION_CUTOVER.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/PRODUCTION_CUTOVER.md b/docs/PRODUCTION_CUTOVER.md index b0d9c83..5a65017 100644 --- a/docs/PRODUCTION_CUTOVER.md +++ b/docs/PRODUCTION_CUTOVER.md @@ -1,15 +1,15 @@ -# WCOA Production Cutover Runbook +# WCOA Production Cutover -This runbook covers migration of the currently running production WCOA stack from the old coupled deployment to the decoupled WCOA stack. +This document covers migration of the currently running production WCOA stack from the old coupled deployment to the decoupled WCOA stack. Use this document for production cutover planning and execution. For baseline host setup and greenfield deployment, use: -- Core host guide: ../../../madrona-portal/docs/AWS_DEPLOY.md +- Core host guide: madrona-portal – docs/AWS_DEPLOY.md - WCOA deploy runbook: ./AWS_DEPLOY_WCOA.md ## Preconditions -Do not begin cutover until all preconditions are true. +Begin when all of these are true: - Core base image and WCOA overlay image are published and pinned by tag. - The target WCOA image tag was validated in staging. From 7b173061b99df07ca12b0eba4e045472f0069635 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 7 Aug 2026 12:18:58 -0700 Subject: [PATCH 34/38] Enhance AWS Deployment and Production Cutover documentation with additional commands and clarifications for WCOA service setup and validation --- docs/AWS_DEPLOY_WCOA.md | 38 ++++++++++- docs/PRODUCTION_CUTOVER.md | 131 +++++++++++++++++++++++++++++++++---- 2 files changed, 153 insertions(+), 16 deletions(-) diff --git a/docs/AWS_DEPLOY_WCOA.md b/docs/AWS_DEPLOY_WCOA.md index 4bbe1e3..e145d68 100644 --- a/docs/AWS_DEPLOY_WCOA.md +++ b/docs/AWS_DEPLOY_WCOA.md @@ -204,8 +204,21 @@ Create a test snapshot: ## systemd service for WCOA +Before enabling the WCOA unit, disable any legacy portal units that can restart the old stack: + +```bash +# Staging +sudo systemctl disable --now staging.madrona-portal.service || true +# Production +sudo systemctl disable --now madrona-portal.service || true +``` + Create unit file such as /etc/systemd/system/wcoa.service: +```bash +sudo nano /etc/systemd/system/wcoa.service +``` + ```ini [Unit] Description=WCOA Docker Stack @@ -216,8 +229,8 @@ Requires=docker.service Type=oneshot RemainAfterExit=yes WorkingDirectory=/home/ubuntu/portals/madrona-apps/wcoa/docker -ExecStart=/usr/bin/docker compose -f compose.prod.yml --env-file .env up -d -ExecStop=/usr/bin/docker compose -f compose.prod.yml --env-file .env down +ExecStart=/usr/bin/docker compose -f /home/ubuntu/portals/madrona-apps/wcoa/docker/compose.prod.yml --env-file /home/ubuntu/portals/madrona-apps/wcoa/docker/.env up -d +ExecStop=/usr/bin/docker compose -f /home/ubuntu/portals/madrona-apps/wcoa/docker/compose.prod.yml --env-file /home/ubuntu/portals/madrona-apps/wcoa/docker/.env down TimeoutStartSec=0 [Install] @@ -232,6 +245,14 @@ sudo systemctl enable wcoa.service sudo systemctl start wcoa.service ``` +Confirm only the intended unit is enabled and that the app container image is WCOA: + +```bash +systemctl list-unit-files | grep -E 'wcoa|madrona' +systemctl status wcoa.service --no-pager +docker ps --format '{{.Names}} {{.Image}}' | grep -E 'wcoa|madrona-portal' +``` + ## Cron jobs for WCOA Install cron entries for DB dump, ES snapshots, and nativeland refresh. @@ -249,6 +270,12 @@ Recommended entries: 31 5 * * * cd /home/ubuntu/portals/madrona-apps/wcoa/docker && docker compose -f compose.prod.yml --env-file .env exec app python marco/manage.py import_nativeland ``` +## Restart Nginx after cutover + +```bash +sudo service nginx restart +``` + ## Release and rollback ### Deploy a new release @@ -262,6 +289,13 @@ docker compose -f docker/compose.prod.yml --env-file docker/.env up -d ``` 3. Verify app, db, elastic, and geoportal health. +4. Verify the running app image matches the intended WCOA tag: + +```bash +docker ps --format '{{.Names}} {{.Image}}' | grep app +``` + +The app image should be `ghcr.io/ecotrust/wcoa:`, not `ghcr.io/ecotrust/madrona-portal:`. ### Rollback diff --git a/docs/PRODUCTION_CUTOVER.md b/docs/PRODUCTION_CUTOVER.md index 5a65017..82d9e05 100644 --- a/docs/PRODUCTION_CUTOVER.md +++ b/docs/PRODUCTION_CUTOVER.md @@ -24,13 +24,13 @@ Before changing anything, capture all current state artifacts and copy them off- Capture checklist: -- Running app image tag and digest. -- Current docker environment file values. -- Current nginx site config. -- Current cron configuration. -- Fresh PostgreSQL dump. -- Fresh Elasticsearch snapshot. -- Media tarball backup. +- [] Running app image tag +- [] Current docker environment file values. +- [] Current nginx site config. +- [] Current cron configuration. +- [] Fresh PostgreSQL dump. +- [] Fresh Elasticsearch snapshot. +- [] Media tarball backup. Example commands: @@ -39,8 +39,8 @@ Example commands: docker ps --format '{{.Names}} {{.Image}}' > /tmp/cutover-running-images.txt # Save current crontab and nginx -crontab -l > /tmp/cutover-crontab.txt -sudo cp /etc/nginx/sites-available/default /tmp/cutover-nginx-default.conf +sudo crontab -l > /tmp/cutover-crontab.txt +sudo cp /etc/nginx/sites-available/madrona-portal /tmp/cutover-nginx-madrona-portal.conf # DB dump and ES snapshot from existing workflow paths (adjust if needed) cd /home/ubuntu/portals/madrona-apps/wcoa @@ -48,7 +48,7 @@ cd /home/ubuntu/portals/madrona-apps/wcoa ./scripts/create_elastic_snapshot.sh -r gp_es_snap # Media backup -cd /home/ubuntu/portals/madrona-apps/wcoa +cd /home/ubuntu/portals/madrona-portal tar -czf /tmp/cutover-media-$(date +%F_%H-%M-%S).tgz docker/media ``` @@ -96,16 +96,46 @@ On the target host: - Pull and boot decoupled stack. ```bash -cd /home/ubuntu/portals/madrona-apps/wcoa -docker compose -f docker/compose.prod.yml --env-file docker/.env pull -docker compose -f docker/compose.prod.yml --env-file docker/.env up -d +mkdir madrona-apps +git clone https://github.com/Ecotrust/wcoa.git +cd wcoa/docker +cp ~/portals/madrona-portal/docker/.env ./ +``` + +### Copy WAR files, media, and backups + +```bash +cp -r ~/portals/madrona-portal/docker/media ./media +cp -r ~/portals/madrona-portal/docker/wars ./wars +cp -r ~/portals/madrona-portal/docker/backups ./backups +``` + +### Stop old stack +```bash +cd ../../../madrona-portal/ +docker compose -f docker/docker-compose.prod.yml down + +# Prevent old stack from being auto-started by systemd after reboot +# Staging +sudo systemctl disable --now staging.madrona-portal.service || true +# Production +sudo systemctl disable --now madrona-portal.service || true +``` + +### Start new stack +```bash +cd /home/ubuntu/portals/madrona-apps/wcoa/docker +docker compose -f compose.prod.yml --env-file ./.env up -d + +# Validate that app is running from WCOA image +docker ps --format '{{.Names}} {{.Image}}' | grep -E 'app|wcoa|madrona-portal' ``` ### Restore data and validate services ```bash # Restore DB from a known-good dump -scripts/db_restore.sh -c ./docker/compose.prod.yml -e ./docker/.env -d ./docker/backups/sql/.sql +scripts/db-restore.sh --core-compose ./docker/compose.prod.yml -e ./docker/.env -d ./docker/backups/sql/.sql # Run migrations docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migrate @@ -117,6 +147,79 @@ If needed for legacy path alignment: docker compose -f docker/compose.prod.yml --env-file docker/.env exec app python marco/manage.py migration_to_layers ``` +### Update nginx + +```bash +sudo vim /etc/nginx/sites-available/madrona-portal +``` + +update paths + +### Add WCOA service and cron entries + +Create unit file such as /etc/systemd/system/wcoa.service: + +```bash +sudo nano /etc/systemd/system/wcoa.service +``` + +```ini +[Unit] +Description=WCOA Docker Stack +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/home/ubuntu/portals/madrona-apps/wcoa/docker +ExecStart=/usr/bin/docker compose -f /home/ubuntu/portals/madrona-apps/wcoa/docker/compose.prod.yml --env-file /home/ubuntu/portals/madrona-apps/wcoa/docker/.env up -d +ExecStop=/usr/bin/docker compose -f /home/ubuntu/portals/madrona-apps/wcoa/docker/compose.prod.yml --env-file /home/ubuntu/portals/madrona-apps/wcoa/docker/.env down +TimeoutStartSec=0 + +[Install] +WantedBy=multi-user.target +``` + +Enable and start: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable wcoa.service +sudo systemctl start wcoa.service +``` + +Confirm only the intended unit is enabled and that the app container image is WCOA: + +```bash +systemctl list-unit-files | grep -E 'wcoa|madrona' +systemctl status wcoa.service --no-pager +docker ps --format '{{.Names}} {{.Image}}' | grep -E 'wcoa|madrona-portal' +``` + +## Cron jobs for WCOA + +Install cron entries for DB dump, ES snapshots, and nativeland refresh. + +Recommended entries: + +```cron +# DB dump retention +15 2 * * * cd /home/ubuntu/portals/madrona-apps/wcoa && /bin/bash -lc './scripts/db_dump.sh -c ./docker/compose.prod.yml -e ./docker/.env -d ./docker/backups/sql && find ./docker/backups/sql -type f -name "*.sql" -mtime +10 -delete' >> /home/ubuntu/portals/madrona-apps/wcoa/docker/backups/db_dump.log 2>&1 + +# Elasticsearch snapshot +15 3 * * * /usr/bin/bash /home/ubuntu/portals/madrona-apps/wcoa/scripts/create_elastic_snapshot.sh -r gp_es_snap + +# NativeLand refresh +31 5 * * * cd /home/ubuntu/portals/madrona-apps/wcoa/docker && docker compose -f compose.prod.yml --env-file .env exec app python marco/manage.py import_nativeland +``` + +## Restart Nginx after cutover + +```bash +sudo service nginx restart +``` + ### Traffic move For side-by-side: From 61f5bf40087028391a68dabde6e82f9e205aeb25 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 11 Aug 2026 14:26:35 -0700 Subject: [PATCH 35/38] Update Docker Buildx action to version 4 for improved functionality --- .github/workflows/build-and-publish-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 9f28f68..696627b 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to GHCR uses: docker/login-action@v3 From e7ff2f22282dfaa66fe65163c61015f36465c271 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 11 Aug 2026 14:28:07 -0700 Subject: [PATCH 36/38] Add QEMU setup step to Docker build workflow --- .github/workflows/build-and-publish-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 696627b..2bacc33 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -21,6 +21,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 From a9f9ea086ab63692cb32f5ccf4dc154658d64e7d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 11 Aug 2026 14:41:58 -0700 Subject: [PATCH 37/38] Remove QEMU and multiple platforms bc of huge build and push action --- .github/workflows/build-and-publish-image.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index 2bacc33..de39340 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -8,7 +8,6 @@ on: env: BASE_TAG: latest IMAGE_NAME: ghcr.io/ecotrust/wcoa - IMAGE_PLATFORMS: linux/amd64,linux/arm64 jobs: build-and-push: @@ -44,7 +43,6 @@ jobs: context: . file: docker/Dockerfile push: true - platforms: ${{ env.IMAGE_PLATFORMS }} build-args: | BASE_TAG=${{ env.BASE_TAG }} tags: | From e5fdccf9686d9f8a9dd0d39e48af1dabd511de03 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 11 Aug 2026 15:24:21 -0700 Subject: [PATCH 38/38] Add IMAGE_PLATFORMS environment variable and update build-push action to include QEMU tag --- .github/workflows/build-and-publish-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml index de39340..d0c0fd3 100644 --- a/.github/workflows/build-and-publish-image.yml +++ b/.github/workflows/build-and-publish-image.yml @@ -8,6 +8,7 @@ on: env: BASE_TAG: latest IMAGE_NAME: ghcr.io/ecotrust/wcoa + IMAGE_PLATFORMS: linux/amd64,linux/arm64 jobs: build-and-push: @@ -43,10 +44,12 @@ jobs: context: . file: docker/Dockerfile push: true + platforms: ${{ env.IMAGE_PLATFORMS }} build-args: | BASE_TAG=${{ env.BASE_TAG }} tags: | ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} ${{ env.IMAGE_NAME }}:latest + ${{ env.IMAGE_NAME }}:qemu cache-from: type=gha cache-to: type=gha,mode=max