From 96ff7596f24625359d3469efc402c12ba5c0762d Mon Sep 17 00:00:00 2001 From: thelinuxguy-ssh Date: Mon, 3 Aug 2026 00:01:17 +0530 Subject: [PATCH] Setup Docker CI pipeline for release --- .github/workflows/release.yml | 52 ++++++++++++++ README.md | 128 ++++++++++++++++++++++++++++++---- docker-compose.yml | 9 ++- 3 files changed, 172 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..24d420e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,52 @@ +name: Publish Docker Image + +on: + release: + types: [published] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=latest + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/README.md b/README.md index 06d57f5..ea4fd24 100644 --- a/README.md +++ b/README.md @@ -90,30 +90,116 @@ The development server runs at `http://localhost:5173`. ### Docker (recommended) -The project includes a `Dockerfile` and `docker-compose.yml` for containerized deployment. +A prebuilt image is published to [GitHub Container Registry](https://github.com/thelinuxguy-ssh/duskmeter/pkgs/container/duskmeter) on every release. Tags follow semantic versioning: `latest`, `v1`, `v1.0`, `v1.0.0`. + +#### Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) and Docker Compose installed +- A VIIRS yearly file downloaded from [EOG](https://eogdata.mines.edu/products/vnl/) and placed at `data/viirs/VNL_npp_{year}_global_vcmslcfg_v2_c*.average.dat.tif.gz` +- A value for `ADMIN_SECRET` — set it in a `.env` file or export it as an environment variable + +#### Understanding the database + +The application uses SQLite with a single file as its database. When running in development (`npm run dev`), the file is named `duskmeter-dev.sqlite`. When running in production (Docker or `node build`), it uses `duskmeter.sqlite`. This separation prevents your development work from accidentally overwriting production data. + +The database file is bind-mounted from your host into the container: + +``` +./duskmeter.sqlite (on your machine) → /app/duskmeter.sqlite (inside the container) +``` + +The container reads and writes directly to your host file. When the container restarts, is stopped, or is replaced with a newer image, the file stays on your machine — nothing is lost. + +#### Option A: Fresh start (no existing data) ```bash -# Copy your dev database for production use (skip if starting fresh) -cp duskmeter-dev.sqlite duskmeter.sqlite +# 1. Ensure the VIIRS file is in place +ls data/viirs/VNL_npp_*.tif.gz -# Build and start -docker compose up --build -d +# 2. Start the container (creates an empty database, seeds 12 default sites) +docker compose up -d -# Ingest VIIRS data +# 3. Ingest the VIIRS GeoTIFF into the database (reads pixel values at each site) curl -X POST "http://localhost:3000/api/admin/ingest-viirs" \ -H "Authorization: Bearer your-secret" -# Run initial sync +# 4. Run the initial sync (fetches Globe at Night data, computes metrics) curl -X POST "http://localhost:3000/api/sync?force=true" \ -H "Authorization: Bearer your-secret" + +# 5. Verify +curl http://localhost:3000/api/health +# {"status":"ok","db":"connected","sites":12} + +# 6. Open the dashboard +open http://localhost:3000 +``` + +#### Option B: Migrate from development + +If you have been running `npm run dev` and already have real VIIRS data and metrics in your database: + +```bash +# Rename the dev database for production use +cp duskmeter-dev.sqlite duskmeter.sqlite + +# Start the container — no re-ingestion needed, data is already in the database +docker compose up -d + +# Verify +curl http://localhost:3000/api/health +# {"status":"ok","db":"connected","sites":12} ``` -Volume mounts: -- `./data/viirs:/app/data/viirs` — VIIRS yearly GeoTIFF files -- `./data/gan:/app/data/gan` — Globe at Night cache -- `./duskmeter.sqlite:/app/duskmeter.sqlite` — Database +#### How data flows -The container auto-restarts unless explicitly stopped. Set `ADMIN_SECRET` in your environment or a `.env` file before running. +| Component | Host location | Container location | What happens | +|---|---|---|---| +| VIIRS GeoTIFF files | `./data/viirs/` | `/app/data/viirs/` | Mounted read/write. Ingest decompresses the file in `/tmp`, reads pixel values, stores results in the database, and deletes the temp file | +| Globe at Night cache | `./data/gan/` | `/app/data/gan/` | Mounted read/write. The container fetches remote data from NOIRLab and caches it here for faster subsequent syncs | +| SQLite database | `./duskmeter.sqlite` | `/app/duskmeter.sqlite` | Bind-mounted. The container reads and writes directly to your host file. Survives container restarts, rebuilds, and image updates | +| Application logs | stdout | stdout | View with `docker compose logs -f duskmeter`. JSON lines with timestamps, levels, and context | + +#### Adding new VIIRS years + +When a new yearly composite is released, download it and place it alongside existing files in `data/viirs/`. Then run ingestion and sync again — only the new year's data is processed: + +```bash +curl -X POST "http://localhost:3000/api/admin/ingest-viirs" \ + -H "Authorization: Bearer your-secret" +curl -X POST "http://localhost:3000/api/sync?force=true" \ + -H "Authorization: Bearer your-secret" +``` + +#### Backups + +The entire application state is in one file. Back it up with: + +```bash +cp duskmeter.sqlite duskmeter.sqlite.$(date +%Y-%m-%d).bak +``` + +The VIIRS files and Globe at Night cache in `./data/` are reproducible — they can be re-downloaded from their respective sources. Only the database needs backing up. + +#### Updating the container + +```bash +docker compose pull # Pull the latest published image +docker compose up -d # Replace the container — database and data files are untouched +``` + +#### Verifying the setup + +```bash +# Health check — confirms DB is connected, shows site count and last sync time +curl http://localhost:3000/api/health + +# List all sites with their corrected trends +curl http://localhost:3000/api/sites | python3 -m json.tool | head -20 + +# Check for active alerts +curl http://localhost:3000/api/alerts | python3 -m json.tool | head -10 +``` ### Standalone Node.js @@ -144,6 +230,24 @@ curl http://localhost:3000/api/health # {"status":"ok","version":"0.1.0","uptime":120,"db":"connected","sites":12,"lastSync":"..."} ``` +## CI/CD + +Two GitHub Actions workflows run automatically: + +| Workflow | Trigger | What It Does | +|---|---|---| +| [CI](.github/workflows/ci.yml) | Push to `main`, pull requests | Type check, run tests, verify production build | +| [Release](.github/workflows/release.yml) | GitHub Release published | Build multi-arch Docker image (`linux/amd64`, `linux/arm64`), push to `ghcr.io/thelinuxguy-ssh/duskmeter` with semver tags + `latest` | + +To create a release: + +```bash +git tag v0.1.0 +git push origin v0.1.0 +``` + +Then create a release on GitHub from that tag. The workflow builds and publishes the container automatically. + --- ## Data sources diff --git a/docker-compose.yml b/docker-compose.yml index ee0108a..5158bcb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,9 @@ services: duskmeter: - build: . + # Use the published image from GitHub Container Registry (default). + # Remove the `image` line and uncomment `build: .` to build locally instead. + image: ghcr.io/thelinuxguy-ssh/duskmeter:latest + # build: . ports: - "3000:3000" environment: @@ -10,11 +13,7 @@ services: - LLM_MODEL=${LLM_MODEL:-llama3} - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434} volumes: - # Mount your VIIRS files and Globe at Night cache - ./data/viirs:/app/data/viirs - ./data/gan:/app/data/gan - # Mount your existing database. If you have duskmeter-dev.sqlite from dev: - # cp duskmeter-dev.sqlite duskmeter.sqlite - # If starting fresh, this file will be created automatically. - ./duskmeter.sqlite:/app/duskmeter.sqlite restart: unless-stopped