From 47d26000c327c1daac00cbb359c24cf3f3ff38da Mon Sep 17 00:00:00 2001 From: Henry Priest Date: Mon, 3 Aug 2026 17:08:27 +0000 Subject: [PATCH] Add CLI support for example 3 (row crop) and wire its demo data Adds prepare-example-3 to workflow_manifest.yaml and magic-ensemble (event-build step argument mapping for 02a_build_events.R/03_xml_build.R), wires get-demo-data to fetch the confirmed example-3 S3 sources (input tarball, CA ERA5 archive, parcels/crops files, management event sync), repoints the shared site_era5_path/field_shape_path/pft_dir/params_from_pft manifest values at the new data (breaking 1b/2a, intentionally), fixes 02a_build_events.R's relative paths to tools/event_prep so it works when invoked via the CLI (cwd = repo root, not examples/3_rowcrop/), and removes the stale pre-CLI manual scripts (04_set_up_runs.R, 05_run_model.R) in favor of the README's CLI-only pipeline. --- examples/3_rowcrop/01_ERA5_nc_to_clim.R | 16 +- examples/3_rowcrop/02a_build_events.R | 15 +- examples/3_rowcrop/04_set_up_runs.R | 97 ---------- examples/3_rowcrop/05_run_model.R | 110 ------------ examples/3_rowcrop/README.md | 187 ++++---------------- examples/3_rowcrop/example_user_config.yaml | 37 ++++ magic-ensemble | 67 +++++-- workflow/00_fetch_s3_and_prepare_run_dir.sh | 73 ++++++++ workflow/workflow_manifest.yaml | 62 ++++++- 9 files changed, 274 insertions(+), 390 deletions(-) delete mode 100755 examples/3_rowcrop/04_set_up_runs.R delete mode 100755 examples/3_rowcrop/05_run_model.R create mode 100644 examples/3_rowcrop/example_user_config.yaml diff --git a/examples/3_rowcrop/01_ERA5_nc_to_clim.R b/examples/3_rowcrop/01_ERA5_nc_to_clim.R index 4c245f4..4805bb0 100755 --- a/examples/3_rowcrop/01_ERA5_nc_to_clim.R +++ b/examples/3_rowcrop/01_ERA5_nc_to_clim.R @@ -65,10 +65,16 @@ args <- optparse::OptionParser(option_list = options) |> future::plan(args$parallel_strategy, workers = args$n_cores) file_info <- read.csv(args$site_info_file) |> - dplyr::distinct(id) |> + dplyr::distinct(id, lat, lon) |> dplyr::mutate( start_date = args$start_date, - end_date = args$end_date + end_date = args$end_date, + # match locations to half-degree ERA5 grid cell centers + # CAUTION: Calculation only correct when all lats are N and all lons are W! + ERA5_grid_cell = paste0( + ((lat + 0.25) %/% 0.5) * 0.5, "N_", + ((abs(lon) + 0.25) %/% 0.5) * 0.5, "W" + ) ) |> dplyr::cross_join(data.frame(ens_id = 1:10)) @@ -77,16 +83,16 @@ if (!dir.exists(args$site_sipnet_met_path)) { } furrr::future_pwalk( file_info, - function(id, start_date, end_date, ens_id, ...) { + function(start_date, end_date, ens_id, ERA5_grid_cell, ...) { PEcAn.SIPNET::met2model.SIPNET( in.path = file.path( args$site_era5_path, - paste("ERA5", id, ens_id, sep = "_") + paste("ERA5", ERA5_grid_cell, ens_id, sep = "_") ), start_date = start_date, end_date = end_date, in.prefix = paste0("ERA5.", ens_id), - outfolder = file.path(args$site_sipnet_met_path, id) + outfolder = file.path(args$site_sipnet_met_path, ERA5_grid_cell) ) } ) diff --git a/examples/3_rowcrop/02a_build_events.R b/examples/3_rowcrop/02a_build_events.R index 5cf693b..6775ed4 100755 --- a/examples/3_rowcrop/02a_build_events.R +++ b/examples/3_rowcrop/02a_build_events.R @@ -53,6 +53,15 @@ args <- optparse::OptionParser(option_list = options) |> library(tidyverse) +# Locate tools/event_prep/ relative to this script's own file location, so it +# works regardless of the caller's working directory (magic-ensemble invokes +# with cwd = repo root; the README's manual workflow uses cwd = this directory). +this_file <- function() { + cmd_args <- commandArgs(trailingOnly = FALSE) + normalizePath(sub("^--file=", "", grep("^--file=", cmd_args, value = TRUE))) +} +event_prep_dir <- normalizePath(file.path(dirname(this_file()), "..", "..", "tools", "event_prep")) + # TODO these probably deserve to be runtime args, # but first better fix other version-specific assumptions below mgmt_subdirs <- list( @@ -76,7 +85,7 @@ cargs <- function(...) { if (args$raw_parquet_dir != "") { PEcAn.logger::logger.info("Cleaning irrigation files") callr::rscript( - "../../tools/event_prep/01a-clean-irrigation.R", + file.path(event_prep_dir, "01a-clean-irrigation.R"), cmdargs = cargs( irr_path = mgmt_subdirs$irri, outdir = args$clean_parquet_dir @@ -84,7 +93,7 @@ if (args$raw_parquet_dir != "") { ) PEcAn.logger::logger.info("Cleaning other management files") callr::rscript( - "../../tools/event_prep/01b-clean-other-events.R", + file.path(event_prep_dir, "01b-clean-other-events.R"), cmdargs = cargs( pheno_dir = mgmt_subdirs$pheno, planting_dir = mgmt_subdirs$plant, @@ -98,7 +107,7 @@ if (args$raw_parquet_dir != "") { PEcAn.logger::logger.info("converting management files to events") callr::rscript( - "../../tools/event_prep/02-events-to-json-and-sipnet.R", + file.path(event_prep_dir, "02-events-to-json-and-sipnet.R"), cmdargs = cargs( site_info_path = args$site_info_path, parquet_dir = args$clean_parquet_dir, diff --git a/examples/3_rowcrop/04_set_up_runs.R b/examples/3_rowcrop/04_set_up_runs.R deleted file mode 100755 index 1d3a4e9..0000000 --- a/examples/3_rowcrop/04_set_up_runs.R +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env Rscript - -# -------------------------------------------------- -# Run-time parameters - -options <- list( - optparse::make_option(c("-s", "--settings"), - default = "settings.xml", - help = paste( - "path to the XML settings file you want to use for this run.", - "Be aware all paths inside the file are interpreted relative to the", - "working directory of the process that invokes run_model.R,", - "not relative to the settings file path" - ) - ) -) |> - # Show default values in help message - purrr::modify(\(x) { - x@help <- paste(x@help, "[default: %default]") - x - }) - -args <- optparse::OptionParser(option_list = options) |> - optparse::parse_args() - - - - -# make sure always to call status.end -options(warn = 1) -options(error = quote({ - try(PEcAn.utils::status.end("ERROR")) - try(PEcAn.remote::kill.tunnel(settings)) - if (!interactive()) { - q(status = 1) - } -})) - -# ---------------------------------------------------------------------- -# PEcAn Workflow -# ---------------------------------------------------------------------- - -library("PEcAn.all") - - -# Report package versions for provenance -PEcAn.all::pecan_version() - -# Open and read in settings file for PEcAn run. -settings <- PEcAn.settings::read.settings(args$settings) - -if (!dir.exists(settings$outdir)) { - dir.create(settings$outdir, recursive = TRUE) -} -PEcAn.logger::logger.setLevel("WARN") - -PEcAn.utils::status.start("DESIGN") -ens_design <- PEcAn.uncertainty::generate_joint_ensemble_design( - settings = settings[[1]], - ensemble_size = settings$ensemble$size -) -write.csv(ens_design$X, file.path(settings$outdir, "input_design.csv")) - -# Temporary hack: -# generate_joint_ensemble_design used to write samples.Rdata as a side effect, -# but recently changed to include the samples in its return value instead. -# Passing these directly to runModule.run.write.configs and -# write_segmented_configs is still to be implemented on the PEcAn side; -# meanwhile we write them back out to disk (which is handy for post-run -# provenance too). -sample_env <- list2env(ens_design$samples) -save( - list = ls(sample_env), - envir = sample_env, - file = file.path(settings$outdir, "samples.Rdata") -) - -settings$ensemble$id <- rlang::hash(ens_design) -PEcAn.utils::status.end() - -# Write model specific configs -if (PEcAn.utils::status.check("CONFIG") == 0) { - PEcAn.utils::status.start("CONFIG") - settings <- PEcAn.workflow::runModule.run.write.configs( - settings, - input_design = ens_design$X - ) - PEcAn.settings::write.settings(settings, outputfile = "pecan.CONFIGS.xml") - PEcAn.utils::status.end() -} - -PEcAn.utils::status.start("CONFIG_SEGMENTS") -run_script_paths <- papply( - settings, - \(s) PEcAn.SIPNET::write_segmented_configs.SIPNET(s, ens_design$X) -) -PEcAn.utils::status.end() diff --git a/examples/3_rowcrop/05_run_model.R b/examples/3_rowcrop/05_run_model.R deleted file mode 100755 index 3988c0e..0000000 --- a/examples/3_rowcrop/05_run_model.R +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env Rscript - -# -------------------------------------------------- -# Run-time parameters - -options <- list( - optparse::make_option(c("-s", "--settings"), - default = "output/pecan.CONFIGS.xml", - help = paste( - "path to the XML settings file you want to use for this run.", - "Be aware all paths inside the file are interpreted relative to the", - "working directory of the process that invokes run_model.R,", - "not relative to the settings file path" - ) - ) -) |> - # Show default values in help message - purrr::modify(\(x) { - x@help <- paste(x@help, "[default: %default]") - x - }) - -args <- optparse::OptionParser(option_list = options) |> - optparse::parse_args() - - - - -# make sure always to call status.end -options(warn = 1) -options(error = quote({ - try(PEcAn.utils::status.end("ERROR")) - try(PEcAn.remote::kill.tunnel(settings)) - if (!interactive()) { - q(status = 1) - } -})) - -# ---------------------------------------------------------------------- -# PEcAn Workflow -# ---------------------------------------------------------------------- - -library("PEcAn.all") - - -# Open and read in settings file for PEcAn run. -settings <- PEcAn.settings::read.settings(args$settings) - -# Report package and model versions for provenance -PEcAn.all::pecan_version() -PEcAn.logger::logger.info(system2(settings$model$binary, "-v", stdout = TRUE)) - -# Start ecosystem model runs -if (PEcAn.utils::status.check("MODEL") == 0) { - PEcAn.utils::status.start("MODEL") - stop_on_error <- as.logical(settings[[c("run", "stop_on_error")]]) - if (length(stop_on_error) == 0) { - # If we're doing an ensemble run, don't stop. If only a single run, we - # should be stopping. - if (is.null(settings[["ensemble"]]) || - as.numeric(settings[[c("ensemble", "size")]]) == 1) { - stop_on_error <- TRUE - } else { - stop_on_error <- FALSE - } - } - PEcAn.workflow::runModule_start_model_runs(settings, - stop.on.error = stop_on_error) - PEcAn.utils::status.end() -} - - -# Get results of model runs -# this function is arguably too chatty, so we'll suppress -# INFO-level log output for this step. -loglevel <- PEcAn.logger::logger.setLevel("WARN") -if (PEcAn.utils::status.check("OUTPUT") == 0) { - PEcAn.utils::status.start("OUTPUT") - runModule.get.results(settings) - PEcAn.utils::status.end() -} -PEcAn.logger::logger.setLevel(loglevel) - - -# Run ensemble analysis on model output. -if ("ensemble" %in% names(settings) - && PEcAn.utils::status.check("ENSEMBLE") == 0) { - PEcAn.utils::status.start("ENSEMBLE") - runModule.run.ensemble.analysis(settings, TRUE) - PEcAn.utils::status.end() -} - - -# Run sensitivity analysis and variance decomposition on model output -if ("sensitivity.analysis" %in% names(settings) && - PEcAn.utils::status.check("SENSITIVITY") == 0) { - PEcAn.utils::status.start("SENSITIVITY") - runModule.run.sensitivity.analysis(settings) - PEcAn.utils::status.end() -} - -# Pecan workflow complete -if (PEcAn.utils::status.check("FINISHED") == 0) { - PEcAn.utils::status.start("FINISHED") - PEcAn.remote::kill.tunnel(settings) - - PEcAn.utils::status.end() -} - -print("---------- PEcAn Workflow Complete ----------") diff --git a/examples/3_rowcrop/README.md b/examples/3_rowcrop/README.md index 625a614..7d9d2b0 100644 --- a/examples/3_rowcrop/README.md +++ b/examples/3_rowcrop/README.md @@ -25,31 +25,36 @@ direct observations of soil carbon and/or biomass are available during the simulation period. -## Caveats - -Instructions assume a local run on MacOS and will be updated for a -Linux + Slurm + Apptainer HPC environment as we finish testing and deployment. - -Aspirationally, any command prefixed with `[host_args]` is one that ought to -work on HPC by "just" adding a system-specific prefix, e.g. -`./01_ERA5_nc_to_clim.R --start_date=2016-01-01` on my machine becomes -`sbatch -n16 --mem=12G --mail-type=ALL --uid=jdoe \ - ./01_ERA5_nc_to_clim.R --start_date=2016-01-01` on yours. - - ## Running the workflow -### 0. Copy prebuilt artifacts and set up validation data +This example is driven by `./magic-ensemble` from the repo root. Copy +`example_user_config.yaml`, edit `run_dir` and any other overrides you need, +then run: ```sh -export AWS_PROFILE=magic -aws s3 sync --exclude='mslsp/*' s3://carb/management/ ./data_raw/management -aws s3 sync s3://carb/data_raw/ERA5_CA_nc ./data_raw/ERA5_CA_nc -aws s3 cp s3://carb/data/workflows/phase_3/magic_example3_input_data_20260711.tgz . -tar xf magic_example3_input_data_20260711.tgz +./magic-ensemble get-demo-data --verbose --config +./magic-ensemble prepare-example-3 --verbose --config +./magic-ensemble run-ensembles --verbose --config ``` -#### Validation data +* `get-demo-data` fetches all confirmed S3 sources into `run_dir`: the + example-3 input tarball (site_info.csv, PFT posteriors, IC-prep caches), the + CA-specific ERA5 archive, `parcels-consolidated.gpkg`/`crops_all_years.parq`, + and the management event sources (harvest/irrigation/phenology/planting/tillage). +* `prepare-example-3` runs, in order: climate-driver conversion + (`01_ERA5_nc_to_clim.R`), the canonical IC build (`workflow/02_ic_build.R`), + management-event generation (`02a_build_events.R`), and settings assembly + (`03_xml_build.R`), producing `settings.xml` in `run_dir`. +* `run-ensembles` dispatches ensemble members (Slurm or local, per + `pecan_dispatch` in your config). + +`site_info.csv` is not checked into this repo -- it ships pre-built inside the +get-demo-data tarball. It was generated once via +`tools/build_site_info.R --location_file= --out_file=site_info.csv` +against `parcels-consolidated.gpkg`/`crops_all_years.parq`; only rerun that +script if you need to change the set of design points. + +### Validation data To set up validation runs, you need access to the cropland soil carbon data files `Harmonized_SiteMngmt_Croplands.csv` and `Harmonized_Data_Croplands.csv`. @@ -60,148 +65,16 @@ consented to use of their data for internal research purposes but explicitly did not consent to public distribution of the data. Contact chelsea.carey@arb.ca.gov for more information about the dataset. -Once obtained, place them in `data_raw/private/HSP` and run +Once obtained, place them in `data_raw/private/HSP` (inside `run_dir`) and run ```{sh} ../../tools/build_validation_siteinfo.R ``` -to create `validation_site_info.csv`. - - -### 1. Convert climate driver files - -TODO: show how to pass n_cores from host_args -(NSLOTS? SLURM_CPUS_PER_TASK?) - -```{sh} -[host_args] ./01_ERA5_nc_to_clim.R \ - --site_era5_path=data_raw/ERA5_CA_nc \ - --site_sipnet_met_path=data/ERA5_CA_SIPNET \ - --site_info_file=data_raw/ERA5_CA_nc/ca_half_degree_grid.csv \ - --start_date=2016-01-01 \ - --end_date=2023-12-31 \ - --n_cores=7 -``` - -### 2. Generate initial site conditions - -We'll run this twice, once for validation sites and once for statewide anchors. -It would also be fine to put both together in the same input and run it once. - -NOTE: ECMWF soil moisture data calls were failing when I tried to run this for anchor sites on 2025-12-08, -so I symlinked `data/IC_prep_val/soil_moisture/` to `data/IC_prep/soil_moisture/`. On a day the server is up, this _should_ not be needed... but also isn't a problem, since that subdirectory contains global 0.25 degree/25 km soil moisture data for the first 10 days of 2016 and can be expected to be identical from one downloading to the next. The fact that we cache that output here is a quirk of how `PEcAn.data.land::extract_SM_CDS` is implemented, not a designed part of the IC workflow. - - -```{sh} -[host_args] ../../workflow/02_ic_build.R \ - --site_info_path=validation_site_info.csv \ - --pft_dir=data_raw/pfts \ - --data_dir=data/IC_prep_val \ - --ic_outdir=data/IC_files - -../../tools/build_site_info.R --location_file=../../data/design_points.csv - -[host_args] ../../workflow/02_ic_build.R \ - --site_info_path=site_info.csv \ - --pft_dir=data_raw/pfts \ - --data_dir=data/IC_prep \ - --ic_outdir=data/IC_files -``` - -### 2a. Generate event files - -Management events are read from files produced by the monitoring pipeline or equivalent sources. -The current version assumes all event types are provided as Parquet files, and that they live in subdirectories of `mgmt_file_dir` that include specific, currently hardcoded, version numbers. See script for the details. -Future versions of the event_build script may support passing separate paths per data product and potentially also add support for alternate storage formats (e.g. allowing csv as well as parquet). - -```{sh} -[host_args] ./02a_build_events.R \ - --site_info_path=validation_site_info.csv \ - --raw_parquet_dir=data_raw/management \ - --event_outdir=data/val_events -# empty raw_parquet_dir to avoid redoing cleaning already done for val -# (Cleaning runs on whole statewide file, not just selected sites) -[host_args] ./02a_build_events.R \ - --site_info_path=site_info.csv \ - --raw_parquet_dir='' \ - --event_outdir=data/events -``` - -For validation we need an additional hack: -`02a_build_events.R` created event files named by their *parcel* id, -but the validation dataset uses a separate set of *site* ids derived by hashing locations, experiment names, and treatment codes. -This is done to (1) keep locations opaque since the validation data are nonpublic, and (2) allow simulation of separate plots (treatments) whose locations all fall in a single parcel of the statewide map. -A future solution would be to teach `03_xml_build.R` how to find event files that are named by parcel id, so that sites which share a parcel do not need to duplicate the file. -For now though, let's duplicate the files of interest so they're named after the site IDs PEcAn will use. For now, I'm not cleaning up the originals afterwards -- they're small and might be used by other runs that reuse this data directory. - -```{r} -vsi <- read.csv("validation_site_info.csv") |> - dplyr::distinct(id, field_id) |> - dplyr::rename(site_id = id) -vsi |> purrr::pwalk( - \(site_id, field_id) file.copy( - paste0("data/val_events/events-", field_id, ".in"), - paste0("data/val_events/events-", site_id, ".in") - ) -) - -# Rename sites inside JSON file, so restart code can match it for PFT changes -# Doing this by pure substitution, not parsing anything -evt_json_txt <- readLines("data/val_events/combined_events.json") -for (i in seq_along(vsi$site_id)) { - evt_json_txt <- gsub( - pattern = paste0('"site_id":"', vsi$field_id[[i]], '"'), - replacement = paste0('"site_id":"', vsi$site_id[[i]], '"'), - x = evt_json_txt - ) -} -writeLines(evt_json_txt, "data/val_events/combined_events.json") - -# And rename inside the phenology file, too -read.csv("data/val_events/phenology.csv") |> - dplyr::rename(field_id = site_id) |> - dplyr::left_join(vsi) |> - write.csv("data/val_events/phenology.csv", row.names = FALSE) -``` - - -### 3. Generate settings file - -```{sh} -[host_args] ./03_xml_build.R \ - --end_date=2023-12-31 \ - --ic_dir=data/IC_files \ - --site_file=validation_site_info.csv \ - --event_dir=data/val_events \ - --output_file=validation_settings.xml \ - --output_dir=val_out -[host_args] ./03_xml_build.R \ - --ic_dir=data/IC_files \ - --end_date=2023-12-31 \ - --site_file=site_info.csv \ - --output_file=settings.xml \ - --output_dir=output -``` - -### 4. Set up model run directories - -TODO: Yes, it's unintuitive that we can't rename the output dir at this -stage instead of in xml_build. - -```{sh} -[host_args] ./04_set_up_runs.R --settings=validation_settings.xml -[host_args] ./04_set_up_runs.R --settings=settings.xml -``` - -### 5. Run model - -```{sh} -export NCPUS=8 -ln -s [your/path/to]/sipnet/sipnet sipnet -[host_args] ./05_run_model.R --settings=val_out/pecan.CONFIGS.xml -[host_args] ./05_run_model.R --settings=output/pecan.CONFIGS.xml -``` +to create `validation_site_info.csv`. Validation runs are not yet wired into +`magic-ensemble` -- rerun the relevant `prepare-example-3` steps by hand with +`--site_file=validation_site_info.csv` and a separate `--output_file`/ +`--output_dir`, following the same pattern `03_xml_build.R --help` documents. -### 6. Validate +### Validate ```{sh} [host_args] ./validate.R \ diff --git a/examples/3_rowcrop/example_user_config.yaml b/examples/3_rowcrop/example_user_config.yaml new file mode 100644 index 0000000..17a3df3 --- /dev/null +++ b/examples/3_rowcrop/example_user_config.yaml @@ -0,0 +1,37 @@ +# Example user config for the 3 row crop workflow. +# Copy this file and edit before running. Only run_dir and pecan_dispatch are required; +# all other keys fall back to the defaults shown. +# +# Unlike 1b/2a, site_info.csv is not staged from this repo -- it ships inside the +# get-demo-data tarball (magic_example3_input_data_20260714.tgz) and lands directly +# in run_dir. It was generated once via tools/build_site_info.R against the phase-3 +# design points; only regenerate it if you need a different set of design points. +# +# Run with: +# ./magic-ensemble get-demo-data --config +# ./magic-ensemble prepare-example-3 --config +# ./magic-ensemble run-ensembles --config + +run_dir: "3-rowcrop-run-directory/" + +start_date: "2016-01-01" +end_date: "2023-12-31" +run_LAI_date: "2016-07-01" +n_ens: 20 +n_met: 10 +ic_ensemble_size: 100 +# Number of management-event ensemble members available in event_dir (set by +# prepare-example-3's event-build step; only override if you regenerate events +# with a different ensemble size). +n_event: 20 +n_workers: 1 +use_apptainer: false +pecan_dispatch: slurm-dispatch +# slurm_partition: "" # Slurm partition for sbatch jobs (slurm-dispatch only). Omit to use the cluster default. + +# AWS CLI named profile for S3 access (get-demo-data command). +aws_profile: magic + +# Stage example-specific input files into run_dir before prepare runs. +external_paths: + template_file: "examples/3_rowcrop/template.xml" diff --git a/magic-ensemble b/magic-ensemble index be1df77..c3dbf92 100755 --- a/magic-ensemble +++ b/magic-ensemble @@ -1,7 +1,7 @@ #!/usr/bin/env bash # magic-ensemble: minimal CLI for CCMMF ensemble workflows. # Usage: ./magic-ensemble [--verbose] --config -# Commands: help | get-demo-data | prepare | prepare-example-1b | prepare-example-2a | run-ensembles +# Commands: help | get-demo-data | prepare | prepare-example-1b | prepare-example-2a | prepare-example-3 | run-ensembles set -euo pipefail @@ -21,6 +21,7 @@ Commands: prepare Run canonical preparation steps using workflow/ scripts. prepare-example-1b Run preparation steps using the 1b statewide woody crops example scripts. prepare-example-2a Run preparation steps using the 2a grass example scripts. + prepare-example-3 Run preparation steps using the 3 row crop example scripts (adds a management-events step). run-ensembles Run step 04 (run model) using existing settings.xml and prepared inputs. Global options (after command): @@ -34,6 +35,7 @@ Examples: ./magic-ensemble prepare --config my_config.yaml --verbose ./magic-ensemble prepare-example-2a --config examples/2a_grass/example_user_config.yaml ./magic-ensemble prepare-example-1b --config examples/1b_statewide_woody/example_user_config.yaml + ./magic-ensemble prepare-example-3 --config examples/3_rowcrop/example_user_config.yaml ./magic-ensemble run-ensembles --config my_config.yaml EOF } @@ -56,7 +58,7 @@ VERBOSE=0 CONFIG_FILE="" while [[ $# -gt 0 ]]; do case "$1" in - help|get-demo-data|prepare|prepare-example-1b|prepare-example-2a|run-ensembles) + help|get-demo-data|prepare|prepare-example-1b|prepare-example-2a|prepare-example-3|run-ensembles) if [[ -z "$COMMAND" ]]; then COMMAND="$1"; shift; continue; fi ;; --verbose) VERBOSE=1; shift; continue ;; @@ -86,7 +88,7 @@ if [[ -z "$COMMAND" || "$COMMAND" == "help" ]]; then exit 0 fi -if [[ "$COMMAND" != "get-demo-data" && "$COMMAND" != "prepare" && "$COMMAND" != "prepare-example-1b" && "$COMMAND" != "prepare-example-2a" && "$COMMAND" != "run-ensembles" ]]; then +if [[ "$COMMAND" != "get-demo-data" && "$COMMAND" != "prepare" && "$COMMAND" != "prepare-example-1b" && "$COMMAND" != "prepare-example-2a" && "$COMMAND" != "prepare-example-3" && "$COMMAND" != "run-ensembles" ]]; then echo "magic-ensemble: Unknown command: $COMMAND" >&2 usage >&2 exit 1 @@ -141,6 +143,9 @@ p_output_file=$(yq eval '.paths.output_file' "$MANIFEST") p_met_dir=$(yq eval '.paths.met_dir' "$MANIFEST") p_ic_dir=$(yq eval '.paths.ic_dir' "$MANIFEST") p_settings_xml=$(yq eval '.paths.settings_xml' "$MANIFEST") +p_management_raw_dir=$(yq eval '.paths.management_raw_dir' "$MANIFEST") +p_management_clean_dir=$(yq eval '.paths.management_clean_dir' "$MANIFEST") +p_event_dir=$(yq eval '.paths.event_dir' "$MANIFEST") params_from_pft=$(yq eval '.params_from_pft' "$MANIFEST") additional_params=$(yq eval '.additional_params' "$MANIFEST") @@ -152,6 +157,7 @@ run_LAI_date_default="2016-07-01" n_ens_default="20" n_met_default="10" ic_ensemble_size_default="100" +n_event_default="20" n_workers_default="1" use_apptainer_default="false" @@ -180,6 +186,7 @@ run_LAI_date=$(get_val "run_LAI_date" "$run_LAI_date_default") n_ens=$(get_val "n_ens" "$n_ens_default") n_met=$(get_val "n_met" "$n_met_default") ic_ensemble_size=$(get_val "ic_ensemble_size" "$ic_ensemble_size_default") +n_event=$(get_val "n_event" "$n_event_default") n_workers=$(get_val "n_workers" "$n_workers_default") use_apptainer_raw=$(get_val "use_apptainer" "$use_apptainer_default") # Normalize: true/yes/1 (case-insensitive) => 1; else 0 @@ -210,6 +217,9 @@ output_file=$(resolve_path "$p_output_file") met_dir=$(resolve_path "$p_met_dir") ic_dir=$(resolve_path "$p_ic_dir") settings_xml=$(resolve_path "$p_settings_xml") +management_raw_dir=$(resolve_path "$p_management_raw_dir") +management_clean_dir=$(resolve_path "$p_management_clean_dir") +event_dir=$(resolve_path "$p_event_dir") # landtrendr_raw_files is comma-separated; resolve each segment (manifest only for now) landtrendr_raw_files="" while IFS= read -r segment; do @@ -544,16 +554,46 @@ run_prepare() { --params_read_from_pft "$params_from_pft" \ --landtrendr_raw_files "$landtrendr_raw_files" \ --additional_params "$additional_params" ;; - 3) run_script $apptainer_arg "$script" \ - --n_ens "$n_ens" \ - --n_met "$n_met" \ - --start_date "$start_date" \ - --end_date "$end_date" \ - --ic_dir "$ic_dir" \ - --met_dir "$met_dir" \ - --site_file "$site_file" \ - --template_file "$template_file" \ - --output_file "$output_file" ;; + 3) + if [[ "$COMMAND" == "prepare-example-3" ]]; then + run_script $apptainer_arg "$script" \ + --site_info_path "$site_info_file" \ + --raw_parquet_dir "$management_raw_dir" \ + --clean_parquet_dir "$management_clean_dir" \ + --event_outdir "$event_dir" \ + --start_date "$start_date" \ + --end_date "$end_date" + else + run_script $apptainer_arg "$script" \ + --n_ens "$n_ens" \ + --n_met "$n_met" \ + --start_date "$start_date" \ + --end_date "$end_date" \ + --ic_dir "$ic_dir" \ + --met_dir "$met_dir" \ + --site_file "$site_file" \ + --template_file "$template_file" \ + --output_file "$output_file" + fi ;; + 4) + if [[ "$COMMAND" == "prepare-example-3" ]]; then + run_script $apptainer_arg "$script" \ + --n_ens "$n_ens" \ + --n_met "$n_met" \ + --n_ic "$ic_ensemble_size" \ + --n_event "$n_event" \ + --start_date "$start_date" \ + --end_date "$end_date" \ + --ic_dir "$ic_dir" \ + --met_dir "$met_dir" \ + --event_dir "$event_dir" \ + --site_file "$site_file" \ + --template_file "$template_file" \ + --output_file "$output_file" \ + --output_dir "output" + else + echo "magic-ensemble: No argument mapping for prepare step index $i (script $script)" >&2; exit 1 + fi ;; *) echo "magic-ensemble: No argument mapping for prepare step index $i (script $script)" >&2; exit 1 ;; esac fi @@ -586,6 +626,7 @@ case "$COMMAND" in prepare) run_prepare ;; prepare-example-1b) run_prepare ;; prepare-example-2a) run_prepare ;; + prepare-example-3) run_prepare ;; run-ensembles) run_run_ensembles ;; *) echo "magic-ensemble: Unknown command: $COMMAND" >&2; exit 1 ;; esac diff --git a/workflow/00_fetch_s3_and_prepare_run_dir.sh b/workflow/00_fetch_s3_and_prepare_run_dir.sh index 5ff9410..3f79555 100755 --- a/workflow/00_fetch_s3_and_prepare_run_dir.sh +++ b/workflow/00_fetch_s3_and_prepare_run_dir.sh @@ -230,4 +230,77 @@ fi echo "00_fetch_s3_and_prepare_run_dir: Extracting soil moisture data into run directory" tar -xzf "$sm_local" -C "$RUN_DIR_ABS" +# --- Example 3 (row crop): download example-3 input tarball and extract --- +ex3_key_prefix=$(yq eval '.s3.example3_input_data_tgz.key_prefix' "$MANIFEST") +ex3_filename=$(yq eval '.s3.example3_input_data_tgz.filename' "$MANIFEST") +ex3_s3_key=$(s3_key "$ex3_key_prefix" "$ex3_filename") +ex3_s3_uri="s3://${s3_bucket}/${ex3_s3_key}" +ex3_local="${RUN_DIR_ABS}/${ex3_filename}" +ex3_report=$(report_path "$ex3_local") +if [[ -f "$ex3_local" ]]; then + echo "00_fetch_s3_and_prepare_run_dir: Example-3 input tarball already present: $ex3_report" +else + echo "00_fetch_s3_and_prepare_run_dir: Downloading example-3 input tarball from S3" + echo "00_fetch_s3_and_prepare_run_dir: Saving to: $ex3_report" + (cd "$RUN_DIR_ABS" && aws s3 cp --profile "$AWS_PROFILE" --endpoint-url "$s3_endpoint" "$ex3_s3_uri" "$ex3_filename") +fi +echo "00_fetch_s3_and_prepare_run_dir: Extracting example-3 input tarball into run directory" +tar -xzf "$ex3_local" -C "$RUN_DIR_ABS" + +# --- Example 3 (row crop): download CA-specific ERA5 archive and extract into data_raw/ --- +era5_ca_key_prefix=$(yq eval '.s3.era5_ca_nc_tgz.key_prefix' "$MANIFEST") +era5_ca_filename=$(yq eval '.s3.era5_ca_nc_tgz.filename' "$MANIFEST") +era5_ca_s3_key=$(s3_key "$era5_ca_key_prefix" "$era5_ca_filename") +era5_ca_s3_uri="s3://${s3_bucket}/${era5_ca_s3_key}" +era5_ca_local="${RUN_DIR_ABS}/${era5_ca_filename}" +era5_ca_report=$(report_path "$era5_ca_local") +if [[ -f "$era5_ca_local" ]]; then + echo "00_fetch_s3_and_prepare_run_dir: ERA5 CA archive already present: $era5_ca_report" +else + echo "00_fetch_s3_and_prepare_run_dir: Downloading ERA5 CA archive from S3" + echo "00_fetch_s3_and_prepare_run_dir: Saving to: $era5_ca_report" + (cd "$RUN_DIR_ABS" && aws s3 cp --profile "$AWS_PROFILE" --endpoint-url "$s3_endpoint" "$era5_ca_s3_uri" "$era5_ca_filename") +fi +echo "00_fetch_s3_and_prepare_run_dir: Extracting ERA5 CA archive into run directory" +mkdir -p "${RUN_DIR_ABS}/data_raw" +tar -xzf "$era5_ca_local" -C "${RUN_DIR_ABS}/data_raw" + +# --- Example 3 (row crop): download parcels-consolidated.gpkg and crops_all_years.parq --- +# NOTE: do not grab the sibling `parcels.gpkg` at the same prefix -- similarly +# named/sized but the wrong file. build_site_info.R and 02_ic_build.R both +# default to parcels-consolidated.gpkg specifically. +crops_v41_dir="${RUN_DIR_ABS}/data_raw/management/crops/v4.1" +mkdir -p "$crops_v41_dir" + +download_single_file() { + local manifest_key="$1" label="$2" + local key_prefix filename s3_key_val s3_uri local_path + key_prefix=$(yq eval ".s3.${manifest_key}.key_prefix" "$MANIFEST") + filename=$(yq eval ".s3.${manifest_key}.filename" "$MANIFEST") + s3_key_val=$(s3_key "$key_prefix" "$filename") + s3_uri="s3://${s3_bucket}/${s3_key_val}" + local_path="${crops_v41_dir}/${filename}" + if [[ -f "$local_path" ]]; then + echo "00_fetch_s3_and_prepare_run_dir: ${label} already present: $(report_path "$local_path")" + else + echo "00_fetch_s3_and_prepare_run_dir: Downloading ${label} from S3" + echo "00_fetch_s3_and_prepare_run_dir: Saving to: $(report_path "$local_path")" + (cd "$crops_v41_dir" && aws s3 cp --profile "$AWS_PROFILE" --endpoint-url "$s3_endpoint" "$s3_uri" "$filename") + fi +} +download_single_file "parcels_gpkg" "parcels-consolidated.gpkg" +download_single_file "crops_all_years_parq" "crops_all_years.parq" + +# --- Example 3 (row crop): sync management event sources (harvest/irrigation/phenology/planting/tillage) --- +# Excludes mslsp/ (unused) and crops/ (parcels-consolidated.gpkg/crops_all_years.parq handled above). +mgmt_events_key_prefix=$(yq eval '.s3.management_events.key_prefix' "$MANIFEST") +mgmt_events_s3_uri="s3://${s3_bucket}/${mgmt_events_key_prefix}/" +mgmt_raw_dir_value=$(yq eval '.paths.management_raw_dir' "$MANIFEST") +mgmt_raw_dir=$(resolve_run_path "$mgmt_raw_dir_value") +mkdir -p "$mgmt_raw_dir" +echo "00_fetch_s3_and_prepare_run_dir: Syncing management event sources from S3 into $(report_path "$mgmt_raw_dir")" +aws s3 sync --profile "$AWS_PROFILE" --endpoint-url "$s3_endpoint" \ + --exclude 'mslsp/*' --exclude 'crops/*' \ + "$mgmt_events_s3_uri" "$mgmt_raw_dir" + echo "00_fetch_s3_and_prepare_run_dir: Done." diff --git a/workflow/workflow_manifest.yaml b/workflow/workflow_manifest.yaml index 31ba121..2341c54 100644 --- a/workflow/workflow_manifest.yaml +++ b/workflow/workflow_manifest.yaml @@ -19,7 +19,7 @@ steps: - script: "workflow/00_fetch_s3_and_prepare_run_dir.sh" r_libraries: [] inputs: [] - outputs: [data_dir, ic_outdir, site_sipnet_met_path] + outputs: [data_dir, ic_outdir, site_sipnet_met_path, management_raw_dir] prepare: - script: "workflow/00_stage_external_inputs.sh" @@ -105,6 +105,39 @@ steps: inputs: [site_file, template_file, ic_dir, met_dir] outputs: [output_file] + prepare-example-3: + - script: "workflow/00_stage_external_inputs.sh" + r_libraries: [] + inputs: [] + outputs: [] + + - script: "examples/3_rowcrop/01_ERA5_nc_to_clim.R" + r_libraries: [future, furrr] + inputs: [site_info_file, site_era5_path] + outputs: [site_sipnet_met_path] + + - script: "workflow/02_ic_build.R" + r_libraries: [tidyverse] + inputs: + [ + site_info_file, + field_shape_path, + pft_dir, + data_dir, + landtrendr_raw_files, + ] + outputs: [ic_outdir, data_dir] + + - script: "examples/3_rowcrop/02a_build_events.R" + r_libraries: [tidyverse, callr] + inputs: [site_info_file, management_raw_dir] + outputs: [management_clean_dir, event_dir] + + - script: "examples/3_rowcrop/03_xml_build.R" + r_libraries: [PEcAn.settings] + inputs: [site_file, template_file, ic_dir, met_dir, event_dir] + outputs: [output_file] + run-ensembles: - script: "workflow/04_run_model.R" r_libraries: [PEcAn.all] @@ -118,11 +151,11 @@ steps: paths: site_info_file: "site_info.csv" site_sipnet_met_path: "data/ERA5_SIPNET" - site_era5_path: "data_raw/ERA5_nc" - field_shape_path: "data_raw/dwr_map/i15_Crop_Mapping_2018.gdb" + site_era5_path: "data_raw/ERA5_CA_nc" + field_shape_path: "data_raw/management/crops/v4.1/parcels-consolidated.gpkg" data_dir: "data/IC_prep" ic_outdir: "IC_files" - pft_dir: "pfts" + pft_dir: "data_raw/pfts" landtrendr_raw_files: "data_raw/ca_biomassfiaald_2016_median.tif,data_raw/ca_biomassfiaald_2016_stdv.tif" site_file: "site_info.csv" template_file: "template.xml" @@ -130,9 +163,12 @@ paths: met_dir: "data/ERA5_SIPNET" ic_dir: "IC_files" settings_xml: "settings.xml" + management_raw_dir: "data_raw/management" + management_clean_dir: "data/management_ensembles" + event_dir: "data/events" # Fixed workflow values (not user overrides) -params_from_pft: "SLA,leafC" +params_from_pft: "SLA,leafC,leafGrowth" additional_params: "varname=wood_carbon_fraction,distn=norm,parama=0.48,paramb=0.005" # S3 resources (not in user config). Remote resources are localized before R runs. @@ -152,6 +188,22 @@ s3: soil_moisture_tgz: key_prefix: "data_raw" filename: "moisture_20160101_20160110.tgz" + example3_input_data_tgz: + key_prefix: "data/workflows/phase_3" + filename: "magic_example3_input_data_20260714.tgz" + era5_ca_nc_tgz: + key_prefix: "data_raw" + filename: "ERA5_CA_nc_2016_2024.tgz" + parcels_gpkg: + key_prefix: "management/crops/v4.1" + filename: "parcels-consolidated.gpkg" + crops_all_years_parq: + key_prefix: "management/crops/v4.1" + filename: "crops_all_years.parq" + management_events: + # Synced (not cp'd) directly into paths.management_raw_dir; excludes + # mslsp/ (unused) and crops/ (covered by parcels_gpkg/crops_all_years_parq above). + key_prefix: "management" # Dispatch options for run-ensembles. The user config selects one by name via pecan_dispatch. # host_xml is the complete ... block to inject into the staged template.xml