diff --git a/Makefile b/Makefile index 79cfaf6c..73e34370 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ deploy: orchestration questions metadata resolve leaderboards curate-questions w questions: manifold metaculus acled infer kalshi yfinance polymarket wikipedia fred dbnomics -orchestration: nightly-worker-job nightly-manager-job compress_buckets +orchestration: nightly-worker-job nightly-manager-job compress_buckets push-datasets-to-git metadata: tag-questions validate-questions @@ -248,6 +248,9 @@ nightly-worker-job: nightly-manager-job: $(MAKE) -C src/nightly_update_workflow/manager || echo "* $@" >> $(MAKE_FAILURE_LOG) +push-datasets-to-git: + $(MAKE) -C src/orchestration/func_push_datasets_to_git || echo "* $@" >> $(MAKE_FAILURE_LOG) + llm-forecaster: llm-forecaster-manager llm-forecaster-worker llm-forecaster-manager: diff --git a/src/base_eval/naive_and_dummy_forecasters/requirements.txt b/src/base_eval/naive_and_dummy_forecasters/requirements.txt index af107f8f..51069f2c 100644 --- a/src/base_eval/naive_and_dummy_forecasters/requirements.txt +++ b/src/base_eval/naive_and_dummy_forecasters/requirements.txt @@ -10,3 +10,4 @@ cmdstanpy scipy termcolor pandera +slack_sdk diff --git a/src/curate_questions/publish_question_set/requirements.txt b/src/curate_questions/publish_question_set/requirements.txt index 70ea3271..db05c11c 100644 --- a/src/curate_questions/publish_question_set/requirements.txt +++ b/src/curate_questions/publish_question_set/requirements.txt @@ -6,3 +6,4 @@ tqdm scipy pandera termcolor +slack_sdk diff --git a/src/helpers/git.py b/src/helpers/git.py index d6d8b6f3..34e11fab 100644 --- a/src/helpers/git.py +++ b/src/helpers/git.py @@ -9,20 +9,20 @@ from git import Actor, Repo -from . import constants, env, keys +from . import constants, keys, slack logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -def clone(repo_url: str) -> Tuple[Repo, str, str]: +def clone(repo_url: str) -> Optional[Tuple[Repo, str, str]]: """Clone a Git repository into a temporary directory with a temporary SSH key. Args: repo_url (str): The SSH URL of the repository to clone. Returns: - Tuple[Repo, str, str]: + Optional[Tuple[Repo, str, str]]: None if the SSH key secret is not set, otherwise: - repo (Repo): The cloned GitPython Repo object. - local_repo_dir (str): The temporary directory where the repository was cloned. - tmp_key_file_path (str): Path to the temporary SSH private key used for cloning. @@ -57,7 +57,7 @@ def clone_and_push_files( files: Dict[str, str], commit_message: str, mirrors: Optional[List[str]] = None, -) -> None: +) -> bool: """Clone a Git repository, add/update files, commit, and push to origin and optional mirrors. Args: @@ -68,13 +68,19 @@ def clone_and_push_files( If None, attempts to load from secrets. Returns: - None. Exits with status 1 if an error is encountered while pushing. + bool: True if a new commit was created, False if HEAD was unchanged or if the SSH key + secret is not set, in which case nothing is cloned or pushed. Origin and mirrors + are pushed either way. Exits with status 1 if the push to origin fails; a failed + mirror push is logged and sent to Slack as a warning. """ if not mirrors: mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL") mirrors = [mirrors] if mirrors else [] - repo, local_repo_dir, tmp_key_file_path = clone(repo_url=repo_url) + cloned = clone(repo_url=repo_url) + if cloned is None: + return False + repo, local_repo_dir, tmp_key_file_path = cloned for source, destination in files.items(): full_destination_path = f"{local_repo_dir}/{destination}" @@ -84,50 +90,47 @@ def clone_and_push_files( shutil.copy(source, full_destination_path, follow_symlinks=False) repo.index.add([destination]) + # Skip empty commits, but always push so a lagging mirror catches up. + has_changes = bool(repo.index.diff(repo.head.commit)) + if not has_changes: + logger.info(f"No new commit for {repo_url}; syncing remotes to HEAD.") + error_encountered = False author = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL) committer = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL) ssh_env = {"GIT_SSH_COMMAND": f"ssh -i {tmp_key_file_path} -o StrictHostKeyChecking=no"} try: - repo.index.commit(commit_message, author=author, committer=committer) + if has_changes: + repo.index.commit(commit_message, author=author, committer=committer) origin = repo.remote(name="origin") - origin.push(env=ssh_env) - for index, mirror_url in enumerate(mirrors): - mirror = repo.create_remote(f"mirror_{index}", url=mirror_url) - mirror.push(env=ssh_env) - repo.delete_remote(mirror.name) - logger.info(f"Pushed to {mirror_url} (mirror) with commit message: {commit_message}") + # A rejected push does not raise in GitPython; surface it explicitly. + origin.push(env=ssh_env).raise_if_error() except Exception as e: error_encountered = True message = e.message if hasattr(e, "message") else str(e) logger.error(f"encountered error when pushing to git: {message}") + # Mirrors are convenience copies; the repo at `repo_url` is the source of truth. A mirror + # that's down or has diverged must not fail the nightly run, so warn on Slack and carry on. + if not error_encountered: + for index, mirror_url in enumerate(mirrors): + try: + mirror = repo.create_remote(f"mirror_{index}", url=mirror_url) + mirror.push(env=ssh_env).raise_if_error() + repo.delete_remote(mirror.name) + logger.info(f"Pushed to {mirror_url} (mirror)") + except Exception as e: + error = e.message if hasattr(e, "message") else str(e) + message = f"Could not push to {mirror_url} (mirror): {error}" + logger.warning(message) + slack.send_message(message=f"*MIRROR PUSH FAILED*\n{message}") + os.remove(tmp_key_file_path) shutil.rmtree(local_repo_dir, ignore_errors=True) if error_encountered: sys.exit(1) - logger.info(f"Pushed to {repo_url} with commit message: {commit_message}") - - -def clone_commit_and_push( - files: Dict[str, str], - commit_message: str, -) -> None: - """Upload files files to Cloud Storage and push updates to Git. - - Args: - files (Dict[str, str]): Mapping of local file paths to their git location. - - Returns: - None - """ - if env.RUNNING_LOCALLY: - return - - clone_and_push_files( - repo_url=keys.API_GITHUB_DATASET_REPO_URL, - files=files, - commit_message=commit_message, - ) + if has_changes: + logger.info(f"Pushed to {repo_url} with commit message: {commit_message}") + return has_changes diff --git a/src/leaderboard/main.py b/src/leaderboard/main.py index 97f9ebd9..8015d1d2 100644 --- a/src/leaderboard/main.py +++ b/src/leaderboard/main.py @@ -32,7 +32,6 @@ dates, decorator, env, - git, resolution, slack, ) @@ -752,7 +751,7 @@ def write_question_fixed_effects( None: Concatenated DataFrame is created (and can be written or processed further inside the function). """ - logger.info(colored("Writing question fixed effects to WEBSITE.", "yellow")) + logger.info(colored("Writing question fixed effects.", "yellow")) dfs = [] for question_type, df_qfe in question_fixed_effects.items(): @@ -779,11 +778,12 @@ def write_question_fixed_effects( ] directory = data_utils.get_mounted_bucket(bucket=env.PUBLIC_RELEASE_BUCKET) - iso_date = dates.get_date_today_as_iso() leaderboard_file_stem = LEADERBOARD_FILE_STEMS[leaderboard_type] + # One file per leaderboard, overwritten nightly. The history lives in the dataset repo, + # where `push_datasets_to_git` sends this file every night. local_filename = ( f"{directory}/question-fixed-effects/" - f"question_fixed_effects.{iso_date}.{leaderboard_file_stem}.json" + f"question_fixed_effects.{leaderboard_file_stem}.json" ) os.makedirs(os.path.dirname(local_filename), exist_ok=True) df.to_json(local_filename, orient="records") @@ -794,7 +794,7 @@ def write_leaderboard_html_file( sorting_column_number: int, leaderboard_type: LeaderboardType, ) -> None: - """Generate HTML and CSV leaderboard files and upload to Bucket & git repo. + """Generate HTML and CSV leaderboard files and upload to Bucket. Args: df (pd.DataFrame): DataFrame containing the leaderboard. @@ -1026,7 +1026,7 @@ def write_leaderboard_html_file( stem = f"leaderboard_{leaderboard_type.value}" destination_folder = "leaderboards/html" - local_filename_html, destination_filename_html = data_utils.write_file_to_bucket( + data_utils.write_file_to_bucket( bucket=env.PUBLIC_RELEASE_BUCKET, basename=f"{stem}.html", destination_folder=f"{destination_folder}", @@ -1040,14 +1040,6 @@ def write_leaderboard_html_file( local_filename_csv = f"{directory}/{destination_filename_csv}" df.to_csv(local_filename_csv, index=False) - git.clone_commit_and_push( - files={ - local_filename_html: destination_filename_html, - local_filename_csv: destination_filename_csv, - }, - commit_message=f"leaderboard {leaderboard_type.value}: automatic update html & csv files.", - ) - def write_preliminary_leaderboard_html_file( df: pd.DataFrame, @@ -1242,7 +1234,7 @@ def write_preliminary_leaderboard_html_file( leaderboard_type = LeaderboardType.PRELIMINARY stem = f"leaderboard_{leaderboard_type.value}" destination_folder = "leaderboards/html" - local_filename_html, destination_filename_html = data_utils.write_file_to_bucket( + data_utils.write_file_to_bucket( bucket=env.PUBLIC_RELEASE_BUCKET, basename=f"{stem}.html", destination_folder=f"{destination_folder}", @@ -1256,14 +1248,6 @@ def write_preliminary_leaderboard_html_file( local_filename_csv = f"{directory}/{destination_filename_csv}" df.to_csv(local_filename_csv, index=False) - git.clone_commit_and_push( - files={ - local_filename_html: destination_filename_html, - local_filename_csv: destination_filename_csv, - }, - commit_message=f"leaderboard {leaderboard_type.value}: automatic update html & csv files.", - ) - def write_leaderboard_js_file_full( df: pd.DataFrame, diff --git a/src/nightly_update_workflow/manager/main.py b/src/nightly_update_workflow/manager/main.py index 72a21cb3..2019008b 100644 --- a/src/nightly_update_workflow/manager/main.py +++ b/src/nightly_update_workflow/manager/main.py @@ -255,6 +255,21 @@ def main(): ), ) + # Push the resolution sets, the leaderboards, the parity dates and the question fixed + # effects to the dataset repo in a single dedicated job. The resolve and leaderboard jobs + # run in parallel, so letting each push its own files races on the repo. Run last, once + # every job that writes to the bucket has finished, so the night goes out as one commit. + dict_to_use_push_datasets_to_git = "push_datasets_to_git" + operation_push_datasets_to_git = call_worker( + dict_to_use=dict_to_use_push_datasets_to_git, + task_count=1, + ) + cloud_run.block_and_check_job_result( + operation=operation_push_datasets_to_git, + name=dict_to_use_push_datasets_to_git, + exit_on_error=True, + ) + slack.send_message(message="Nightly update succeeded 😊") diff --git a/src/nightly_update_workflow/worker/main.py b/src/nightly_update_workflow/worker/main.py index 822b6e72..7b07aea2 100644 --- a/src/nightly_update_workflow/worker/main.py +++ b/src/nightly_update_workflow/worker/main.py @@ -25,6 +25,12 @@ def get_resolve_forecasts(): return [[("func-resolve-forecasts", True, cloud_run.timeout_1h * 3, task_count)]] +push_datasets_to_git = [ + [ + ("func-push-datasets-to-git", True, cloud_run.timeout_1h, 1), + ] +] + leaderboards = [ [ ("func-leaderboard-tournament", True, cloud_run.timeout_1h * 4, 1), @@ -142,7 +148,8 @@ def main(): Env variables: CLOUD_RUN_TASK_INDEX: automatically set by Cloud Run Jobs - DICT_TO_USE: one of `fetch_and_update`, `metadata`, `resolve_forecasts`, `leaderboards`. + DICT_TO_USE: one of `fetch_and_update`, `metadata`, `resolve_forecasts`, + `leaderboards`, `push_datasets_to_git`. """ dict_mapping = { "fetch_and_update": get_fetch_and_update(), @@ -151,6 +158,7 @@ def main(): "publish_question_set_make_llm_baseline": get_publish_question_set_make_llm_baseline(), "resolve_forecasts": get_resolve_forecasts(), "leaderboards": leaderboards, + "push_datasets_to_git": push_datasets_to_git, "naive_and_dummy_forecasters": get_naive_and_dummy_forecasters(), "website": website, } diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py index 44ba17f8..3eaaab0f 100644 --- a/src/orchestration/_io.py +++ b/src/orchestration/_io.py @@ -5,6 +5,7 @@ from __future__ import annotations +import fnmatch import json import logging import os @@ -237,10 +238,11 @@ def upload_hash_mapping(raw_json: str, source_name: str) -> None: def upload_resolution_set(df: pd.DataFrame, forecast_due_date: str, question_set_filename: str): - """Upload resolution set to GCS and push to git.""" - from helpers import git # noqa: E402 - from helpers import keys # noqa: E402 + """Upload resolution set to GCS. + Only uploads to the bucket; the file reaches the dataset repo later via + `push_datasets_to_git`. + """ basename = f"{forecast_due_date}_resolution_set.json" local_filename = f"/tmp/{basename}" df = df[["id", "source", "direction", "resolution_date", "resolved_to", "resolved"]] @@ -263,14 +265,74 @@ def upload_resolution_set(df: pd.DataFrame, forecast_due_date: str, question_set ) logger.info(f"Uploaded Resolution File {local_filename} to {upload_folder}.") + +# Bucket folder -> (git folder, basename pattern) for everything the nightly run publishes to +# the dataset repo. The resolution sets and the leaderboard html & csv keep the locations they +# already have in the repo; the parity dates and the question fixed effects are new to it. The +# patterns keep files that live in the same bucket folder but are not published (e.g. the SOTA +# graph csv files) out of the repo. +DATASET_PUSH_FOLDERS = ( + ("datasets/resolution_sets", "datasets/resolution_sets", "*_resolution_set.json"), + ("leaderboards/html", "leaderboards/html", "leaderboard_*.html"), + ("leaderboards/csv", "leaderboards/csv", "leaderboard_*.csv"), + # `[!0-9]` skips the dated files that predate the switch to one file per leaderboard + # (question_fixed_effects.2026-08-27.baseline_leaderboard.json) if any are left in the bucket. + ( + "question-fixed-effects", + "datasets/question_fixed_effects", + "question_fixed_effects.[!0-9]*.json", + ), + ("simulated_llm_parity", "datasets/parity_dates", "parity_dates.*.json"), +) + + +def push_datasets_to_git() -> None: + """Push resolution sets, leaderboards, parity dates & question fixed effects to git. + + Everything the nightly run publishes goes out in a single commit. Run as one job at the + end of the run rather than having each of the parallel resolve and leaderboard jobs push + its own files, which avoids the race condition of concurrent pushes to the repository. + """ + from helpers import git, keys # noqa: E402 + + if env.RUNNING_LOCALLY: + logger.info("Running locally; not pushing datasets to git.") + return + + files = {} + for bucket_folder, git_folder, pattern in DATASET_PUSH_FOLDERS: + blob_names = [ + blob_name + for blob_name in gcp.storage.list_with_prefix( + bucket_name=env.PUBLIC_RELEASE_BUCKET, + prefix=f"{bucket_folder}/", + ) + if fnmatch.fnmatch(os.path.basename(blob_name), pattern) + ] + for blob_name in blob_names: + basename = os.path.basename(blob_name) + local_filename = f"/tmp/{basename}" + gcp.storage.download( + bucket_name=env.PUBLIC_RELEASE_BUCKET, + filename=blob_name, + local_filename=local_filename, + ) + files[local_filename] = f"{git_folder}/{basename}" + + if not files: + logger.warning("No dataset files found in the bucket; nothing to push.") + return + mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL") mirrors = [mirrors] if mirrors else [] - git.clone_and_push_files( + committed = git.clone_and_push_files( repo_url=keys.API_GITHUB_DATASET_REPO_URL, - files={local_filename: f"{upload_folder}/{basename}"}, - commit_message=f"resolution set: automatic update for {question_set_filename}.", + files=files, + commit_message="datasets: automatic nightly update.", mirrors=mirrors, ) + if committed: + logger.info(f"Pushed {len(files)} dataset files to git in a single commit.") def upload_processed_forecast_file(data: dict, forecast_due_date: str, filename: str): diff --git a/src/orchestration/func_push_datasets_to_git/Makefile b/src/orchestration/func_push_datasets_to_git/Makefile new file mode 100644 index 00000000..f2d2031f --- /dev/null +++ b/src/orchestration/func_push_datasets_to_git/Makefile @@ -0,0 +1,40 @@ +all : + $(MAKE) clean + $(MAKE) deploy + +.PHONY : all clean deploy + +UPLOAD_DIR = upload +ROOT_DIR ?= $(abspath ../../..)/ +include $(ROOT_DIR)orchestration_upload.mk + +.gcloudignore: + cp -r $(ROOT_DIR)src/helpers/.gcloudignore . + +Dockerfile: $(ROOT_DIR)src/helpers/Dockerfile.template + sed \ + -e 's/REGION/$(CLOUD_DEPLOY_REGION)/g' \ + -e 's/STACK/google-22-full/g' \ + -e 's/PYTHON_VERSION/python312/g' \ + $< > Dockerfile + +NUM_CPUS = 2 + +deploy : main.py .gcloudignore requirements.txt Dockerfile + $(stage-orchestration-upload) + gcloud run jobs deploy \ + func-push-datasets-to-git \ + --project $(CLOUD_PROJECT) \ + --region $(CLOUD_DEPLOY_REGION) \ + --tasks 1 \ + --parallelism 1 \ + --task-timeout 1h \ + --memory 8Gi \ + --cpu $(NUM_CPUS) \ + --max-retries 0 \ + --service-account $(QUESTION_BANK_BUCKET_SERVICE_ACCOUNT) \ + --set-env-vars $(DEFAULT_CLOUD_FUNCTION_ENV_VARS) \ + --source $(UPLOAD_DIR) + +clean : + rm -rf $(UPLOAD_DIR) .gcloudignore Dockerfile diff --git a/src/orchestration/func_push_datasets_to_git/main.py b/src/orchestration/func_push_datasets_to_git/main.py new file mode 100644 index 00000000..662fcf82 --- /dev/null +++ b/src/orchestration/func_push_datasets_to_git/main.py @@ -0,0 +1,23 @@ +"""Cloud Run job: push everything the nightly run publishes to git in a single commit. + +See `orchestration._io.push_datasets_to_git`. +""" + +import logging +from typing import Any + +from helpers import decorator +from orchestration import _io + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@decorator.log_runtime +def driver(_: Any) -> None: + """Push resolution sets, leaderboards, parity dates & question fixed effects to git.""" + _io.push_datasets_to_git() + + +if __name__ == "__main__": + driver(None) diff --git a/src/orchestration/func_push_datasets_to_git/requirements.txt b/src/orchestration/func_push_datasets_to_git/requirements.txt new file mode 100644 index 00000000..09b8cc5b --- /dev/null +++ b/src/orchestration/func_push_datasets_to_git/requirements.txt @@ -0,0 +1,9 @@ +google-cloud-storage +google-cloud-secret-manager +pandas>=2.2.2,<3.0 +GitPython +termcolor +pandera +pytz +python-dateutil +slack_sdk diff --git a/src/www.forecastbench.org/Makefile b/src/www.forecastbench.org/Makefile index 23be3b60..23502126 100644 --- a/src/www.forecastbench.org/Makefile +++ b/src/www.forecastbench.org/Makefile @@ -39,7 +39,6 @@ deploy : entrypoint.sh Dockerfile .gcloudignore --exclude='assets/js/leaderboard_tournament_compact.js' \ --exclude='assets/js/leaderboard_tournament_full.js' \ --exclude='assets/js/leaderboard_preliminary_full.js' \ - --exclude='assets/data/question-fixed-effects/placeholder.json' \ --exclude='assets/data/sota_graph_tournament.csv' \ --exclude='assets/data/parity_dates.baseline_leaderboard.json' \ --exclude='assets/data/parity_dates.tournament_leaderboard.json' \ diff --git a/src/www.forecastbench.org/assets/data/question-fixed-effects/placeholder.json b/src/www.forecastbench.org/assets/data/question-fixed-effects/placeholder.json deleted file mode 100644 index e69de29b..00000000 diff --git a/src/www.forecastbench.org/datasets/index.md b/src/www.forecastbench.org/datasets/index.md index 6837308d..d1820499 100644 --- a/src/www.forecastbench.org/datasets/index.md +++ b/src/www.forecastbench.org/datasets/index.md @@ -13,6 +13,7 @@ footer_scripts:

We provide as much data as possible via our datasets repository on GitHub.

Leaderboards. The leaderboards are updated nightly and stored in git, allowing you to track model ranking over time.

Resolution values. Resolution values are also updated nightly and stored in git.

+

Question fixed effects. The question fixed effects , a byproduct of producing the leaderboards, are updated nightly and stored in git. Higher values imply more difficult questions.

Question sets. Question sets are released every two weeks through this repository.

Human survey data. The superforecaster and general public forecast sets from the 2024-07-21 survey round are available as well.

This repository is mirrored to Hugging Face .

@@ -27,7 +28,6 @@ footer_scripts:

We provide some other files as direct downloads.

Forecast sets. Download all forecasts that have been submitted for evalution here{% if site.data.direct_download_file_sizes.forecast_sets %} ({{ site.data.direct_download_file_sizes.forecast_sets }}B){% endif %}.

Processed forecast sets. Download all processed forecast files here{% if site.data.direct_download_file_sizes.processed_forecast_sets %} ({{ site.data.direct_download_file_sizes.processed_forecast_sets }}B){% endif %}.

-

Question fixed effect estimates. For those interested in detailed question-level analysis, we provide the question fixed effects estimates, generated when updating the leaderboard.

diff --git a/src/www.forecastbench.org/datasets/question-fixed-effects/index.md b/src/www.forecastbench.org/datasets/question-fixed-effects/index.md deleted file mode 100644 index 9367569d..00000000 --- a/src/www.forecastbench.org/datasets/question-fixed-effects/index.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -layout: splash -title: "Question Fixed Effects" -permalink: /datasets/question-fixed-effects/ ---- - -
-
-
-

Question fixed effect estimates

-

This page provides access to the estimated question fixed effects for all questions evaluated on the current baseline and tournament leaderboards. These files are a byproduct of producing the leaderboards. We provide them in hopes of evaluating the fixed effects used to score forecasting performance. For transparency, we provide the estimated question fixed effects for download. NB: higher scores imply more difficult questions.

-

New files are generated nightly as a byproduct of updating the leaderboards.

-
-
-
- -
-
-
-

File format

-

The question fixed effects files are provided as JSON files with the following fields:

-
    -
  • source: The source from which the question was pulled or generated.
    • -
  • id: The question ID (unique given source).
    • -
  • horizon: The forecast horizon in days (null for market questions)
    • -
  • forecast_due_date: The forecast due date associated with the question set the question comes from
    • -
  • leaderboard_type: The leaderboard associated with the estimate (baseline or tournament).
    • -
  • question_fixed_effect: The question fixed effect estimate.
    • -
-

For more information about these fields, see the wiki.

-
-
-
- -
-
-
-

Available files

- {% assign question_files = site.static_files | where_exp: "file", "file.path contains 'assets/data/question-fixed-effects/'" %} - {% if question_files.size > 0 %} - {% assign sorted_files = question_files | sort: "modified_time" | reverse %} -
    - {% for file in sorted_files limit: 10 %} -
  • - {{ file.name }} -
    • - {% endfor %} -
- {% else %} -

No question fixed effects files are currently available.

-

Files will appear here when they are uploaded.

- {% endif %} -
-
-
diff --git a/src/www.forecastbench.org/entrypoint.sh.template b/src/www.forecastbench.org/entrypoint.sh.template index 0bad6df1..0a66b610 100644 --- a/src/www.forecastbench.org/entrypoint.sh.template +++ b/src/www.forecastbench.org/entrypoint.sh.template @@ -15,12 +15,6 @@ cp -r . "$ASSETS_MNT/" # Copy all required files to the build dir before Jekyll build PUBLIC_RELEASE_MNT="$MOUNT_POINT/PUBLIC_RELEASE_BUCKET" -# question fixed effects files -QUESTION_FE_DIR="assets/data/question-fixed-effects" -mkdir -p "$ASSETS_MNT/$QUESTION_FE_DIR" -QUESTION_FE_SRC="$PUBLIC_RELEASE_MNT/question-fixed-effects" -cp -a "$QUESTION_FE_SRC"/question_fixed_effects.*.json "$ASSETS_MNT/$QUESTION_FE_DIR/" - # SOTA LLM Super parity dates SOTA_LLM_SUPER_DIR="assets/data/" mkdir -p "$ASSETS_MNT/$SOTA_LLM_SUPER_DIR"