Fix/148, 181 resolution and leaderboard sets to one commit

Makefile

@@ -120,7 +120,7 @@ deploy: orchestration questions metadata resolve leaderboards curate-questions w
 
 questions: manifold metaculus acled infer kalshi yfinance polymarket wikipedia fred dbnomics
 
-orchestration: nightly-worker-job nightly-manager-job compress_buckets
+orchestration: nightly-worker-job nightly-manager-job compress_buckets push-datasets-to-git
 
 metadata: tag-questions validate-questions
 
@@ -248,6 +248,9 @@ nightly-worker-job:
 nightly-manager-job:
 	$(MAKE) -C src/nightly_update_workflow/manager || echo "* $@" >> $(MAKE_FAILURE_LOG)
 
+push-datasets-to-git:
+	$(MAKE) -C src/orchestration/func_push_datasets_to_git || echo "* $@" >> $(MAKE_FAILURE_LOG)
+
 llm-forecaster: llm-forecaster-manager llm-forecaster-worker
 
 llm-forecaster-manager:

src/base_eval/naive_and_dummy_forecasters/requirements.txt

@@ -10,3 +10,4 @@ cmdstanpy
 scipy
 termcolor
 pandera
+slack_sdk

src/curate_questions/publish_question_set/requirements.txt

@@ -6,3 +6,4 @@ tqdm
 scipy
 pandera
 termcolor
+slack_sdk

src/helpers/git.py

@@ -9,20 +9,20 @@
 
 from git import Actor, Repo
 
-from . import constants, env, keys
+from . import constants, keys, slack
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
-def clone(repo_url: str) -> Tuple[Repo, str, str]:
+def clone(repo_url: str) -> Optional[Tuple[Repo, str, str]]:
     """Clone a Git repository into a temporary directory with a temporary SSH key.
 
     Args:
         repo_url (str): The SSH URL of the repository to clone.
 
     Returns:
-        Tuple[Repo, str, str]:
+        Optional[Tuple[Repo, str, str]]: None if the SSH key secret is not set, otherwise:
             - repo (Repo): The cloned GitPython Repo object.
             - local_repo_dir (str): The temporary directory where the repository was cloned.
             - tmp_key_file_path (str): Path to the temporary SSH private key used for cloning.
@@ -57,7 +57,7 @@ def clone_and_push_files(
     files: Dict[str, str],
     commit_message: str,
     mirrors: Optional[List[str]] = None,
-) -> None:
+) -> bool:
     """Clone a Git repository, add/update files, commit, and push to origin and optional mirrors.
 
     Args:
@@ -68,13 +68,19 @@ def clone_and_push_files(
                                        If None, attempts to load from secrets.
 
     Returns:
-        None. Exits with status 1 if an error is encountered while pushing.
+        bool: True if a new commit was created, False if HEAD was unchanged or if the SSH key
+              secret is not set, in which case nothing is cloned or pushed. Origin and mirrors
+              are pushed either way. Exits with status 1 if the push to origin fails; a failed
+              mirror push is logged and sent to Slack as a warning.
     """
     if not mirrors:
         mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL")
         mirrors = [mirrors] if mirrors else []
 
-    repo, local_repo_dir, tmp_key_file_path = clone(repo_url=repo_url)
+    cloned = clone(repo_url=repo_url)
+    if cloned is None:
+        return False
+    repo, local_repo_dir, tmp_key_file_path = cloned
 
     for source, destination in files.items():
         full_destination_path = f"{local_repo_dir}/{destination}"
@@ -84,50 +90,47 @@ def clone_and_push_files(
         shutil.copy(source, full_destination_path, follow_symlinks=False)
         repo.index.add([destination])
 
+    # Skip empty commits, but always push so a lagging mirror catches up.
+    has_changes = bool(repo.index.diff(repo.head.commit))
+    if not has_changes:
+        logger.info(f"No new commit for {repo_url}; syncing remotes to HEAD.")
+
     error_encountered = False
     author = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL)
     committer = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL)
     ssh_env = {"GIT_SSH_COMMAND": f"ssh -i {tmp_key_file_path} -o StrictHostKeyChecking=no"}
     try:
-        repo.index.commit(commit_message, author=author, committer=committer)
+        if has_changes:
+            repo.index.commit(commit_message, author=author, committer=committer)
         origin = repo.remote(name="origin")
-        origin.push(env=ssh_env)
-        for index, mirror_url in enumerate(mirrors):
-            mirror = repo.create_remote(f"mirror_{index}", url=mirror_url)
-            mirror.push(env=ssh_env)
-            repo.delete_remote(mirror.name)
-            logger.info(f"Pushed to {mirror_url} (mirror) with commit message: {commit_message}")
+        # A rejected push does not raise in GitPython; surface it explicitly.
+        origin.push(env=ssh_env).raise_if_error()
     except Exception as e:
         error_encountered = True
         message = e.message if hasattr(e, "message") else str(e)
         logger.error(f"encountered error when pushing to git: {message}")
 
+    # Mirrors are convenience copies; the repo at `repo_url` is the source of truth. A mirror
+    # that's down or has diverged must not fail the nightly run, so warn on Slack and carry on.
+    if not error_encountered:
+        for index, mirror_url in enumerate(mirrors):
+            try:
+                mirror = repo.create_remote(f"mirror_{index}", url=mirror_url)
+                mirror.push(env=ssh_env).raise_if_error()
+                repo.delete_remote(mirror.name)
+                logger.info(f"Pushed to {mirror_url} (mirror)")
+            except Exception as e:
+                error = e.message if hasattr(e, "message") else str(e)
+                message = f"Could not push to {mirror_url} (mirror): {error}"
+                logger.warning(message)
+                slack.send_message(message=f"*MIRROR PUSH FAILED*\n{message}")
+
     os.remove(tmp_key_file_path)
     shutil.rmtree(local_repo_dir, ignore_errors=True)
 
     if error_encountered:
         sys.exit(1)
 
-    logger.info(f"Pushed to {repo_url} with commit message: {commit_message}")
-
-
-def clone_commit_and_push(
-    files: Dict[str, str],
-    commit_message: str,
-) -> None:
-    """Upload files files to Cloud Storage and push updates to Git.
-
-    Args:
-        files (Dict[str, str]): Mapping of local file paths to their git location.
-
-    Returns:
-        None
-    """
-    if env.RUNNING_LOCALLY:
-        return
-
-    clone_and_push_files(
-        repo_url=keys.API_GITHUB_DATASET_REPO_URL,
-        files=files,
-        commit_message=commit_message,
-    )
+    if has_changes:
+        logger.info(f"Pushed to {repo_url} with commit message: {commit_message}")
+    return has_changes

src/leaderboard/main.py

@@ -32,7 +32,6 @@
     dates,
     decorator,
     env,
-    git,
     resolution,
     slack,
 )
@@ -752,7 +751,7 @@ def write_question_fixed_effects(
         None: Concatenated DataFrame is created (and can be written or processed
         further inside the function).
     """
-    logger.info(colored("Writing question fixed effects to WEBSITE.", "yellow"))
+    logger.info(colored("Writing question fixed effects.", "yellow"))
 
     dfs = []
     for question_type, df_qfe in question_fixed_effects.items():
@@ -779,11 +778,12 @@ def write_question_fixed_effects(
     ]
 
     directory = data_utils.get_mounted_bucket(bucket=env.PUBLIC_RELEASE_BUCKET)
-    iso_date = dates.get_date_today_as_iso()
     leaderboard_file_stem = LEADERBOARD_FILE_STEMS[leaderboard_type]
+    # One file per leaderboard, overwritten nightly. The history lives in the dataset repo,
+    # where `push_datasets_to_git` sends this file every night.
     local_filename = (
         f"{directory}/question-fixed-effects/"
-        f"question_fixed_effects.{iso_date}.{leaderboard_file_stem}.json"
+        f"question_fixed_effects.{leaderboard_file_stem}.json"
     )
     os.makedirs(os.path.dirname(local_filename), exist_ok=True)
     df.to_json(local_filename, orient="records")
@@ -794,7 +794,7 @@ def write_leaderboard_html_file(
     sorting_column_number: int,
     leaderboard_type: LeaderboardType,
 ) -> None:
-    """Generate HTML and CSV leaderboard files and upload to Bucket & git repo.
+    """Generate HTML and CSV leaderboard files and upload to Bucket.
 
     Args:
         df (pd.DataFrame): DataFrame containing the leaderboard.
@@ -1026,7 +1026,7 @@ def write_leaderboard_html_file(
 
     stem = f"leaderboard_{leaderboard_type.value}"
     destination_folder = "leaderboards/html"
-    local_filename_html, destination_filename_html = data_utils.write_file_to_bucket(
+    data_utils.write_file_to_bucket(
         bucket=env.PUBLIC_RELEASE_BUCKET,
         basename=f"{stem}.html",
         destination_folder=f"{destination_folder}",
@@ -1040,14 +1040,6 @@ def write_leaderboard_html_file(
     local_filename_csv = f"{directory}/{destination_filename_csv}"
     df.to_csv(local_filename_csv, index=False)
 
-    git.clone_commit_and_push(
-        files={
-            local_filename_html: destination_filename_html,
-            local_filename_csv: destination_filename_csv,
-        },
-        commit_message=f"leaderboard {leaderboard_type.value}: automatic update html & csv files.",
-    )
-
 
 def write_preliminary_leaderboard_html_file(
     df: pd.DataFrame,
@@ -1242,7 +1234,7 @@ def write_preliminary_leaderboard_html_file(
     leaderboard_type = LeaderboardType.PRELIMINARY
     stem = f"leaderboard_{leaderboard_type.value}"
     destination_folder = "leaderboards/html"
-    local_filename_html, destination_filename_html = data_utils.write_file_to_bucket(
+    data_utils.write_file_to_bucket(
         bucket=env.PUBLIC_RELEASE_BUCKET,
         basename=f"{stem}.html",
         destination_folder=f"{destination_folder}",
@@ -1256,14 +1248,6 @@ def write_preliminary_leaderboard_html_file(
     local_filename_csv = f"{directory}/{destination_filename_csv}"
     df.to_csv(local_filename_csv, index=False)
 
-    git.clone_commit_and_push(
-        files={
-            local_filename_html: destination_filename_html,
-            local_filename_csv: destination_filename_csv,
-        },
-        commit_message=f"leaderboard {leaderboard_type.value}: automatic update html & csv files.",
-    )
-
 
 def write_leaderboard_js_file_full(
     df: pd.DataFrame,

src/nightly_update_workflow/manager/main.py

@@ -255,6 +255,21 @@ def main():
         ),
     )
 
+    # Push the resolution sets, the leaderboards, the parity dates and the question fixed
+    # effects to the dataset repo in a single dedicated job. The resolve and leaderboard jobs
+    # run in parallel, so letting each push its own files races on the repo. Run last, once
+    # every job that writes to the bucket has finished, so the night goes out as one commit.
+    dict_to_use_push_datasets_to_git = "push_datasets_to_git"
+    operation_push_datasets_to_git = call_worker(
+        dict_to_use=dict_to_use_push_datasets_to_git,
+        task_count=1,
+    )
+    cloud_run.block_and_check_job_result(
+        operation=operation_push_datasets_to_git,
+        name=dict_to_use_push_datasets_to_git,
+        exit_on_error=True,
+    )
+
     slack.send_message(message="Nightly update succeeded 😊")
 
 

src/nightly_update_workflow/worker/main.py

@@ -25,6 +25,12 @@ def get_resolve_forecasts():
     return [[("func-resolve-forecasts", True, cloud_run.timeout_1h * 3, task_count)]]
 
 
+push_datasets_to_git = [
+    [
+        ("func-push-datasets-to-git", True, cloud_run.timeout_1h, 1),
+    ]
+]
+
 leaderboards = [
     [
         ("func-leaderboard-tournament", True, cloud_run.timeout_1h * 4, 1),
@@ -142,7 +148,8 @@ def main():
 
     Env variables:
     CLOUD_RUN_TASK_INDEX: automatically set by Cloud Run Jobs
-    DICT_TO_USE: one of `fetch_and_update`, `metadata`, `resolve_forecasts`, `leaderboards`.
+    DICT_TO_USE: one of `fetch_and_update`, `metadata`, `resolve_forecasts`,
+                 `leaderboards`, `push_datasets_to_git`.
     """
     dict_mapping = {
         "fetch_and_update": get_fetch_and_update(),
@@ -151,6 +158,7 @@ def main():
         "publish_question_set_make_llm_baseline": get_publish_question_set_make_llm_baseline(),
         "resolve_forecasts": get_resolve_forecasts(),
         "leaderboards": leaderboards,
+        "push_datasets_to_git": push_datasets_to_git,
         "naive_and_dummy_forecasters": get_naive_and_dummy_forecasters(),
         "website": website,
     }