From 9a2ff6e29403d27c40c6f9ea5f7f8e158a89a459 Mon Sep 17 00:00:00 2001
From: tmanik <timothyemanik@gmail.com>
Date: Mon, 13 Jul 2026 14:38:28 -0400
Subject: [PATCH] fix: retry transient OpenSearch 500s, log full tracebacks

AOSS began intermittently returning HTTP 500 ('Internal error occurred
while processing request') on ~50% of valid search requests, killing
every other chat query. opensearch-py's default retry config only
retries 502/503/504, so these were surfaced straight to users.

- Retry search requests on 500/502/503/504 (up to 3 attempts) and on
  timeouts; verified 10/10 queries succeed against the degraded
  collection vs 5/10 without retries
- Use logger.exception in lambda_handler and invoke_model so failures
  log full tracebacks instead of a one-line message
- Narrow the collector image Docker build context so backend-only
  edits no longer rebuild the content-sync container and replace its
  task definition on every deploy
---
 .dockerignore                   | 20 +++++++-------------
 src/backend/chatbot_backend.py  | 10 +++++-----
 src/backend/opensearch_query.py |  4 ++++
 3 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 5e81f2f..263a5ae 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,16 +1,10 @@
-# Keeps the collector image build context (repo root) small.
-# Only config.yaml and ingest_utils/confluence_processor are copied in.
-.git
-cdk.out
-node_modules
-frontend
-planning-docs
-**/__pycache__
-**/*.pyc
-.venv
-venv
+# Exclude everything, then re-include only what the collector image needs
+*
+!config.yaml
+!ingest_utils
 ingest_utils/confluence_processor/confluence_downloads
 ingest_utils/confluence_processor/google_drive_downloads
 ingest_utils/confluence_processor/service-credentials.json
-ingest_utils/confluence_processor/names.env
-ingest_utils/confluence_processor/names.env.copy
+ingest_utils/confluence_processor/names.env*
+**/__pycache__
+**/*.pyc
\ No newline at end of file
diff --git a/src/backend/chatbot_backend.py b/src/backend/chatbot_backend.py
index 8d2ffcf..f0eb6ee 100644
--- a/src/backend/chatbot_backend.py
+++ b/src/backend/chatbot_backend.py
@@ -18,7 +18,7 @@
 dynamodb = boto3.resource("dynamodb")
 ssm = boto3.client("ssm")
 
-# Reject oversized queries before any billable Bedrock/OpenSearch call (S5).
+# Reject oversized queries before any billable Bedrock/OpenSearch call
 MAX_QUERY_CHARS = 4000
 
 # Cache for prompts
@@ -187,8 +187,8 @@ def invoke_model(
 
         return response["output"]["message"]["content"][0]["text"]
 
-    except Exception as e:
-        logger.error(f"Error invoking the model: {str(e)}")
+    except Exception:
+        logger.exception("Error invoking the model")
         return None
 
 
@@ -615,8 +615,8 @@ def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
             ),
         }
 
-    except Exception as e:
-        logger.error(f"Error in lambda_handler: {e}")
+    except Exception:
+        logger.exception("Error in lambda_handler")
         return {
             "statusCode": 500,
             "body": json.dumps("Error processing message"),
diff --git a/src/backend/opensearch_query.py b/src/backend/opensearch_query.py
index 41930ed..fa426da 100644
--- a/src/backend/opensearch_query.py
+++ b/src/backend/opensearch_query.py
@@ -27,6 +27,10 @@ def initialize_opensearch():
         use_ssl=True,
         verify_certs=True,
         connection_class=RequestsHttpConnection,
+        # AOSS intermittently returns 500s on valid queries; retry through them
+        max_retries=3,
+        retry_on_status=(500, 502, 503, 504),
+        retry_on_timeout=True,
     )
 
     return client