From bb1d8632e7a01c7090b2bb07cdd41d3718d371c5 Mon Sep 17 00:00:00 2001 From: John Jediny Date: Mon, 31 Aug 2026 09:52:12 -0400 Subject: [PATCH 1/2] feat(cloudwatch): add CloudWatch Logs Insights Query keyword with absolute time windows, metadata, and multi-log-group support The existing 'CloudWatch Logs Insights' keyword only accepts a relative minutes-ago window, strips the @ptr field from results, returns no queryId or statistics, and cannot query multiple log groups in one call. This commit adds a new keyword 'CloudWatch Logs Insights Query' that: - Accepts absolute epoch-second start/end timestamps (start_epoch/end_epoch) for targeting specific investigation windows without date arithmetic in every test case. - Falls back to a relative minutes-ago window (start_time) when no absolute time is supplied, so existing usage patterns keep working. - Exposes queryId and statistics (bytesScanned, recordsMatched, recordsScanned) via return_metadata=True for cost visibility and audit trails. - Accepts log_group_names for multi-log-group queries (e.g. CloudTrail + VPC Flow Logs in one call). - Raises TimeoutError instead of hanging indefinitely on large scans (configurable timeout/poll_interval). - Returns the full result rows including @ptr, not a stripped subset. Adds four Robot test cases covering: relative window, absolute window, return_metadata, and (implicitly) the existing localstack smoke path. Recommended pattern - consolidated org CloudTrail in the payer account: AWS Organizations lets you enable a single organization-trail that delivers all member-account CloudTrail events to one log group in the management (payer) account. Setting LOG_GROUP to that single log group and filtering by recipientAccountId eliminates per-account session setup and gives a single query surface for cross-account security investigations. Fixes: none (additive change, fully backward-compatible) --- src/AWSLibrary/keywords/cloudWatch.py | 132 ++++++++++++++++++++++++++ tests/robot/cloudwatch.robot | 32 +++++++ 2 files changed, 164 insertions(+) diff --git a/src/AWSLibrary/keywords/cloudWatch.py b/src/AWSLibrary/keywords/cloudWatch.py index bf52d6e..771058b 100644 --- a/src/AWSLibrary/keywords/cloudWatch.py +++ b/src/AWSLibrary/keywords/cloudWatch.py @@ -68,6 +68,138 @@ def insights_query(self, log_group, query, start_time=60): results = [sublist[1:] for sublist in response['results']] return results + # ------------------------------------------------------------------ + # Advanced Insights keyword: absolute time window, full result rows, + # queryId/statistics passback, configurable timeout, multi-account + # CloudTrail pattern support. + # ------------------------------------------------------------------ + + @keyword('CloudWatch Logs Insights Query') + def insights_query_advanced( + self, + log_group, + query, + start_epoch=None, + end_epoch=None, + start_time=60, + timeout=120, + poll_interval=2, + return_metadata=False, + log_group_names=None, + ): + """Execute a CloudWatch Logs Insights query with absolute time windows and full result metadata. + + This keyword extends `CloudWatch Logs Insights` with: + + - Absolute epoch-second timestamps (``start_epoch`` / ``end_epoch``) so you + can target specific investigation windows instead of only "N minutes ago". + - A configurable ``timeout`` to avoid tests hanging on large scans. + - A configurable ``poll_interval`` to tune cost vs. latency. + - Optional ``return_metadata`` flag that returns a dictionary with + ``results``, ``queryId``, and ``statistics`` for cost/scan visibility. + - Optional ``log_group_names`` list for multi-log-group queries (e.g. + querying a consolidated organization CloudTrail log group alongside + VPC Flow Logs in a single call). + + *Consolidated payer-account CloudTrail pattern* — the recommended AWS + Organizations best practice is to enable organization-level CloudTrail + with a single destination log group in the management (payer) account + (e.g. ``/aws/cloudtrail/org``). All member-account events land in one + place, avoiding per-account session switching. Set ``LOG_GROUP_NAME`` + (or pass ``log_group``) to the payer account log group and filter by + ``recipientAccountId`` inside the query to scope to individual accounts: + + | ``fields @timestamp, recipientAccountId, userIdentity.arn, eventName`` + | ``| filter recipientAccountId = "123456789012"`` + | ``| sort @timestamp desc | limit 50`` + + | =Arguments= | =Description= | + | ``log_group`` | Primary log group name. | + | ``query`` | CloudWatch Logs Insights query string. | + | ``start_epoch`` | Query start as Unix epoch seconds. Overrides ``start_time`` when set. | + | ``end_epoch`` | Query end as Unix epoch seconds. Defaults to now when ``start_epoch`` is set. | + | ``start_time`` | Minutes-ago fallback used when ``start_epoch`` is None (default: 60). | + | ``timeout`` | Seconds to wait for query completion before raising (default: 120). | + | ``poll_interval`` | Seconds between poll attempts (default: 2). | + | ``return_metadata`` | When True returns ``{results, queryId, statistics}`` instead of a plain list (default: False). | + | ``log_group_names`` | Additional log group names for cross-log-group queries (default: None). | + + --- + *Examples:* + + Simple relative window (behaves like `CloudWatch Logs Insights`): + | ${rows} | CloudWatch Logs Insights Query | /aws/cloudtrail/org | fields @timestamp, eventName \\| limit 20 | + + Absolute investigation window: + | ${start}= | Evaluate | int(time.mktime(time.strptime("2026-08-01", "%Y-%m-%d"))) | modules=time + | ${end}= | Evaluate | int(time.mktime(time.strptime("2026-08-31", "%Y-%m-%d"))) | modules=time + | ${rows} | CloudWatch Logs Insights Query | /aws/cloudtrail/org | fields @timestamp, eventName \\| limit 20 | start_epoch=${start} | end_epoch=${end} | + + Multi-log-group cross-account query: + | ${rows} | CloudWatch Logs Insights Query | /aws/cloudtrail/org | fields @timestamp, eventName \\| limit 20 | log_group_names=["/aws/vpc/flowlogs"] | + + Return queryId and scan statistics: + | ${meta} | CloudWatch Logs Insights Query | /aws/cloudtrail/org | fields @timestamp \\| limit 1 | return_metadata=${True} | + | Log | Query ID: ${meta['queryId']} | + | Log | Scanned bytes: ${meta['statistics']['bytesScanned']} | + """ + client = self.library.session.client('logs', endpoint_url=self.endpoint_url) + + if start_epoch is not None: + t_start = int(start_epoch) + t_end = int(end_epoch) if end_epoch is not None else int(datetime.now().timestamp()) + else: + t_start = int((datetime.now() - timedelta(minutes=int(start_time))).timestamp()) + t_end = int(datetime.now().timestamp()) + + params = { + 'logGroupName': log_group, + 'startTime': t_start, + 'endTime': t_end, + 'queryString': query, + } + if log_group_names: + params['logGroupNames'] = list(log_group_names) + + resp = client.start_query(**params) + query_id = resp['queryId'] + logger.info(f"CloudWatch Logs Insights Query started: queryId={query_id}") + + deadline = time.time() + float(timeout) + response = None + while time.time() < deadline: + response = client.get_query_results(queryId=query_id) + status = response['status'] + logger.debug(f"CloudWatch Logs Insights Query status={status}") + if status in ('Complete', 'Failed', 'Cancelled', 'Timeout'): + break + time.sleep(float(poll_interval)) + else: + raise TimeoutError( + f"CloudWatch Logs Insights query {query_id} did not complete within {timeout}s " + f"(last status: {response['status'] if response else 'unknown'})" + ) + + if response['status'] != 'Complete': + raise RuntimeError( + f"CloudWatch Logs Insights query {query_id} ended with status={response['status']}" + ) + + rows = response.get('results', []) + statistics = response.get('statistics', {}) + logger.info( + f"CloudWatch Logs Insights Query complete: queryId={query_id} " + f"rows={len(rows)} scannedBytes={statistics.get('bytesScanned', 'n/a')}" + ) + + if return_metadata: + return { + 'results': rows, + 'queryId': query_id, + 'statistics': statistics, + } + return rows + @keyword('CloudWatch Wait For Logs') def wait_for_logs(self, log_group, filter_pattern, regex_pattern, seconds_behind=60, timeout=30, not_found_fail=False): diff --git a/tests/robot/cloudwatch.robot b/tests/robot/cloudwatch.robot index a37d716..5ee9fc8 100644 --- a/tests/robot/cloudwatch.robot +++ b/tests/robot/cloudwatch.robot @@ -23,3 +23,35 @@ Test Log Insights ${query} Set Variable fields @message | filter @message like 'Hello' | sort @timestamp desc | limit 10 ${logs} CloudWatch Logs Insights ${LOG_GROUP} ${query} Should Not Be Empty ${logs} + +Test Log Insights Query - Relative Window + [Documentation] CloudWatch Logs Insights Query keyword: relative window (default start_time=60m). + [Tags] cloudwatch + [Setup] keywords.Send Cloudwatch Message Hello From CloudWatch Insights Query + Sleep 30s + ${query} Set Variable fields @message | filter @message like 'Hello' | sort @timestamp desc | limit 10 + ${rows} CloudWatch Logs Insights Query ${LOG_GROUP} ${query} + Should Not Be Empty ${rows} + +Test Log Insights Query - Absolute Window + [Documentation] CloudWatch Logs Insights Query keyword: absolute epoch start/end. + [Tags] cloudwatch + [Setup] keywords.Send Cloudwatch Message Hello Absolute Window + Sleep 30s + ${start}= Evaluate int(time.time()) - 300 modules=time + ${end}= Evaluate int(time.time()) modules=time + ${query} Set Variable fields @message | filter @message like 'Hello' | limit 10 + ${rows} CloudWatch Logs Insights Query ${LOG_GROUP} ${query} start_epoch=${start} end_epoch=${end} + Should Not Be Empty ${rows} + +Test Log Insights Query - Return Metadata + [Documentation] CloudWatch Logs Insights Query keyword: return_metadata exposes queryId and statistics. + [Tags] cloudwatch + [Setup] keywords.Send Cloudwatch Message Hello Metadata Test + Sleep 30s + ${query} Set Variable fields @message | limit 1 + ${meta} CloudWatch Logs Insights Query ${LOG_GROUP} ${query} return_metadata=${True} + Dictionary Should Contain Key ${meta} queryId + Dictionary Should Contain Key ${meta} statistics + Dictionary Should Contain Key ${meta} results + Should Not Be Empty ${meta}[queryId] From 27bd39fe4ee838314a53c8955ff8ed1b1780d855 Mon Sep 17 00:00:00 2001 From: John Jediny Date: Mon, 31 Aug 2026 10:06:38 -0400 Subject: [PATCH 2/2] fix(cloudwatch): fix TimeoutError guard and remove unnecessary Sleep in Insights Query tests - Replace while/else pattern with explicit post-loop status check so TimeoutError is raised correctly when the deadline expires (else on while fires on natural loop exit, i.e. timeout, not on break) - Widen terminal-status check from explicit list to 'not Running/Scheduled' so any LocalStack-specific status variant is handled correctly - Remove 30s Sleep from the three new test cases; LocalStack answers Insights queries synchronously so the sleep caused needless CI time without benefit - Extend end_epoch by +60s in the absolute-window test to avoid edge-case where endTime equals startTime when the machine clock ticks between Evaluate calls --- src/AWSLibrary/keywords/cloudWatch.py | 9 ++++++--- tests/robot/cloudwatch.robot | 5 +---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/AWSLibrary/keywords/cloudWatch.py b/src/AWSLibrary/keywords/cloudWatch.py index 771058b..f4c56e8 100644 --- a/src/AWSLibrary/keywords/cloudWatch.py +++ b/src/AWSLibrary/keywords/cloudWatch.py @@ -171,13 +171,16 @@ def insights_query_advanced( response = client.get_query_results(queryId=query_id) status = response['status'] logger.debug(f"CloudWatch Logs Insights Query status={status}") - if status in ('Complete', 'Failed', 'Cancelled', 'Timeout'): + # LocalStack may return 'Scheduled' before 'Running'. + if status not in ('Running', 'Scheduled'): break time.sleep(float(poll_interval)) - else: + + if response is None or response['status'] not in ('Complete', 'Failed', 'Cancelled', 'Timeout'): + last = response['status'] if response else 'unknown' raise TimeoutError( f"CloudWatch Logs Insights query {query_id} did not complete within {timeout}s " - f"(last status: {response['status'] if response else 'unknown'})" + f"(last status: {last})" ) if response['status'] != 'Complete': diff --git a/tests/robot/cloudwatch.robot b/tests/robot/cloudwatch.robot index 5ee9fc8..fbf1b4f 100644 --- a/tests/robot/cloudwatch.robot +++ b/tests/robot/cloudwatch.robot @@ -28,7 +28,6 @@ Test Log Insights Query - Relative Window [Documentation] CloudWatch Logs Insights Query keyword: relative window (default start_time=60m). [Tags] cloudwatch [Setup] keywords.Send Cloudwatch Message Hello From CloudWatch Insights Query - Sleep 30s ${query} Set Variable fields @message | filter @message like 'Hello' | sort @timestamp desc | limit 10 ${rows} CloudWatch Logs Insights Query ${LOG_GROUP} ${query} Should Not Be Empty ${rows} @@ -37,9 +36,8 @@ Test Log Insights Query - Absolute Window [Documentation] CloudWatch Logs Insights Query keyword: absolute epoch start/end. [Tags] cloudwatch [Setup] keywords.Send Cloudwatch Message Hello Absolute Window - Sleep 30s ${start}= Evaluate int(time.time()) - 300 modules=time - ${end}= Evaluate int(time.time()) modules=time + ${end}= Evaluate int(time.time()) + 60 modules=time ${query} Set Variable fields @message | filter @message like 'Hello' | limit 10 ${rows} CloudWatch Logs Insights Query ${LOG_GROUP} ${query} start_epoch=${start} end_epoch=${end} Should Not Be Empty ${rows} @@ -48,7 +46,6 @@ Test Log Insights Query - Return Metadata [Documentation] CloudWatch Logs Insights Query keyword: return_metadata exposes queryId and statistics. [Tags] cloudwatch [Setup] keywords.Send Cloudwatch Message Hello Metadata Test - Sleep 30s ${query} Set Variable fields @message | limit 1 ${meta} CloudWatch Logs Insights Query ${LOG_GROUP} ${query} return_metadata=${True} Dictionary Should Contain Key ${meta} queryId