diff --git a/fact/src/config/reloader/mod.rs b/fact/src/config/reloader/mod.rs index 29829f35..6b4c6028 100644 --- a/fact/src/config/reloader/mod.rs +++ b/fact/src/config/reloader/mod.rs @@ -116,7 +116,7 @@ impl Reloader { let path = PathBuf::from(file); if path.exists() { let mtime = match path.metadata() { - Ok(m) => m.mtime(), + Ok(m) => m.mtime_nsec(), Err(e) => { warn!("Failed to stat {file}: {e}"); warn!("Configuration reloading may not work"); diff --git a/tests/conftest.py b/tests/conftest.py index b66581c8..d64ce678 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import json import os +import time from shutil import rmtree from tempfile import NamedTemporaryFile, mkdtemp from time import sleep @@ -13,11 +14,26 @@ import requests import yaml +from metrics import MetricsSnapshot from server import EventServer, GrpcServer, OtlpServer -# Declare files holding fixtures pytest_plugins = ['test_editors.commons'] +ENDPOINT_ADDRESS = '127.0.0.1:9000' + + +def base_config() -> dict: + return { + 'paths': [], + 'endpoint': { + 'address': ENDPOINT_ADDRESS, + 'expose_metrics': True, + 'health_check': True, + }, + 'json': True, + 'scan_interval': 0, + } + @pytest.fixture def monitored_dir(): @@ -33,7 +49,7 @@ def monitored_dir(): @pytest.fixture def test_file(monitored_dir: str): """ - Create a temporary file for tests + Create a temporary file for tests. This file needs to exist when fact starts up for the inode tracking algorithm to work. @@ -77,21 +93,23 @@ def _get_output_modes(config: pytest.Config) -> list[str]: def pytest_generate_tests(metafunc: pytest.Metafunc): - if 'server' in metafunc.fixturenames: + if 'server' in metafunc.definition._fixtureinfo.argnames: modes = _get_output_modes(metafunc.config) metafunc.parametrize('server', modes, indirect=True) -@pytest.fixture +@pytest.fixture(scope='session') def server(request: pytest.FixtureRequest): """ Start and stop an event server. Parameterised via --output to create either a GrpcServer or an - OtlpServer. When --output=all, every test that uses this fixture - runs once per output mode. + OtlpServer. When --output=all, every test that directly + requests this fixture runs once per output mode. Tests that + only get it transitively (via autouse ``fact_config``) default + to grpc. """ - mode = request.param + mode = getattr(request, 'param', 'grpc') if mode == 'otlp': s: EventServer = OtlpServer() else: @@ -143,99 +161,40 @@ def dump_container_inspect( json.dump(container_inspect, f, indent=2) -@pytest.fixture -def fact_config( - request: pytest.FixtureRequest, - monitored_dir: str, - logs_dir: str, - server: EventServer, -): - cwd = os.getcwd() - config: dict = { - 'paths': [ - f'{monitored_dir}', - f'{monitored_dir}/**/*', - '/mounted/**/*', - '/container-dir/**/*', - ], - 'endpoint': { - 'address': '127.0.0.1:9000', - 'expose_metrics': True, - 'health_check': True, - }, - 'json': True, - 'scan_interval': 0, - } - - if server.output_mode == 'otlp': - config['otel'] = {'endpoint': 'http://127.0.0.1:4318/v1/logs'} - else: - config['grpc'] = {'url': 'http://127.0.0.1:9999'} +@pytest.fixture(scope='session') +def fact_config_file(): + """ + Session-scoped config file shared across all tests. - config_file = NamedTemporaryFile( # noqa: SIM115 + The file is created once with a baseline config and rewritten + by the per-test ``fact_config`` fixture before each test. + """ + cwd = os.getcwd() + config = base_config() + f = NamedTemporaryFile( # noqa: SIM115 prefix='fact-config-', suffix='.yml', dir=cwd, mode='w', ) - yaml.dump(config, config_file) - - yield config, config_file.name - with ( - open(os.path.join(logs_dir, 'fact.yml'), 'w') as f, - open(config_file.name) as r, - ): - f.write(r.read()) - config_file.close() - - -@pytest.fixture -def test_container( - request: pytest.FixtureRequest, - docker_client: docker.DockerClient, - monitored_dir: str, - ignored_dir: str, -): - """ - Run a container for triggering events in. - """ - container = docker_client.containers.run( - 'quay.io/fedora/fedora:43', - detach=True, - tty=True, - volumes={ - ignored_dir: { - 'bind': '/mounted', - 'mode': 'z', - }, - monitored_dir: { - 'bind': '/unmonitored', - 'mode': 'z', - }, - }, - name='fedora', - ) - container.exec_run('mkdir /container-dir') - - yield container + yaml.dump(config, f) + yield f.name + f.close() - container.stop(timeout=1) - container.remove() - -@pytest.fixture(autouse=True) +@pytest.fixture(scope='session', autouse=True) def fact( request: pytest.FixtureRequest, docker_client: docker.DockerClient, - fact_config: tuple[dict, str], - server: EventServer, - logs_dir: str, - test_file: str, + fact_config_file: str, ): """ - Run the fact docker container for integration tests. + Session-scoped fact container shared across all tests. + + The container is started once and kept alive for the entire test + session. Between tests the ``fact_config`` fixture rewrites the + config file and sends SIGHUP so that fact picks up new paths. """ - config, config_file = fact_config image = request.config.getoption('--image') assert isinstance(image, str) container = docker_client.containers.run( @@ -256,20 +215,21 @@ def fact( 'bind': '/host', 'mode': 'ro', }, - config_file: { + fact_config_file: { 'bind': '/etc/stackrox/fact.yml', 'mode': 'ro', }, }, ) - container_log = os.path.join(logs_dir, 'fact.log') - # Wait for container to be ready + session_logs = os.path.join(os.getcwd(), 'logs') + container_log = os.path.join(session_logs, 'fact.log') + + os.makedirs(session_logs, exist_ok=True) + for _ in range(10): try: - resp = requests.get( - f'http://{config["endpoint"]["address"]}/health_check' - ) + resp = requests.get(f'http://{ENDPOINT_ADDRESS}/health_check') if resp.status_code == 200: break except (requests.RequestException, requests.ConnectionError) as e: @@ -283,26 +243,141 @@ def fact( yield container - # Capture prometheus metrics before stopping the container - if config['endpoint']['expose_metrics']: - metric_log = os.path.join(logs_dir, 'metrics') - resp = requests.get(f'http://{config["endpoint"]["address"]}/metrics') + try: + resp = requests.get(f'http://{ENDPOINT_ADDRESS}/metrics') if resp.status_code == 200: - with open(metric_log, 'w') as f: + with open(os.path.join(session_logs, 'metrics'), 'w') as f: f.write(resp.text) + except requests.RequestException: + pass container.stop(timeout=5) exit_status = container.wait(timeout=2) try: dump_logs(container, container_log) dump_container_inspect( - docker_client, container, os.path.join(logs_dir, 'container.json') + docker_client, + container, + os.path.join(session_logs, 'container.json'), ) finally: container.remove() assert exit_status['StatusCode'] == 0 +def reload_fact( + container: docker.models.containers.Container, + config: dict, + config_file: str, + delay: float = 0.1, +): + with open(config_file, 'w') as f: + yaml.dump(config, f) + container.kill('SIGHUP') + sleep(delay) + + +@pytest.fixture(autouse=True) +def fact_config( + fact: docker.models.containers.Container, + fact_config_file: str, + monitored_dir: str, + server: EventServer, + logs_dir: str, + test_file: str, +): + """ + Per-test config that hot-reloads fact via SIGHUP. + + On setup: writes a config with the test's ``monitored_dir`` paths, + drains stale events from the server queue, and signals fact to + reload. + + On teardown: restores the baseline config (empty paths) so that + fact does not fire on directory cleanup, and captures + timestamp-sliced container logs for the test. + """ + test_start = time.time() + + config = base_config() + if server.output_mode == 'otlp': + config['otel'] = {'endpoint': 'http://127.0.0.1:4318/v1/logs'} + else: + config['grpc'] = {'url': 'http://127.0.0.1:9999'} + config['paths'] = [ + monitored_dir, + f'{monitored_dir}/**/*', + '/mounted/**/*', + '/container-dir/**/*', + ] + + server.drain() + reload_fact(fact, config, fact_config_file) + + metrics_before = MetricsSnapshot.fetch(ENDPOINT_ADDRESS) + + yield config, fact_config_file + + with ( + open(os.path.join(logs_dir, 'fact.yml'), 'w') as out, + open(fact_config_file) as src, + ): + out.write(src.read()) + + try: + metrics_after = MetricsSnapshot.fetch(ENDPOINT_ADDRESS) + metrics_delta = metrics_before.delta(metrics_after) + with open(os.path.join(logs_dir, 'metrics'), 'w') as f: + f.write(metrics_delta.to_text()) + except Exception: + pass + + try: + test_end = time.time() + log_bytes = fact.logs(since=test_start, until=test_end) + with open(os.path.join(logs_dir, 'fact.log'), 'wb') as f: + f.write(log_bytes) + except Exception: + pass + + config = base_config() + reload_fact(fact, config, fact_config_file) + + +@pytest.fixture +def test_container( + request: pytest.FixtureRequest, + docker_client: docker.DockerClient, + monitored_dir: str, + ignored_dir: str, +): + """ + Run a container for triggering events in. + """ + container = docker_client.containers.run( + 'quay.io/fedora/fedora:43', + detach=True, + tty=True, + volumes={ + ignored_dir: { + 'bind': '/mounted', + 'mode': 'z', + }, + monitored_dir: { + 'bind': '/unmonitored', + 'mode': 'z', + }, + }, + name='fedora', + ) + container.exec_run('mkdir /container-dir') + + yield container + + container.stop(timeout=1) + container.remove() + + def pytest_addoption(parser: pytest.Parser): parser.addoption( '--image', diff --git a/tests/metrics.py b/tests/metrics.py new file mode 100644 index 00000000..7d8bbfae --- /dev/null +++ b/tests/metrics.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import requests +from prometheus_client import CollectorRegistry +from prometheus_client.metrics_core import Metric +from prometheus_client.openmetrics.exposition import generate_latest +from prometheus_client.openmetrics.parser import ( + text_string_to_metric_families, +) +from prometheus_client.samples import Sample + + +class MetricsSnapshot: + """Parsed snapshot of Prometheus metrics from fact's /metrics endpoint. + + Stores the full list of ``Metric`` family objects returned by the + prometheus-client parser so that snapshots can be diffed and + serialised back to valid OpenMetrics text exposition format. + """ + + def __init__(self, families: list[Metric]): + self._families = families + self._index: dict[str, dict[str, float]] = {} + for fam in families: + by_label: dict[str, float] = {} + for s in fam.samples: + key = _label_key(s.labels) + by_label[key] = s.value + self._index[fam.name] = by_label + + @classmethod + def from_text(cls, text: str) -> MetricsSnapshot: + families = list(text_string_to_metric_families(text)) + return cls(families) + + @classmethod + def fetch(cls, endpoint: str) -> MetricsSnapshot: + resp = requests.get(f'http://{endpoint}/metrics') + resp.raise_for_status() + return cls.from_text(resp.text) + + def get( + self, + metric_name: str, + labels: dict[str, str] | None = None, + ) -> str | None: + """Look up a single metric value. + + ``metric_name`` is matched as a substring against family + names (mirrors the old ``get_metric_value`` behaviour). + ``labels`` must match exactly when provided. + + Returns the value as a string, or ``None``. + """ + labels = labels or {} + key = _label_key(labels) + for name, by_label in self._index.items(): + if metric_name not in name: + continue + if key in by_label: + return str(int(by_label[key])) + return None + + def delta(self, after: MetricsSnapshot) -> MetricsSnapshot: + """Return a new snapshot containing metrics.""" + delta_families: list[Metric] = [] + for fam in after._families: + before_labels = self._index.get(fam.name, {}) + delta_samples: list[Sample] = [] + for s in fam.samples: + key = _label_key(s.labels) + before_val = before_labels.get(key, 0.0) + d = s.value - before_val + delta_samples.append(Sample(s.name, s.labels, d, s.timestamp)) + if delta_samples: + m = Metric(fam.name, fam.documentation, fam.type) + m.samples = delta_samples + delta_families.append(m) + return MetricsSnapshot(delta_families) + + def to_text(self) -> str: + """Serialise to OpenMetrics text exposition format.""" + registry = CollectorRegistry() + registry.register(_FamilyCollector(self._families)) + return generate_latest(registry).decode('utf-8') + + +class _FamilyCollector: + """Minimal Collector that replays pre-built Metric objects.""" + + def __init__(self, families: list[Metric]): + self._families = families + + def collect(self) -> list[Metric]: + return self._families + + def describe(self) -> list[Metric]: + return self._families + + +def _label_key(labels: dict[str, str]) -> str: + return ','.join(f'{k}={v}' for k, v in sorted(labels.items())) + + +def get_metric_value( + fact_config: tuple[dict, str], + metric_name: str, + labels: dict[str, str] | None = None, +) -> str | None: + """Query Prometheus metrics endpoint to get the value of a metric. + + Args: + fact_config: The fact configuration tuple + (config dict, config file path). + metric_name: Name of the metric to query + (e.g., "host_scanner_scan"). + labels: Optional dict of label filters + (e.g., {"label": "InodeRemoved"}). + + Returns: + The metric value as a string if found, None otherwise. + """ + config, _ = fact_config + snapshot = MetricsSnapshot.fetch(config['endpoint']['address']) + return snapshot.get(metric_name, labels) diff --git a/tests/requirements.txt b/tests/requirements.txt index 9aad2f3c..4a25358a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,6 +2,7 @@ docker==7.1.0 grpcio==1.76.0 grpcio-tools==1.76.0 opentelemetry-proto==1.41.1 +prometheus-client==0.26.0 pytest==8.4.1 requests==2.32.4 pyyaml==6.0.3 diff --git a/tests/server.py b/tests/server.py index f274737a..033dfc94 100644 --- a/tests/server.py +++ b/tests/server.py @@ -105,6 +105,10 @@ def is_empty(self) -> bool: """Check if the internal queue of events is empty.""" return len(self.queue) == 0 + def drain(self): + """Remove all pending events from the queue.""" + self.queue.clear() + def is_running(self) -> bool: """Check if the server is currently running.""" return self.running.is_set() diff --git a/tests/test_path_rmdir.py b/tests/test_path_rmdir.py index fcefa185..bd91dffa 100644 --- a/tests/test_path_rmdir.py +++ b/tests/test_path_rmdir.py @@ -7,8 +7,8 @@ import pytest from event import Event, EventType, Process +from metrics import get_metric_value from server import EventServer -from utils import get_metric_value def get_inode_removed_count(fact_config: tuple[dict, str]): diff --git a/tests/test_rate_limit.py b/tests/test_rate_limit.py index 53bb16f5..be2797d1 100644 --- a/tests/test_rate_limit.py +++ b/tests/test_rate_limit.py @@ -6,10 +6,11 @@ import docker.models.containers import pytest -import requests import yaml +from conftest import ENDPOINT_ADDRESS from event import Event, EventType, Process +from metrics import MetricsSnapshot from server import EventServer @@ -39,12 +40,14 @@ def test_rate_limit_drops_events( server: EventServer, ): """ - Test that the rate limiter drops events when the rate limit is exceeded. + Test that the rate limiter drops events when the rate limit + is exceeded. """ - config, _ = rate_limited_config num_files = 100 - start_time = time.time() + before = MetricsSnapshot.fetch(ENDPOINT_ADDRESS) + + start_time = time.time() for i in range(num_files): fut = os.path.join(monitored_dir, f'file_{i}.txt') with open(fut, 'w') as f: @@ -67,32 +70,24 @@ def test_rate_limit_drops_events( f'but received all {received_count}' ) - metrics_response = requests.get( - f'http://{config["endpoint"]["address"]}/metrics', - ) - assert metrics_response.status_code == 200 + after = MetricsSnapshot.fetch(ENDPOINT_ADDRESS) + delta = before.delta(after) - metrics_text = metrics_response.text - assert 'rate_limiter_events' in metrics_text, ( - 'rate_limiter_events metric not found' + delta_dropped = delta.get('rate_limiter_events', {'label': 'Dropped'}) + assert delta_dropped is not None, ( + 'rate_limiter_events{label="Dropped"} metric not found' ) - - dropped_count = 0 - for line in metrics_text.split('\n'): - if 'rate_limiter_events' in line and 'label="Dropped"' in line: - parts = line.split() - if len(parts) >= 2: - dropped_count = int(parts[1]) - break + dropped_count = int(delta_dropped) assert dropped_count > 0, ( 'Expected rate limiter to report dropped events in metrics' ) total_accounted = received_count + dropped_count - assert total_accounted == num_files, ( - 'Expected rate limiter to see all events' + f'Expected received ({received_count}) + dropped ' + f'({dropped_count}) to equal {num_files}, ' + f'got {total_accounted}' ) @@ -102,13 +97,15 @@ def test_rate_limit_unlimited( fact_config: tuple[dict, str], ): """ - Test that the default config (rate_limit=0) allows all events through. + Test that the default config (rate_limit=0) allows all events + through. """ - config, _ = fact_config num_files = 20 events = [] process = Process.from_proc() + before = MetricsSnapshot.fetch(ENDPOINT_ADDRESS) + for i in range(num_files): fut = os.path.join(monitored_dir, f'file_{i}.txt') with open(fut, 'w') as f: @@ -125,20 +122,11 @@ def test_rate_limit_unlimited( server.wait_events(events) - metrics_response = requests.get( - f'http://{config["endpoint"]["address"]}/metrics', - ) - assert metrics_response.status_code == 200 - - metrics_text = metrics_response.text + after = MetricsSnapshot.fetch(ENDPOINT_ADDRESS) + delta = before.delta(after) - dropped_count = 0 - for line in metrics_text.split('\n'): - if 'rate_limiter_events' in line and 'label="Dropped"' in line: - parts = line.split() - if len(parts) >= 2: - dropped_count = int(parts[1]) - break + delta_dropped = delta.get('rate_limiter_events', {'label': 'Dropped'}) + dropped_count = int(delta_dropped) if delta_dropped else 0 assert dropped_count == 0, ( 'Expected no dropped events with unlimited ' diff --git a/tests/utils.py b/tests/utils.py index 272fa96a..8f8fe029 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,8 +3,6 @@ import os import re -import requests - def join_path_with_filename(directory: str, filename: str | bytes): """ @@ -100,42 +98,3 @@ def btf_has_symbol(symbol: str) -> bool: return False except OSError: return False - - -def get_metric_value( - fact_config: tuple[dict, str], - metric_name: str, - labels: dict[str, str] | None = None, -): - """ - Query Prometheus metrics endpoint to get the value of a metric. - - Args: - fact_config: The fact configuration tuple - (config dict, config file path). - metric_name: Name of the metric to query - (e.g., "host_scanner_scan"). - labels: Optional dict of label filters - (e.g., {"label": "InodeRemoved"}). - - Returns: - The metric value as a string if found, None otherwise. - """ - config, _ = fact_config - response = requests.get(f'http://{config["endpoint"]["address"]}/metrics') - assert response.status_code == 200 - - labels = labels or {} - - for line in response.text.split('\n'): - if metric_name not in line: - continue - - # Check if all label filters match - if all(f'{k}="{v}"' in line for k, v in labels.items()): - # Format: metric_name{label="value"} 42 - parts = line.split() - if len(parts) >= 2: - return parts[-1] - - return None