diff --git a/CHANGELOG.md b/CHANGELOG.md index c3cf55354..9462fefb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* **role:graylog_sidecar**: New role to install and configure the Graylog Sidecar log collector agent on RHEL-family hosts. It ships logs to a Graylog server and is managed centrally from the Graylog web UI via tags. +* **role:repo_graylog_sidecar**: New role deploying the Graylog Sidecar package repository (separate from the main Graylog repository), with support for custom mirrors and basic auth like the other `repo_*` roles. * **role:files**: The `files:directories`, `files:files` and `files:symlinks` tags manage one kind of file system entity each, so a single directory or symlink can be deployed without touching the rest. * **plugin:bitwarden_item**: The lookup can be told not to create secrets, so a lookup that finds no matching item aborts the run instead of silently generating a new password. Set `LFOPS_BITWARDEN_LOOKUP_ITEM_CREATE=false`, or `create = false` in the `[bitwarden_item_lookup]` section of your `ansible.cfg`; the default is the previous behaviour. diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 7db4f2a19..e1c380236 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -49,6 +49,7 @@ Which Ansible role is proven to run on which OS? | grav | | | x | (x) | (x) | | | | | | graylog_datanode | x | x | x | x | (x) | (x) | (x) | (x) | | | graylog_server | x | x | x | (x) | (x) | (x) | (x) | (x) | | +| graylog_sidecar | | | (x) | (x) | x | | | | | | haveged | (x) | (x) | x | x | (x) | (x) | (x) | (x) | | | hetzner_vm | | | | | | | | | Fedora 35+ | | hostname | (x) | (x) | x | x | x | (x) | (x) | (x) | | @@ -134,6 +135,7 @@ Which Ansible role is proven to run on which OS? | repo_gitlab_runner | | | x | (x) | (x) | | | | | | repo_grafana | x | x | x | x | (x) | (x) | (x) | (x) | | | repo_graylog | x | x | x | (x) | (x) | (x) | (x) | (x) | | +| repo_graylog_sidecar | | | (x) | (x) | x | | | | | | repo_icinga | x | x | x | x | x | x | (x) | (x) | | | repo_influxdb | x | x | x | x | (x) | (x) | (x) | (x) | | | repo_mariadb | x | x | x | x | x | (x) | x | (x) | | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd393201d..32e00f33b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -672,7 +672,7 @@ Make sure to use the following format when passing multiple injections to avoid * Use the target path for the file in the `template` folder, for example: `templates/etc/httpd/sites-available/default.conf.j2`. This makes it clear what the file is for, and avoids name collisions. * Always use the `.j2` file extension for files in the `template` folder. * If deploying self-written scripts, copy them to `/usr/local/sbin` (due to SELinux). Internal helper scripts that are only ever run by a systemd unit (not invoked by an admin and not exec'd by a confined domain) MAY instead live in `/usr/local/libexec`. Files there get the `usr_t` type, and the targeted policy lets a root `oneshot` service (which runs in `init_t`) execute them in place via `execute_no_trans`, so there is no AVC denial on RHEL/Rocky 8, 9 and 10. Keep admin-invokable commands in `/usr/local/sbin`, and never put a script a confined domain must exec under `/usr/local/libexec`. -* Keep templates as close to the original file as possible. This makes handling of rpmnew/rpmsave files easier. +* Keep templates as close to the original file as possible. This makes handling of rpmnew/rpmsave files easier. Where upstream ships a fully-commented default config (with every setting documented and commented out), start the template from that exact file and preserve its comments, ordering and commented-out defaults verbatim. Only uncomment and template the keys the role actually manages; leave everything else as the upstream comment. A package update then drops an `*.rpmnew` whose only differences from the deployed file are the handful of managed lines, so the diff is short and reviewable. See `roles/graylog_sidecar/templates/etc/graylog/sidecar/sidecar.yml.j2`. * Add the following task after deploying a file that might get rpmnew or rpmsave files (or their Debian equivalents): ```yaml - name: 'Remove rpmnew / rpmsave (and Debian equivalents)' diff --git a/extensions/molecule/graylog_sidecar/converge.yml b/extensions/molecule/graylog_sidecar/converge.yml new file mode 100644 index 000000000..8747d9e14 --- /dev/null +++ b/extensions/molecule/graylog_sidecar/converge.yml @@ -0,0 +1,2 @@ +- name: 'Converge graylog_sidecar playbook' + ansible.builtin.import_playbook: 'linuxfabrik.lfops.graylog_sidecar' diff --git a/extensions/molecule/graylog_sidecar/inventory/group_vars/systems_under_test.yml b/extensions/molecule/graylog_sidecar/inventory/group_vars/systems_under_test.yml new file mode 100644 index 000000000..c38817c22 --- /dev/null +++ b/extensions/molecule/graylog_sidecar/inventory/group_vars/systems_under_test.yml @@ -0,0 +1,21 @@ +# Variables the playbook under test needs, applied to every system under test. + +# Mandatory: the repository is versioned independently of the Graylog server. +repo_graylog_sidecar__version: '1.5' + +# Mandatory: the sidecar needs a server to talk to. No Graylog server runs in this scenario, so +# the URL points at a documentation domain (RFC 2606). The sidecar starts anyway and retries, +# which is what verify.yml uses to prove it read the configured value. +graylog_sidecar__server_url: 'http://graylog.example.com:9000/api/' +graylog_sidecar__server_api_token: 'linuxfabrik' + +# Optional, set to a non-default so the scenario exercises the templating rather than only the +# shipped defaults. +graylog_sidecar__tags: + - 'linuxfabrik-molecule' +graylog_sidecar__update_interval: 30 + +# Narrowed to the collector the RPM actually ships, which is what a real deployment should do. +# Also exercises the list rendering with a value other than the shipped default. +graylog_sidecar__collector_binaries_accesslist: + - '/usr/lib/graylog-sidecar/filebeat' diff --git a/extensions/molecule/graylog_sidecar/inventory/hosts.yml b/extensions/molecule/graylog_sidecar/inventory/hosts.yml new file mode 100644 index 000000000..81c367a37 --- /dev/null +++ b/extensions/molecule/graylog_sidecar/inventory/hosts.yml @@ -0,0 +1,15 @@ +# yamllint disable rule:empty-values + +# Scenario inventory, layered on top of the shared inventory +# (extensions/molecule/inventory) through the two --inventory flags in config.yml. + +lfops_graylog_sidecar: + children: + systems_under_test: + +# The Graylog Sidecar repository only ships RPMs, so the role is RHEL-family only. +systems_under_test: + hosts: + rocky8-vm: + rocky9-vm: + rocky10-vm: diff --git a/extensions/molecule/graylog_sidecar/molecule.yml b/extensions/molecule/graylog_sidecar/molecule.yml new file mode 100644 index 000000000..4d424040b --- /dev/null +++ b/extensions/molecule/graylog_sidecar/molecule.yml @@ -0,0 +1,8 @@ +# Molecule scenario marker. Required even when empty, so that Molecule discovers the scenario. +# Everything this scenario needs comes from extensions/molecule/config.yml; nothing is overridden +# here. +# +# The scenario converges without a Graylog server in the test setup. That is deliberate: the +# sidecar keeps running and retries when the server is unreachable, so the whole role is still +# exercised, and the retry attempts in the journal prove the running process picked up the +# configured server_url. diff --git a/extensions/molecule/graylog_sidecar/verify.yml b/extensions/molecule/graylog_sidecar/verify.yml new file mode 100644 index 000000000..9b15a8169 --- /dev/null +++ b/extensions/molecule/graylog_sidecar/verify.yml @@ -0,0 +1,67 @@ +# verify.yml runs after converge, and again after the idempotence step (see the test_sequence in +# config.yml). It checks the observable end result with read-only modules, never by re-running +# the role. +# +# The two checks worth having here are the systemd unit and the journal. The package ships no +# unit file at all (https://github.com/Graylog2/collector-sidecar/issues/225), so the unit only +# exists if the role's `graylog-sidecar -service install` task really ran. And since the sidecar +# has no local status endpoint, the journal is the only place where the running process reveals +# which server_url it was configured with. +- name: 'Verify the graylog_sidecar playbook installed a running sidecar' + hosts: 'systems_under_test' + gather_facts: false + tasks: + + # Floor check: the package can only come from the repository that repo_graylog_sidecar + # deployed, so this covers both roles. + - name: 'Gather the package facts' + ansible.builtin.package_facts: # yamllint disable-line rule:empty-values + + - name: 'Assert that the graylog-sidecar package is installed' + ansible.builtin.assert: + that: '"graylog-sidecar" in ansible_facts["packages"]' + + # The unit is generated by the sidecar binary, not shipped by the package. Without the role's + # `-service install` task there would be nothing to start. + - name: 'stat /etc/systemd/system/graylog-sidecar.service' + ansible.builtin.stat: + path: '/etc/systemd/system/graylog-sidecar.service' + register: '__molecule__graylog_sidecar_unit_stat_result' + + - name: 'Assert that the systemd unit was registered' + ansible.builtin.assert: + that: '__molecule__graylog_sidecar_unit_stat_result["stat"]["exists"]' + + # sidecar.yml holds the Graylog API token, so its mode is a security property of the role, + # not a detail of the template module. The package ships the file 0600 and the role must not + # widen that. + - name: 'stat /etc/graylog/sidecar/sidecar.yml' + ansible.builtin.stat: + path: '/etc/graylog/sidecar/sidecar.yml' + register: '__molecule__graylog_sidecar_config_stat_result' + + - name: 'Assert that the config holding the API token is not readable by other users' + ansible.builtin.assert: + that: '__molecule__graylog_sidecar_config_stat_result["stat"]["mode"] == "0600"' + + - name: 'Gather the service facts' + ansible.builtin.service_facts: # yamllint disable-line rule:empty-values + + - name: 'Assert that graylog-sidecar.service is enabled and running' + ansible.builtin.assert: + that: + - 'ansible_facts["services"]["graylog-sidecar.service"]["status"] == "enabled"' + - 'ansible_facts["services"]["graylog-sidecar.service"]["state"] == "running"' + + # The important check: the running process uses the configured server_url. The sidecar logs + # to the journal (/var/log/graylog-sidecar/ stays empty under systemd) and reports every + # contact attempt, including the URL it took from the templated sidecar.yml. This proves the + # whole chain: group_vars -> template -> service reads the file -> behaviour. + - name: 'journalctl --unit graylog-sidecar.service' + ansible.builtin.command: 'journalctl --unit=graylog-sidecar.service --no-pager' + changed_when: false + register: '__molecule__graylog_sidecar_journal_result' + + - name: 'Assert that the sidecar contacts the configured Graylog server' + ansible.builtin.assert: + that: '"graylog.example.com:9000" in __molecule__graylog_sidecar_journal_result["stdout"]' diff --git a/playbooks/README.md b/playbooks/README.md index a17fbc301..bbdbb5046 100644 --- a/playbooks/README.md +++ b/playbooks/README.md @@ -373,6 +373,14 @@ Calls the following roles (in order): * [grafana_grizzly](https://github.com/Linuxfabrik/lfops/tree/main/roles/grafana_grizzly) +## graylog_sidecar.yml + +Calls the following roles (in order): + +* [repo_graylog_sidecar](https://github.com/Linuxfabrik/lfops/tree/main/roles/repo_graylog_sidecar): `graylog_sidecar__skip_repo_graylog_sidecar` +* [graylog_sidecar](https://github.com/Linuxfabrik/lfops/tree/main/roles/graylog_sidecar) + + ## haveged.yml Calls the following roles (in order): @@ -900,6 +908,13 @@ Calls the following roles (in order): * [repo_graylog](https://github.com/Linuxfabrik/lfops/tree/main/roles/repo_graylog) +## repo_graylog_sidecar.yml + +Calls the following roles (in order): + +* [repo_graylog_sidecar](https://github.com/Linuxfabrik/lfops/tree/main/roles/repo_graylog_sidecar) + + ## repo_icinga.yml Calls the following roles (in order): diff --git a/playbooks/all.yml b/playbooks/all.yml index 5e80cda73..b8637b290 100644 --- a/playbooks/all.yml +++ b/playbooks/all.yml @@ -40,6 +40,7 @@ - import_playbook: 'glpi_agent.yml' - import_playbook: 'grafana.yml' - import_playbook: 'grafana_grizzly.yml' +- import_playbook: 'graylog_sidecar.yml' - import_playbook: 'haveged.yml' - import_playbook: 'hetzner_vm.yml' - import_playbook: 'hostname.yml' @@ -107,6 +108,7 @@ - import_playbook: 'repo_gitlab_runner.yml' - import_playbook: 'repo_grafana.yml' - import_playbook: 'repo_graylog.yml' +- import_playbook: 'repo_graylog_sidecar.yml' - import_playbook: 'repo_icinga.yml' - import_playbook: 'repo_influxdb.yml' - import_playbook: 'repo_mariadb.yml' diff --git a/playbooks/graylog_sidecar.yml b/playbooks/graylog_sidecar.yml new file mode 100644 index 000000000..21cb4f3de --- /dev/null +++ b/playbooks/graylog_sidecar.yml @@ -0,0 +1,33 @@ +- name: 'Playbook linuxfabrik.lfops.graylog_sidecar' + hosts: + - 'lfops_graylog_sidecar' + + pre_tasks: + - ansible.builtin.import_role: + name: 'shared' + tasks_from: 'log-start.yml' + tags: + - 'always' + + - ansible.builtin.import_role: + name: 'shared' + tasks_from: 'global-variables.yml' + tags: + - 'always' + + + roles: + + - role: 'linuxfabrik.lfops.repo_graylog_sidecar' + when: + - 'not graylog_sidecar__skip_repo_graylog_sidecar | d(false) | bool' + + - role: 'linuxfabrik.lfops.graylog_sidecar' + + + post_tasks: + - ansible.builtin.import_role: + name: 'shared' + tasks_from: 'log-end.yml' + tags: + - 'always' diff --git a/playbooks/repo_graylog_sidecar.yml b/playbooks/repo_graylog_sidecar.yml new file mode 100644 index 000000000..ae54f5bc4 --- /dev/null +++ b/playbooks/repo_graylog_sidecar.yml @@ -0,0 +1,29 @@ +- name: 'Playbook linuxfabrik.lfops.repo_graylog_sidecar' + hosts: + - 'lfops_repo_graylog_sidecar' + + pre_tasks: + - ansible.builtin.import_role: + name: 'shared' + tasks_from: 'log-start.yml' + tags: + - 'always' + + - ansible.builtin.import_role: + name: 'shared' + tasks_from: 'global-variables.yml' + tags: + - 'always' + + + roles: + + - role: 'linuxfabrik.lfops.repo_graylog_sidecar' + + + post_tasks: + - ansible.builtin.import_role: + name: 'shared' + tasks_from: 'log-end.yml' + tags: + - 'always' diff --git a/roles/graylog_sidecar/README.md b/roles/graylog_sidecar/README.md new file mode 100644 index 000000000..ab210c958 --- /dev/null +++ b/roles/graylog_sidecar/README.md @@ -0,0 +1,166 @@ +# Ansible Role linuxfabrik.lfops.graylog_sidecar + +This role installs and configures the [Graylog Sidecar](https://go2docs.graylog.org/current/getting_in_log_data/install_sidecar_on_linux.htm). The Sidecar is a lightweight agent that runs on log-producing hosts, fetches collector configurations from a Graylog server and supervises the log collector backends (Filebeat etc.) that ship the logs. + + +*Available in the next LFOps release.* + + +## How the Role Behaves + +* The configuration file `/etc/graylog/sidecar/sidecar.yml` is fully templated. On every run it is re-rendered from the role's template (a timestamped backup is kept), so out-of-band manual edits are overwritten. Manage all settings through the role variables below. +* A configuration change notifies a handler that restarts `graylog-sidecar.service`. The restart is skipped when the service was just started in the same run (redundant), when `graylog_sidecar__service_state` is `stopped`, or when restarts are deferred LFOps-wide via `lfops__skip_restart_handlers`. +* The systemd unit is registered once via `graylog-sidecar -service install`. The package ships no unit file, the binary generates `/etc/systemd/system/graylog-sidecar.service` and enables it. The task is guarded so it runs only when the unit file is missing, because a second call fails instead of doing nothing. As a side effect of the generator enabling the unit, a run limited to `--tags graylog_sidecar:configure` leaves the service enabled even when `graylog_sidecar__service_enabled` is `false`; a full run corrects this afterwards. +* This role does not install or configure collector backends and does not assign collector configurations. Manage those in the Graylog web UI under *System/Sidecars*, matching them to the sidecar via `graylog_sidecar__tags`. What a host actually ships is therefore decided on the server, not in this role's variables. +* The sidecar runs as `root` and executes the collector binaries the server tells it to. `graylog_sidecar__collector_binaries_accesslist` is the boundary for that trust: only listed paths may be executed. Keep it as tight as your collectors allow. +* `/etc/graylog/sidecar/sidecar.yml` contains `graylog_sidecar__server_api_token` and is deployed `0600 root:root`. The timestamped backups the role keeps next to it inherit those permissions. + + +## Dependent Roles + +Any [LFOps playbook](https://github.com/Linuxfabrik/lfops/blob/main/playbooks/README.md) that installs this role runs these for you. Optional ones can be disabled via the playbook's skip variables. + +* The Graylog Sidecar repository must be enabled (role: [linuxfabrik.lfops.repo_graylog_sidecar](https://github.com/Linuxfabrik/lfops/tree/main/roles/repo_graylog_sidecar)). The `graylog-sidecar` package is served from it. + + +## Requirements + +* A reachable Graylog server. + +Manual steps: + +* Create an API token for the sidecar in the Graylog web UI under *System/Sidecars* and store it in your inventory as `graylog_sidecar__server_api_token`. + + +## Tags + +`graylog_sidecar` + +* Installs the `graylog-sidecar` package, deploys the configuration and ensures the service is in the desired state. +* Triggers: graylog-sidecar.service restart. + +`graylog_sidecar:configure` + +* Deploys the configuration file and registers the systemd unit. +* Triggers: graylog-sidecar.service restart. + +`graylog_sidecar:state` + +* Manages the service state (start, stop, enable, disable). +* Triggers: none. + + +## Mandatory Role Variables + +`graylog_sidecar__server_api_token` + +* The API token to authenticate against the Graylog server API. Create one in the Graylog web UI under *System/Sidecars*. +* Type: String. + +`graylog_sidecar__server_url` + +* The URL to the Graylog server API. +* Type: String. + +Example: +```yaml +# mandatory +graylog_sidecar__server_api_token: 'linuxfabrik' +graylog_sidecar__server_url: 'http://graylog.example.com:9000/api/' +``` + + +## Optional Role Variables + +`graylog_sidecar__collector_binaries_accesslist` + +* Absolute paths of the binaries the Graylog server is allowed to make this sidecar execute. Wildcards follow [Go's `filepath.Match`](https://pkg.go.dev/path/filepath#Match). The default mirrors the sidecar's own platform default for Linux, so the role changes nothing about which collectors may run; it only makes the list visible and configurable. Narrow it to the collectors you actually use. An empty list disables the check altogether, which lets anyone able to edit a collector configuration in the Graylog web UI execute an arbitrary binary as `root` on every host running this sidecar. +* Type: List of strings. +* Default: + + ```yaml + - '/usr/bin/auditbeat' + - '/usr/bin/filebeat' + - '/usr/bin/heartbeat' + - '/usr/bin/journalbeat' + - '/usr/bin/metricbeat' + - '/usr/bin/nxlog' + - '/usr/bin/packetbeat' + - '/usr/lib/graylog-sidecar/auditbeat' + - '/usr/lib/graylog-sidecar/filebeat' + - '/usr/share/auditbeat/bin/auditbeat' + - '/usr/share/filebeat/bin/filebeat' + - '/usr/share/heartbeat/bin/heartbeat' + - '/usr/share/journalbeat/bin/journalbeat' + - '/usr/share/metricbeat/bin/metricbeat' + - '/usr/share/packetbeat/bin/packetbeat' + - '/opt/nxlog/bin/nxlog' + ``` + +`graylog_sidecar__node_name` + +* The node name of the sidecar. If empty, the sidecar uses the hostname of the host it runs on. +* Type: String. +* Default: `''` + +`graylog_sidecar__send_status` + +* Transmit detailed sidecar information like collector statuses, metrics and log file lists. Disabling it reduces load on the Graylog server but disables some features in the server UI. +* Type: Bool. +* Default: `true` + +`graylog_sidecar__service_enabled` + +* Enables or disables the service, analogous to `systemctl enable/disable`. +* Type: Bool. +* Default: `true` + +`graylog_sidecar__service_state` + +* Changes the state of the service, analogous to `systemctl start/stop/restart/reload`. +* Type: String. One of `reloaded`, `restarted`, `started`, `stopped`. +* Default: `'started'` + +`graylog_sidecar__tags` + +* Tags to assign to this sidecar. Collector configurations matching any of these tags are automatically applied to the sidecar. Defaults to `default`, the tag of the collector configuration Graylog ships out of the box, so a freshly deployed sidecar starts collecting without further setup. Set to an empty list to register the sidecar without applying any configuration. +* Type: List of strings. +* Default: `['default']` + +`graylog_sidecar__tls_skip_verify` + +* Skip the verification of TLS connections to the Graylog server. +* Type: Bool. +* Default: `false` + +`graylog_sidecar__update_interval` + +* How often, in seconds, the sidecar contacts the Graylog server for keep-alive and configuration update requests. +* Type: Number. +* Default: `10` + +Example: +```yaml +# optional +graylog_sidecar__collector_binaries_accesslist: + - '/usr/lib/graylog-sidecar/filebeat' +graylog_sidecar__node_name: 'web01' +graylog_sidecar__send_status: true +graylog_sidecar__service_enabled: true +graylog_sidecar__service_state: 'started' +graylog_sidecar__tags: + - 'linux' + - 'webserver' +graylog_sidecar__tls_skip_verify: false +graylog_sidecar__update_interval: 10 +``` + + +## License + +[The Unlicense](https://unlicense.org/) + + +## Author Information + +[Linuxfabrik GmbH, Zurich](https://www.linuxfabrik.ch) diff --git a/roles/graylog_sidecar/defaults/main.yml b/roles/graylog_sidecar/defaults/main.yml new file mode 100644 index 000000000..fd3d10cbe --- /dev/null +++ b/roles/graylog_sidecar/defaults/main.yml @@ -0,0 +1,25 @@ +graylog_sidecar__collector_binaries_accesslist: + - '/usr/bin/auditbeat' + - '/usr/bin/filebeat' + - '/usr/bin/heartbeat' + - '/usr/bin/journalbeat' + - '/usr/bin/metricbeat' + - '/usr/bin/nxlog' + - '/usr/bin/packetbeat' + - '/usr/lib/graylog-sidecar/auditbeat' + - '/usr/lib/graylog-sidecar/filebeat' + - '/usr/share/auditbeat/bin/auditbeat' + - '/usr/share/filebeat/bin/filebeat' + - '/usr/share/heartbeat/bin/heartbeat' + - '/usr/share/journalbeat/bin/journalbeat' + - '/usr/share/metricbeat/bin/metricbeat' + - '/usr/share/packetbeat/bin/packetbeat' + - '/opt/nxlog/bin/nxlog' +graylog_sidecar__node_name: '' +graylog_sidecar__send_status: true +graylog_sidecar__service_enabled: true +graylog_sidecar__service_state: 'started' +graylog_sidecar__tags: + - 'default' +graylog_sidecar__tls_skip_verify: false +graylog_sidecar__update_interval: 10 diff --git a/roles/graylog_sidecar/handlers/main.yml b/roles/graylog_sidecar/handlers/main.yml new file mode 100644 index 000000000..813b996e6 --- /dev/null +++ b/roles/graylog_sidecar/handlers/main.yml @@ -0,0 +1,14 @@ +# skip restart if the operator deferred restarts LFOps-wide (lfops__skip_restart_handlers), if +# the service was just started (redundant), or if the user wants it stopped. +# __graylog_sidecar__service_state_result is registered on a dedicated state-only task (not the +# enabled task), so that enabled-only changes don't prevent the restart. +# `is not defined` covers a run that skips the state block (`--skip-tags graylog_sidecar:state`), +# where the result is never registered. +- name: 'graylog_sidecar: restart graylog-sidecar' + ansible.builtin.service: + name: 'graylog-sidecar.service' + state: 'restarted' + when: + - 'not (lfops__skip_restart_handlers | d(false) | bool)' + - '__graylog_sidecar__service_state_result is not defined or __graylog_sidecar__service_state_result is not changed' + - 'graylog_sidecar__service_state != "stopped"' diff --git a/roles/graylog_sidecar/meta/argument_specs.yml b/roles/graylog_sidecar/meta/argument_specs.yml new file mode 100644 index 000000000..2bcd1cbc8 --- /dev/null +++ b/roles/graylog_sidecar/meta/argument_specs.yml @@ -0,0 +1,78 @@ +# argument_specs validates required variables and types automatically at role entry. +# use this for simple "is defined" / type checks. for complex validations +# (value ranges, cross-variable logic), use ansible.builtin.assert in the tasks. +argument_specs: + main: + options: + + graylog_sidecar__collector_binaries_accesslist: + type: 'list' + elements: 'str' + required: false + description: >- + Absolute paths of the binaries the Graylog server is allowed to make + this sidecar execute. Supports the wildcards of Go's filepath.Match. + An empty list disables the check. + + graylog_sidecar__node_name: + type: 'str' + required: false + default: '' + description: 'The node name of the sidecar. Empty means use the hostname.' + + graylog_sidecar__send_status: + type: 'bool' + required: false + default: true + description: 'Transmit detailed sidecar information (collector statuses, metrics, log file lists).' + + graylog_sidecar__server_api_token: + type: 'str' + required: true + description: 'The API token to authenticate against the Graylog server API.' + + graylog_sidecar__server_url: + type: 'str' + required: true + description: 'The URL to the Graylog server API, e.g. "http://graylog.example.com:9000/api/".' + + graylog_sidecar__service_enabled: + type: 'bool' + required: false + default: true + description: 'Enables or disables the service at boot.' + + graylog_sidecar__service_state: + type: 'str' + required: false + default: 'started' + choices: + - 'reloaded' + - 'restarted' + - 'started' + - 'stopped' + description: 'Desired runtime state of the graylog-sidecar service.' + + graylog_sidecar__tags: + type: 'list' + elements: 'str' + required: false + default: + - 'default' + description: >- + Tags to assign to this sidecar for collector configuration matching. + Defaults to "default", the tag of the collector configuration Graylog + ships out of the box. Set to an empty list to register the sidecar + without applying any configuration. + + graylog_sidecar__tls_skip_verify: + type: 'bool' + required: false + default: false + description: 'Skip the verification of TLS connections.' + + graylog_sidecar__update_interval: + type: 'int' + required: false + default: 10 + description: 'The update interval in seconds.' diff --git a/roles/graylog_sidecar/tasks/main.yml b/roles/graylog_sidecar/tasks/main.yml new file mode 100644 index 000000000..9202015e0 --- /dev/null +++ b/roles/graylog_sidecar/tasks/main.yml @@ -0,0 +1,65 @@ +- block: + + - name: 'Install graylog-sidecar' + ansible.builtin.package: + name: + - 'graylog-sidecar' + state: 'present' + + tags: + - 'graylog_sidecar' + + +- block: + + - name: 'Deploy /etc/graylog/sidecar/sidecar.yml' + ansible.builtin.template: + backup: true + src: 'etc/graylog/sidecar/sidecar.yml.j2' + dest: '/etc/graylog/sidecar/sidecar.yml' + owner: 'root' + group: 'root' + mode: 0o600 # contains graylog_sidecar__server_api_token; the package ships it 0600 too + notify: 'graylog_sidecar: restart graylog-sidecar' + + - name: 'Remove rpmnew / rpmsave (and Debian equivalents)' + ansible.builtin.include_role: + name: 'shared' + tasks_from: 'remove-rpmnew-rpmsave.yml' + vars: + shared__remove_rpmnew_rpmsave_config_file: '/etc/graylog/sidecar/sidecar.yml' + + # the package ships no systemd unit at all; the binary generates one (and enables it). + # upstream asked for the package to do this in 2018 and it is still open: + # https://github.com/Graylog2/collector-sidecar/issues/225 + # the call is not idempotent either - a second run exits 1 with "Init already exists" - + # hence the `creates` guard. + - name: 'graylog-sidecar -service install' + ansible.builtin.command: 'graylog-sidecar -service install' + args: + creates: '/etc/systemd/system/graylog-sidecar.service' + + tags: + - 'graylog_sidecar' + - 'graylog_sidecar:configure' + + +- block: + + # split enabled and state into separate tasks so that the register on the state task + # only reflects actual state changes (started/stopped), not enabled/disabled changes. + # this lets the restart handler correctly skip a restart when the service was just started. + - name: 'systemctl {{ graylog_sidecar__service_enabled | bool | ternary("enable", "disable") }} graylog-sidecar.service' + ansible.builtin.service: + name: 'graylog-sidecar.service' + enabled: '{{ graylog_sidecar__service_enabled | bool }}' + + - name: 'systemctl {{ graylog_sidecar__service_state | regex_replace("p?ed$", "") }} graylog-sidecar.service' + ansible.builtin.service: + name: 'graylog-sidecar.service' + state: '{{ graylog_sidecar__service_state }}' + register: '__graylog_sidecar__service_state_result' + + tags: + - 'graylog_sidecar' + - 'graylog_sidecar:state' diff --git a/roles/graylog_sidecar/templates/etc/graylog/sidecar/sidecar.yml.j2 b/roles/graylog_sidecar/templates/etc/graylog/sidecar/sidecar.yml.j2 new file mode 100644 index 000000000..41ce94e66 --- /dev/null +++ b/roles/graylog_sidecar/templates/etc/graylog/sidecar/sidecar.yml.j2 @@ -0,0 +1,103 @@ +# {{ ansible_managed }} +# 2026081101 + +# The URL to the Graylog server API. +server_url: "{{ graylog_sidecar__server_url }}" + +# The API token to use to authenticate against the Graylog server API. +# This field is mandatory +server_api_token: "{{ graylog_sidecar__server_api_token }}" + +# The node ID of the sidecar. This can be a path to a file or an ID string. +# If set to a file and the file doesn't exist, the sidecar will generate an +# unique ID and writes it to the configured path. +# +# Example file path: "file:/etc/graylog/sidecar/node-id" +# Example ID string: "6033137e-d56b-47fc-9762-cd699c11a5a9" +# +# ATTENTION: Every sidecar instance needs a unique ID! +# +#node_id: "file:/etc/graylog/sidecar/node-id" + +# The node name of the sidecar. If this is empty, the sidecar will use the +# hostname of the host it is running on. +{% if graylog_sidecar__node_name | length > 0 %} +node_name: "{{ graylog_sidecar__node_name }}" +{% else %} +#node_name: "" +{% endif %} + +# The update interval in seconds. This configures how often the sidecar will +# contact the Graylog server for keep-alive and configuration update requests. +update_interval: {{ graylog_sidecar__update_interval }} + +# This configures if the sidecar should skip the verification of TLS connections. +# Default: false +tls_skip_verify: {{ graylog_sidecar__tls_skip_verify | bool | lower }} + +# This enables/disables the transmission of detailed sidecar information like +# collector statues, metrics and log file lists. It can be disabled to reduce +# load on the Graylog server if needed. (disables some features in the server UI) +send_status: {{ graylog_sidecar__send_status | bool | lower }} + +# A list of directories to scan for log files. The sidecar will scan each +# directory for log files and submits them to the server on each update. +# +# Example: +# list_log_files: +# - "/var/log/nginx" +# - "/opt/app/logs" +# +# Default: empty list +#list_log_files: [] + +# Directory where the sidecar stores internal data. +#cache_path: "/var/cache/graylog-sidecar" + +# Directory where the sidecar stores logs for collectors and the sidecar itself. +#log_path: "/var/log/graylog-sidecar" + +# The maximum size of the log file before it gets rotated. +#log_rotate_max_file_size: "10MiB" + +# The maximum number of old log files to retain. +#log_rotate_keep_files: 10 + +# How long to wait for the config validation command. +#collector_validation_timeout: "1m" + +# How long to wait for the collector to gracefully shutdown. +# After this timeout the sidecar tries to terminate the collector with SIGKILL +#collector_shutdown_timeout: "10s" + +# Directory where the sidecar generates configurations for collectors. +#collector_configuration_directory: "/var/lib/graylog-sidecar/generated" + +# A list of tags to assign to this sidecar. Collector configuration matching any of these tags will automatically be +# applied to the sidecar. +{% if graylog_sidecar__tags | length > 0 %} +tags: +{% for tag in graylog_sidecar__tags %} + - "{{ tag }}" +{% endfor %} +{% else %} +tags: [] +{% endif %} + +# A list of binaries which are allowed to be executed by the Sidecar. An empty list disables the access list feature. +# Wildcards can be used, for a full pattern description see https://golang.org/pkg/path/filepath/#Match +# Example: +# collector_binaries_accesslist: +# - "/usr/bin/filebeat" +# - "/opt/collectors/*" +# +# Example disable access listing: +# collector_binaries_accesslist: [] +{% if graylog_sidecar__collector_binaries_accesslist | length > 0 %} +collector_binaries_accesslist: +{% for binary in graylog_sidecar__collector_binaries_accesslist %} + - "{{ binary }}" +{% endfor %} +{% else %} +collector_binaries_accesslist: [] +{% endif %} diff --git a/roles/repo_graylog_sidecar/README.md b/roles/repo_graylog_sidecar/README.md new file mode 100644 index 000000000..fe9ab6512 --- /dev/null +++ b/roles/repo_graylog_sidecar/README.md @@ -0,0 +1,62 @@ +# Ansible Role linuxfabrik.lfops.repo_graylog_sidecar + +This role deploys the [Graylog Sidecar Repository](https://go2docs.graylog.org/current/getting_in_log_data/install_sidecar_on_linux.htm). The Graylog Sidecar is served from a dedicated repository, separate from the main Graylog repository deployed by [linuxfabrik.lfops.repo_graylog](https://github.com/Linuxfabrik/lfops/tree/main/roles/repo_graylog). Although the Graylog project ships a repository package for this, LFOps prefers to install the repository configuration manually, because it's the only way to handle custom mirror servers. + + +*Available in the next LFOps release.* + + +## Tags + +`repo_graylog_sidecar` + +* Deploys the Graylog Sidecar Repository. +* Triggers: none. + + +## Mandatory Role Variables + +`repo_graylog_sidecar__version` + +* The Graylog Sidecar repo version to install, in `'major.minor'` format. The Sidecar repository is versioned independently of the Graylog server. One of `'1.0'` through `'1.5'` as of 2026-06. [See the Graylog Sidecar releases for a current list](https://github.com/Graylog2/collector-sidecar/releases). +* Type: String. + +Example: +```yaml +# mandatory +repo_graylog_sidecar__version: '1.5' +``` + + +## Optional Role Variables + +`repo_graylog_sidecar__basic_auth_login` + +* Use HTTP basic auth to login to the repository. Only takes effect together with a custom mirror URL; the default public repositories do not use basic auth. Defaults to `lfops__repo_basic_auth_login`, making it easy to set this for all `repo_*` roles. +* Type: String. +* Default: `'{{ lfops__repo_basic_auth_login | default("") }}'` + +`repo_graylog_sidecar__mirror_url` + +* Set the URL to a custom mirror server providing the repository. Defaults to `lfops__repo_mirror_url` to allow easily setting the same URL for all `repo_*` roles. If `lfops__repo_mirror_url` is not set, the default mirrors of the repo are used. +* Type: String. +* Default: `'{{ lfops__repo_mirror_url | default("") }}'` + +Example: +```yaml +# optional +repo_graylog_sidecar__basic_auth_login: + username: 'my-username' + password: 'linuxfabrik' +repo_graylog_sidecar__mirror_url: 'https://mirror.example.com' +``` + + +## License + +[The Unlicense](https://unlicense.org/) + + +## Author Information + +[Linuxfabrik GmbH, Zurich](https://www.linuxfabrik.ch) diff --git a/roles/repo_graylog_sidecar/defaults/main.yml b/roles/repo_graylog_sidecar/defaults/main.yml new file mode 100644 index 000000000..837f6c015 --- /dev/null +++ b/roles/repo_graylog_sidecar/defaults/main.yml @@ -0,0 +1,2 @@ +repo_graylog_sidecar__basic_auth_login: '{{ lfops__repo_basic_auth_login | default("") }}' +repo_graylog_sidecar__mirror_url: '{{ lfops__repo_mirror_url | default("") }}' diff --git a/roles/repo_graylog_sidecar/meta/argument_specs.yml b/roles/repo_graylog_sidecar/meta/argument_specs.yml new file mode 100644 index 000000000..71efdab5c --- /dev/null +++ b/roles/repo_graylog_sidecar/meta/argument_specs.yml @@ -0,0 +1,31 @@ +argument_specs: + main: + options: + + repo_graylog_sidecar__basic_auth_login: + # 'raw' rather than 'dict', because the default in defaults/main.yml + # resolves to '' (empty string) when lfops__repo_basic_auth_login is + # not set; a strict 'dict' spec would reject the empty default. + type: 'raw' + required: false + description: >- + HTTP basic auth credentials for the Graylog Sidecar repository. + Expected as a dict with `username` and `password` keys. Typically + fed by `linuxfabrik.lfops.bitwarden_item`, which returns the full + Bitwarden item with additional keys. + + repo_graylog_sidecar__mirror_url: + type: 'str' + required: false + description: >- + URL of a custom mirror server providing the repository. Defaults + to `lfops__repo_mirror_url`; if that is also unset, the default + upstream mirrors are used. + + repo_graylog_sidecar__version: + type: 'str' + required: true + description: >- + The Graylog Sidecar repo version to install, in `major.minor` + format. The Sidecar repository is versioned independently of the + Graylog server. diff --git a/roles/repo_graylog_sidecar/tasks/RedHat.yml b/roles/repo_graylog_sidecar/tasks/RedHat.yml new file mode 100644 index 000000000..444ccc48b --- /dev/null +++ b/roles/repo_graylog_sidecar/tasks/RedHat.yml @@ -0,0 +1,38 @@ +- block: + + # the same pubkey signs all Graylog repositories, served from the debian path + - name: 'curl https://packages.graylog2.org/repo/debian/pubkey.gpg --output /tmp/ansible.RPM-GPG-KEY-graylog-sidecar' + ansible.builtin.get_url: + url: 'https://packages.graylog2.org/repo/debian/pubkey.gpg' + dest: '/tmp/ansible.RPM-GPG-KEY-graylog-sidecar' + mode: 0o644 + delegate_to: 'localhost' + become: false + changed_when: false # not an actual config change on the server + check_mode: false # run task even if `--check` is specified + + - name: 'copy /tmp/ansible.RPM-GPG-KEY-graylog-sidecar to remote host' + ansible.builtin.copy: + src: '/tmp/ansible.RPM-GPG-KEY-graylog-sidecar' + dest: '/etc/pki/rpm-gpg/RPM-GPG-KEY-graylog-sidecar' + mode: 0o644 + + # https://go2docs.graylog.org/current/getting_in_log_data/install_sidecar_on_linux.htm + - name: 'Deploy the graylog-sidecar repo (mirror: {{ repo_graylog_sidecar__mirror_url }})' + ansible.builtin.template: + backup: true + src: 'etc/yum.repos.d/graylog-sidecar.repo.j2' + dest: '/etc/yum.repos.d/graylog-sidecar.repo' + owner: 'root' + group: 'root' + mode: 0o644 + + - name: 'Remove rpmnew / rpmsave (and Debian equivalents)' + ansible.builtin.include_role: + name: 'shared' + tasks_from: 'remove-rpmnew-rpmsave.yml' + vars: + shared__remove_rpmnew_rpmsave_config_file: '/etc/yum.repos.d/graylog-sidecar.repo' + + tags: + - 'repo_graylog_sidecar' diff --git a/roles/repo_graylog_sidecar/tasks/main.yml b/roles/repo_graylog_sidecar/tasks/main.yml new file mode 100644 index 000000000..4f290d1fb --- /dev/null +++ b/roles/repo_graylog_sidecar/tasks/main.yml @@ -0,0 +1,18 @@ +- name: 'Perform platform/version specific tasks' + ansible.builtin.include_tasks: '{{ __task_file }}' + when: '__task_file | length > 0' + vars: + __task_file: '{{ lookup("ansible.builtin.first_found", __first_found_options) }}' + __first_found_options: + files: + - '{{ ansible_facts["distribution"] }}{{ ansible_facts["distribution_version"] }}.yml' + - '{{ ansible_facts["distribution"] }}{{ ansible_facts["distribution_major_version"] }}.yml' + - '{{ ansible_facts["distribution"] }}.yml' + - '{{ ansible_facts["os_family"] }}{{ ansible_facts["distribution_version"] }}.yml' + - '{{ ansible_facts["os_family"] }}{{ ansible_facts["distribution_major_version"] }}.yml' + - '{{ ansible_facts["os_family"] }}.yml' + paths: + - '{{ role_path }}/tasks' + skip: true + tags: + - 'always' diff --git a/roles/repo_graylog_sidecar/templates/etc/yum.repos.d/graylog-sidecar.repo.j2 b/roles/repo_graylog_sidecar/templates/etc/yum.repos.d/graylog-sidecar.repo.j2 new file mode 100644 index 000000000..6ef18e228 --- /dev/null +++ b/roles/repo_graylog_sidecar/templates/etc/yum.repos.d/graylog-sidecar.repo.j2 @@ -0,0 +1,18 @@ +# {{ ansible_managed }} +# 2026052801 + +# https://go2docs.graylog.org/current/getting_in_log_data/install_sidecar_on_linux.htm +[graylog-sidecar] +name=graylog-sidecar +{% if repo_graylog_sidecar__mirror_url is defined and repo_graylog_sidecar__mirror_url | length %} +baseurl={{ repo_graylog_sidecar__mirror_url }}/graylog/repo/el/sidecar-stable/{{ repo_graylog_sidecar__version }}/$basearch/ +{% else %} +baseurl=https://packages.graylog2.org/repo/el/sidecar-stable/{{ repo_graylog_sidecar__version }}/$basearch/ +{% endif %} +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-graylog-sidecar +{% if repo_graylog_sidecar__mirror_url is defined and repo_graylog_sidecar__mirror_url | length and repo_graylog_sidecar__basic_auth_login is defined and repo_graylog_sidecar__basic_auth_login | length %} +username={{ repo_graylog_sidecar__basic_auth_login["username"] }} +password={{ repo_graylog_sidecar__basic_auth_login["password"] }} +{% endif %}