diff --git a/rock.json b/rock.json index 42de52d..8f78686 100644 --- a/rock.json +++ b/rock.json @@ -9,7 +9,7 @@ { "type": "amazon-ebs", "region": "eu-west-2", - "source_ami": "ami-210be846", + "source_ami": "ami-f57a9292", "instance_type": "t2.small", "ssh_username": "root", "ami_name": "juxt-rock-{{user `commit_ref`}}-{{timestamp}}" diff --git a/scripts/install-cfn-bootstrap.sh b/scripts/install-cfn-bootstrap.sh new file mode 100755 index 0000000..f667f84 --- /dev/null +++ b/scripts/install-cfn-bootstrap.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -eux + +pacman --noconfirm -Syu "python2" + +working_dir="/tmp/aws-cfn-bootstrap" + +mkdir -p "$working_dir" + +cd "$working_dir" +wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-1.4-30.tar.gz + +tar xzf aws-cfn-bootstrap*.tar.gz +cd aws-cfn-bootstrap-1*/ + +sed -i "s/env python'/env python2'/" setup.py +python2 setup.py install + +rm -rf "$working_dir" diff --git a/scripts/install-custom.sh b/scripts/install-custom.sh index 9900d74..5d9b672 100755 --- a/scripts/install-custom.sh +++ b/scripts/install-custom.sh @@ -64,6 +64,11 @@ chmod -R 774 "$PKG_DIR" # containing PKGBUILD files. cd "$PKG_DIR" +# Dependencies from AUR of our packages need fetching in order to be available +# for builds. Currently there doesn't appear to be a way to automatically get +# list. +echo 'python2-pystache' | su --preserve-environment -c 'aurfetch' rock + # Packages in AUR etc. have both a `PKGBUILD` file and a `.SRCINFO`. As we don't # want to have to remember to create this src info file everytime we edit a # package definition we create the files only when we need them. @@ -80,7 +85,8 @@ build_pkgs install_pkgs \ codedeploy-agent \ journald-cloud-watch-script \ - systemd-cloud-watch + systemd-cloud-watch \ + aws-cfn-bootstrap # To install any packages from the AUR can now be done like so: # aursync ... diff --git a/share/aws-cfn-bootstrap/PKGBUILD b/share/aws-cfn-bootstrap/PKGBUILD new file mode 100644 index 0000000..910025b --- /dev/null +++ b/share/aws-cfn-bootstrap/PKGBUILD @@ -0,0 +1,25 @@ +pkgname=aws-cfn-bootstrap +pkgver=1.4_30 +pkgrel=1 +pkgdesc="" +arch=('i686' 'x86_64') +url='https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-helper-scripts-reference.html' +license=('Apache-2.0') +# TODO: python2-pystache is only in the AUR, how do we handle that? +depends=('python2-pystache') +# >=1.5.2,<2.0 +optdepends=('python2-daemon: Support for running cfn-hup as a daemon') +makedepends=('python2-setuptools') +source=(https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-${pkgver//_/-}.tar.gz) +md5sums=('5715176852b84761c63097f2778c99e4') + +build() { + cd $pkgname-${pkgver/_*/} + python2 setup.py build +} + +package() { + cd $pkgname-${pkgver/_*/} + python2 setup.py install --root="$pkgdir" --optimize=1 + sed -i "s|#!/usr/bin/env python$|#!/usr/bin/env python2|" ${pkgdir}/usr/bin/* +} diff --git a/share/journald-cloud-watch-script/PKGBUILD b/share/journald-cloud-watch-script/PKGBUILD index 4df1c04..efcbd85 100644 --- a/share/journald-cloud-watch-script/PKGBUILD +++ b/share/journald-cloud-watch-script/PKGBUILD @@ -9,9 +9,12 @@ license=('GPL') groups=() depends=('aws-cli' 'bash' 'jq') source=(tail-journald - journald-cloud-watch-script.service) -md5sums=('8907d8a5eadddd7c36730b4e7eff0f11' - '957ed4f7b25b8246aa518aee1474a20e') + journald-cloud-watch-script.service + journald-cloud-watch-script.install) +md5sums=('b561e5e0aab3eb3c44f6919e2acec2b3' + '553401317d43abbd4b0147b7bec5820d' + '7a25597c93a176efede06b2c551ab534') +install=journald-cloud-watch-script.install package() { cd "$srcdir" diff --git a/share/journald-cloud-watch-script/journald-cloud-watch-script.install b/share/journald-cloud-watch-script/journald-cloud-watch-script.install new file mode 100644 index 0000000..cdb9a07 --- /dev/null +++ b/share/journald-cloud-watch-script/journald-cloud-watch-script.install @@ -0,0 +1,13 @@ +post_install() { + if ! getent group cw-journald >/dev/null; then + groupadd cw-journald + fi + if ! getent passwd cw-journald >/dev/null; then + useradd -c 'CloudWatch Journald Script User' -g cw-journald -G systemd-journal -s /bin/nologin cw-journald + passwd -l cw-journald >/dev/null + fi + if [ ! -d '/var/lib/journald-cloud-watch-script' ]; then + mkdir -p '/var/lib/journald-cloud-watch-script' + chown cw-journald:cw-journald '/var/lib/journald-cloud-watch-script' + fi +} diff --git a/share/journald-cloud-watch-script/journald-cloud-watch-script.service b/share/journald-cloud-watch-script/journald-cloud-watch-script.service index a5c7433..06e015d 100644 --- a/share/journald-cloud-watch-script/journald-cloud-watch-script.service +++ b/share/journald-cloud-watch-script/journald-cloud-watch-script.service @@ -4,8 +4,8 @@ Wants=basic.target After=basic.target network.target [Service] -User=nobody -Group=nobody +User=cw-journald +Group=cw-journald ExecStart=/usr/bin/tail-journald KillMode=process Restart=on-failure diff --git a/share/journald-cloud-watch-script/tail-journald b/share/journald-cloud-watch-script/tail-journald index 855cc1b..3d805a9 100755 --- a/share/journald-cloud-watch-script/tail-journald +++ b/share/journald-cloud-watch-script/tail-journald @@ -30,7 +30,8 @@ create_or_get_current_cursor () { aws logs create-log-stream \ --region=$AWS_REGION \ --log-group-name=$LOG_GROUP_NAME \ - --log-stream-name=$LOG_STREAM_NAME + --log-stream-name=$LOG_STREAM_NAME \ + || return 0 fi } @@ -49,7 +50,17 @@ journal_cursor_rows () { } format_row () { - jq "{timestamp: $(date +%s%3N), message: @json}" + jq ' + # A fork of jq 1.6 walk which also walks over the keys + def walk(f): + . as $in + | if type == "object" then + reduce keys[] as $key + ( {}; . + { ($key | walk(f)): ($in[$key] | walk(f)) } ) | f + elif type == "array" then map( walk(f) ) | f + else f + end; + {timestamp: '"$(date +%s%3N)"', message: @json "\(walk(if type == "string" then .[:16384] else . end))"}' } write_batch () { @@ -59,20 +70,18 @@ write_batch () { return 0 fi - BATCH=$(echo "$rows" |\ - format_row |\ - jq -s .) if [[ $(cat $AWS_CURSOR_FILE) ]]; then sequence_token="--sequence-token=$(cat $AWS_CURSOR_FILE)" else sequence_token="" fi + aws logs put-log-events \ --region="$AWS_REGION" \ --log-group-name="$LOG_GROUP_NAME" \ --log-stream-name="$LOG_STREAM_NAME" \ "$sequence_token" \ - --log-events="$BATCH" |\ + --log-events 'file://'<(echo "$rows" | format_row | jq -s .)|\ jq -r ".nextSequenceToken" > "$AWS_CURSOR_FILE" } diff --git a/terraform/rock.sh b/terraform/rock.sh index ea0de38..2a03337 100755 --- a/terraform/rock.sh +++ b/terraform/rock.sh @@ -16,3 +16,6 @@ EOF # # Fix is in this PR: https://github.com/advantageous/systemd-cloud-watch/pull/16 # systemctl start systemd-cloud-watch + +# Fortunately there's an alternative bash script +systemctl start journald-cloud-watch-script