Manage Xymon's APT alert (only repos outdated)

xymon/sample.messages/apt.alert

@@ -0,0 +1,28 @@
+BBCOLORLEVEL="red"
+BBALPHAMSG="HOST.DOMAIN.ORG:apt red [168321]
+red Wed Aug 22 11:26:34 2018 - apt NOT ok
+Debian GNU/Linux 9.5 (stretch)
+
+&red Security updates (4): apt-get install openssh-client openssh-server openssh-sftp-server ssh
+   openssh-client (1:7.4p1-10+deb9u3 1:7.4p1-10+deb9u4)
+   openssh-server (1:7.4p1-10+deb9u3 1:7.4p1-10+deb9u4)
+   openssh-sftp-server (1:7.4p1-10+deb9u3 1:7.4p1-10+deb9u4)
+   ssh (1:7.4p1-10+deb9u3 1:7.4p1-10+deb9u4)
+
+&red Last apt update: 3.0 day(s) ago
+
+
+See http://localhost/xymon-cgi/svcstatus.sh?HOST=HOST.DOMAIN.ORG&SERVICE=apt
+"
+ACKCODE="168321"
+RCPT="1234567890"
+BBHOSTNAME="HOST.DOMAIN.ORG"
+MACHIP="111222333444"
+BBSVCNAME="apt"
+BBSVCNUM="0"
+BBHOSTSVC="HOST.DOMAIN.ORG.apt"
+BBHOSTSVCCOMMAS="HOST,DOMAIN,ORG.apt"
+BBNUMERIC="000111222333444168321"
+RECOVERED="0"
+DOWNSECS="36"
+DOWNSECSMSG=""

xymon/xymon.apt.alert.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+# Purpose {{{
+## If Xymon server says that the last apt update is too old, try to run a new one.
+## 1. Create a ssh keyring for xymon user {{{
+# sudo mkdir -p -- /var/lib/xymon/.ssh/
+# sudo ssh-keygen -f /var/lib/xymon/.ssh/id_rsa -N '' -q
+# sudo chown -R xymon:xymon /var/lib/xymon/.ssh/
+## }}}
+## 2. Remote user {{{
+# Ensure to have the ${REMOTE_SSH_USER} available on remote hosts and allowed to connect with SSH.
+# Restrict the SSH access to a single SSH key from the Xymon server IP (~${REMOTE_SSH_USER}/.ssh/authorized_keys) :
+## from="IP.SRV.XYM.ON" ssh-rsa AAAAA…
+# Allow sudo commands to restart services and run apt update (/etc/sudoers.d/xymon-ssh) :
+## xymon-ssh ALL=(root:root) NOPASSWD: /usr/bin/apt update
+## xymon-ssh ALL=(root:root) NOPASSWD: /bin/systemctl restart *
+## }}}
+
+# }}}
+# Vars {{{
+DEBUG=1
+
+LOCAL_SSH_USER="xymon"
+REMOTE_SSH_USER="xymon-ssh"
+
+temp_dir=$(mktemp -d -t xymon-apt-alert-XXXXXX.tmp)
+debug_stdout="${temp_dir}/debug.stdout"
+debug_stderr="${temp_dir}/debug.stderr"
+# }}}
+
+# Create log files
+touch "${debug_stdout}" "${debug_stderr}"
+
+# Manage only apt probe {{{
+if [ "${BBSVCNAME}" = "apt" ]; then
+	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${BBHOSTNAME} — ${BBSVCNAME} error" >> "${debug_stdout}"
+else
+	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${BBHOSTNAME} — ${BBSVCNAME} probe is not managed." >> "${debug_stderr}"
+	[ "${DEBUG}" -eq "0" ] || rm -rf -- "${temp_dir}"
+	exit 0
+fi
+# }}}
+
+# Check if repos need to be updated {{{
+if echo "${BBALPHAMSG}" | grep -qE "\\&(red|yellow) Last apt update.*ago$" ; then
+	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test APT repos — APT repos need to be updated." >> "${debug_stdout}"
+	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test APT repos — ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${REMOTE_SSH_USER}@${BBHOSTNAME} sudo apt update" >> "${debug_stdout}"
+	ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${REMOTE_SSH_USER}"@"${BBHOSTNAME}" "sudo apt update" >> "${debug_stdout}" 2>> "${debug_stderr}"
+	# Also restart xymon-client service {{{
+	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test APT repos — xymon-client also need to be restarted." >> "${debug_stdout}"
+	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : Test APT repos — ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${REMOTE_SSH_USER}@${BBHOSTNAME} sudo systemctl restart xymon-client.service" >> "${debug_stdout}"
+	ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${REMOTE_SSH_USER}"@"${BBHOSTNAME}" "sudo systemctl restart xymon-client.service" >> "${debug_stdout}" 2>> "${debug_stderr}"
+	# }}}
+fi
+# }}}
+
+# Remove empty error file
+[ -s "${debug_stderr}" ] || rm -f "${debug_stderr}"
+
+# Remove temp_dir if DEBUG is disable
+[ "${DEBUG}" -eq "0" ] || rm -rf -- "${temp_dir}"
+
+exit 0