scripts/xymon/xymon.libs.alert.sh

136 lines
5.2 KiB
Bash
Executable File
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/sh
# Purpose {{{
## If Xymon server says that a service is in error on a remote host, try to restart this service.
## 1. Create a ssh keyring for xymon user {{{
# sudo mkdir -p -- /var/lib/xymon/.ssh/
# sudo ssh-keygen -f /var/lib/xymon/.ssh/id_rsa -N '' -q
# sudo chown -R xymon:xymon /var/lib/xymon/.ssh/
## }}}
## 2. Remote user {{{
# Ensure to have the ${REMOTE_SSH_USER} available on remote hosts and allowed to connect with SSH.
# Restrict the SSH access to a single SSH key from the Xymon server IP (~${REMOTE_SSH_USER}/.ssh/authorized_keys):
## from="IP.SRV.XYM.ON" ssh-rsa AAAAA…
# Allow sudo commands to restart services (/etc/sudoers.d/xymon-ssh):
## xymon-ssh ALL=(root:root) NOPASSWD: /bin/systemctl restart *
## }}}
# }}}
# Vars {{{
DEBUG=1
LOCAL_SSH_USER="xymon"
REMOTE_SSH_USER="xymon-ssh"
temp_dir=$(mktemp -d -t xymon-libs-alert-XXXXXX.tmp)
debug_stdout="${temp_dir}/debug.stdout"
debug_stderr="${temp_dir}/debug.stderr"
service_list="${temp_dir}/services.error.list"
# }}}
# Create log files
touch "${debug_stdout}" "${debug_stderr}"
# Manage only libs probe {{{
if [ "${BBSVCNAME}" = "libs" ]; then
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: ${BBHOSTNAME}${BBSVCNAME} error" >> "${debug_stdout}"
else
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: ${BBHOSTNAME}${BBSVCNAME} probe is not managed." >> "${debug_stderr}"
[ "${DEBUG}" -eq "0" ] || rm -rf -- "${temp_dir}"
exit 0
fi
# }}}
# Check if host need to reboot {{{
if echo "${BBALPHAMSG}" | grep -qE "\\&yellow Machine should be reboot.*" ; then
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: Test kernel — The host need to be rebooted." >> "${debug_stdout}"
fi
# }}}
# Check if a service need to restart
if echo "${BBALPHAMSG}" | grep -qE "\\&yellow The following processes.*" ; then
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: Test service — Some services need to be restarted." >> "${debug_stdout}"
# Get the list of binaries path
echo "${BBALPHAMSG}" | sed -n 's/^ \(\/.*\) (.*)/\1/p' > "${service_list}"
while IFS= read -r bin_path; do
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: while bin_path loop — ${bin_path} use old libs." >> "${debug_stdout}"
# Match binaries path and services name {{{
case "${bin_path}" in
# Apache2
'/usr/sbin/apache2' )
service_name="apache2"
;;
# Arpwatch
'/usr/sbin/arpwatch' )
service_name="arpwatch"
;;
# Dbus
'/usr/bin/dbus-daemon' )
service_name="dbus"
;;
# Postfix
'/usr/lib/postfix/sbin/pickup' | '/usr/lib/postfix/qmgr' | '/usr/lib/postfix/sbin/tlsmgr' | '/usr/lib/postfix/sbin/qmgr' )
service_name="postfix"
;;
# Nginx
'/usr/sbin/nginx' )
service_name="nginx"
;;
# Netdata
'/usr/sbin/netdata' )
service_name="netdata"
;;
# Ntp
'/usr/sbin/ntpd' )
service_name="ntp"
;;
# PHP-FPM
'/usr/sbin/php-fpm7.0' )
service_name="php7.0-fpm"
;;
# Systemd-journald
'/lib/systemd/systemd-journald' )
service_name="systemd-journald"
;;
# Systemd-logind
'/lib/systemd/systemd-logind' )
service_name="systemd-logind"
;;
# default
* )
service_name="NOT.MANAGED"
;;
esac
# }}}
# Restart service if needed {{{
if [ "${service_name}" != "NOT.MANAGED" ] && [ "${service_name}" != "${previous_service_name}" ]; then
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: while bin_path loop — ${service_name} need to be restarted." >> "${debug_stdout}"
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: while bin_path loop — ssh -n -o StrictHostKeyChecking=no ${REMOTE_SSH_USER}@${BBHOSTNAME} sudo systemctl restart ${service_name}.service" >> "${debug_stdout}"
ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${REMOTE_SSH_USER}"@"${BBHOSTNAME}" "sudo systemctl restart ${service_name}.service" >> "${debug_stdout}" 2>> "${debug_stderr}"
previous_service_name="${service_name}"
else
if [ "${service_name}" = "NOT.MANAGED" ]; then
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: while bin_path loop — service for ${bin_path} is not managed." >> "${debug_stdout}"
else
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: while bin_path loop — ${service_name} was already restarted." >> "${debug_stdout}"
fi
fi
# }}}
done < "${service_list}"
# Also restart xymon-client service {{{
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: Test service — xymon-client also need to be restarted." >> "${debug_stdout}"
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG: Test service — ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${REMOTE_SSH_USER}@${BBHOSTNAME} sudo systemctl restart xymon-client.service" >> "${debug_stdout}"
ssh -n -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${REMOTE_SSH_USER}"@"${BBHOSTNAME}" "sudo systemctl restart xymon-client.service" >> "${debug_stdout}" 2>> "${debug_stderr}"
# }}}
fi
# Remove empty error file
[ -s "${debug_stderr}" ] || rm -f "${debug_stderr}"
# Remove temp_dir if DEBUG is disable
[ "${DEBUG}" -eq "0" ] || rm -rf -- "${temp_dir}"
exit 0