From 76b628c04d44dfb64f7b0de3e8a52b4af077ae3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gardais=20J=C3=A9r=C3=A9my?= Date: Tue, 14 Aug 2018 17:25:55 +0200 Subject: [PATCH] Start a xymon script to restart a service in error --- xymon/xymon.procs.alert.sh | 93 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 xymon/xymon.procs.alert.sh diff --git a/xymon/xymon.procs.alert.sh b/xymon/xymon.procs.alert.sh new file mode 100755 index 0000000..1572e56 --- /dev/null +++ b/xymon/xymon.procs.alert.sh @@ -0,0 +1,93 @@ +#!/bin/sh + +# Vars {{{ +DEBUG=0 + +BBCOLORLEVEL="red" +BBHOSTNAME="deploy.ipr.univ-rennes1.fr" +BBSVCNAME="procs" +BBALPHAMSG="deploy.ipr.univ-rennes1.fr:procs red [729848] +red Fri Aug 10 15:35:26 CEST 2018 - Processes NOT ok +&green journald (found 1, req. 1 or more) +&green logind (found 1, req. 1 or more) +&green CRON (found 1, req. 1 or more) +&yellow ATD (found 0, req. 1 or more) +&green MTA-Stretch (found 2, req. 1 or more) +&green SSHD (found 3, req. 1 or more) +&green SSSD (found 1, req. 1 or more) +&red Fail2Ban (found 0, req. 1 or more) +&green tftpd (found 1, req. between 1 and 1) +&green ARPwatch (found 1, req. between 1 and 1) + + PID PPID USER STARTED S PRI %CPU TIME %MEM RSZ VSZ CMD + 1 0 root Aug 02 S 19 0.0 00:00:03 0.5 5492 217412 /sbin/init + 43 1 root Aug 02 S 19 0.0 00:00:06 2.8 30068 124220 /lib/systemd/systemd-journald + 89 1 root Aug 02 S 19 0.0 00:00:00 0.1 1500 22560 /sbin/dhclient -4 -v -pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases -I -df /var/lib/dhcp/dhclient6.eth0.leases eth0 + 121 1 daemon Aug 02 S 19 0.0 00:00:00 0.1 1636 27968 /usr/sbin/atd -f -l 9.6 -b 98 + 125 1 root Aug 02 S 19 0.0 00:00:01 0.1 1724 321836 /usr/sbin/rsyslogd -n + 126 1 root Aug 02 S 19 0.0 00:00:00 0.1 1980 29668 /usr/sbin/cron -f + 137 1 root Aug 02 S 19 0.0 00:00:03 0.3 3328 171944 /usr/sbin/sssd -i -f + 284 137 root Aug 02 S 19 0.0 00:00:07 0.7 7768 212360 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_be --domain ur1 --uid 0 --gid 0 --debug-to-files + 308 137 root Aug 02 S 19 0.0 00:00:09 0.3 4028 168376 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_nss --uid 0 --gid 0 --debug-to-files + 309 137 root Aug 02 S 19 0.0 00:00:03 0.3 3628 147912 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_pam --uid 0 --gid 0 --debug-to-files + 310 137 root Aug 02 S 19 0.0 00:00:03 0.2 2200 141356 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_autofs --uid 0 --gid 0 --debug-to-files + 198 1 root Aug 02 S 19 0.0 00:00:00 0.2 2288 71988 /usr/sbin/sshd -D + 594703 198 root 15:34:47 S 19 0.0 00:00:00 0.6 7280 134176 \\_ sshd: jegardai [priv] + 594918 594703 jegardai 15:34:48 S 19 0.0 00:00:00 0.3 3980 134176 \\_ sshd: jegardai@pts/2 + 594919 594918 jegardai 15:34:48 S 19 0.0 00:00:00 0.6 6316 62044 \\_ zsh + 204 1 root Aug 02 S 19 0.0 00:00:00 0.0 892 17076 /usr/sbin/in.tftpd --listen --user tftp --address [::]:69 --secure --port-range 2070:2076 /srv/tftp + 331 1 root Aug 02 S 19 0.0 00:00:00 0.1 1468 14316 /sbin/agetty -o -p -- \\u --noclear --keep-baud console 115200,38400,9600 linux + 332 1 root Aug 02 S 19 0.0 00:00:00 0.1 1576 14316 /sbin/agetty -o -p -- \\u --noclear --keep-baud tty1 115200,38400,9600 linux + 1322 1 root Aug 02 S 19 0.0 00:00:00 0.1 1780 68928 /lib/systemd/systemd --user + 1324 1322 root Aug 02 S 19 0.0 00:00:00 0.0 96 111572 \\_ (sd-pam) + 1643 1 arpwatch Aug 02 S 19 0.0 00:00:01 0.3 4084 36084 /usr/sbin/arpwatch -u arpwatch -N -p + 2105 1 root Aug 02 S 19 0.0 00:00:03 0.3 3484 83252 /usr/lib/postfix/sbin/master -w + 2107 2105 postfix Aug 02 S 19 0.0 00:00:00 0.5 5704 95656 \\_ qmgr -l -t unix -u + 4516 2105 postfix Aug 02 S 19 0.0 00:00:00 0.6 6396 99844 \\_ tlsmgr -l -t unix -u -c + 592131 2105 postfix 14:14:06 S 19 0.0 00:00:00 0.6 6548 95480 \\_ pickup -l -t unix -u -c + 594683 2105 postfix 15:32:49 S 19 0.0 00:00:00 0.6 6420 95480 \\_ showq -t unix -u -c + 2206 1 message+ Aug 02 S 19 0.0 00:00:00 0.1 1068 67636 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation + 2207 1 root Aug 02 S 19 0.0 00:00:00 0.3 3808 73508 /lib/systemd/systemd-logind + 7896 1 jegardai Aug 02 S 19 0.0 00:00:00 0.1 1820 68932 /lib/systemd/systemd --user + 7898 7896 jegardai Aug 02 S 19 0.0 00:00:00 0.0 256 132260 \\_ (sd-pam) + 163996 1 netdata Aug 03 S 19 0.8 01:30:54 7.2 75652 229332 /usr/sbin/netdata -D + 164014 163996 netdata Aug 03 S 19 0.4 00:46:51 0.7 8136 113012 \\_ /usr/bin/python /usr/lib/x86_64-linux-gnu/netdata/plugins.d/python.d.plugin 1 + 587208 163996 netdata 11:51:34 R 19 0.3 00:00:37 0.2 2632 75012 \\_ /usr/lib/x86_64-linux-gnu/netdata/plugins.d/apps.plugin 1 + 594189 163996 netdata 15:17:36 S 19 0.0 00:00:01 0.2 2688 9696 \\_ bash /usr/lib/x86_64-linux-gnu/netdata/plugins.d/tc-qos-helper.sh 1 + 594627 1 xymon 15:30:50 S 19 0.0 00:00:00 0.0 724 4292 sh -c vmstat 300 2 1>/var/lib/xymon/tmp/xymon_vmstat.deploy.ipr.univ-rennes1.fr.594578 2>&1; mv /var/lib/xymon/tmp/xymon_vmstat.deploy.ipr.univ-rennes1.fr.594578 /var/lib/xymon/tmp/xymon_vmstat.deploy.ipr.univ-rennes1.fr + 594629 594627 xymon 15:30:50 S 19 0.0 00:00:00 0.1 1376 24900 \\_ vmstat 300 2 + 595038 1 xymon 15:35:26 S 19 0.0 00:00:00 0.1 1572 4272 /usr/lib/xymon/client/bin/xymonlaunch --config=/etc/xymon/clientlaunch.cfg --log=/var/log/xymon/clientlaunch.log --pidfile=/var/run/xymon/clientlaunch.pid + 595043 595038 xymon 15:35:26 S 19 0.0 00:00:00 0.1 1432 4292 \\_ /bin/sh /usr/lib/xymon/client/bin/xymonclient.sh + 595065 595043 xymon 15:35:26 S 19 0.0 00:00:00 0.1 1584 4292 | \\_ /bin/sh /usr/lib/xymon/client/bin/xymonclient-linux.sh + 595107 595065 xymon 15:35:26 R 19 0.0 00:00:00 0.2 2784 44404 | \\_ ps -Aww f -o pid,ppid,user,start,state,pri,pcpu,time:12,pmem,rsz:10,vsz:10,cmd + 595044 595038 xymon 15:35:26 S 19 0.0 00:00:00 0.4 5188 18300 \\_ /usr/bin/perl -w /usr/lib/xymon/client/ext/apt + 595081 595044 xymon 15:35:26 R 19 0.0 00:00:00 3.9 40956 70040 \\_ apt-cache policy acl adduser apt apt-listchanges apt-transport-https apt-utils aptitude aptitude-common arpwatch at base-files base-passwd bash bash-completion bind9-host binutils bsd-mailx bsdmainutils bsdutils bzip2 ca-certificates coreutils cpio cpp cpp-6 cracklib-runtime cron curl dash dbus dctrl-tools debconf debconf-i18n debian-archive-keyring debian-faq debian-goodies debianutils debsecan debsums dh-python diffutils dirmngr distro-info-data dmidecode dmsetup doc-debian dpkg e2fslibs:amd64 e2fsprogs ed etckeeper fail2ban file findutils fontconfig-config fonts-dejavu-core fonts-font-awesome fping gcc-6-base:amd64 gettext-base git git-man gnupg gnupg-agent gnutls-bin gpgv grep groff-base gzip hobbit-plugins hostname htop iftop ifupdown init init-system-helpers iproute2 iputils-ping isc-dhcp-client isc-dhcp-common kmod krb5-locales less libacl1:amd64 libapparmor1:amd64 libapt-inst2.0:amd64 libapt-pkg5.0:amd64 libasprintf0v5:amd64 libassuan0:amd64 libattr1:amd64 libaudit-common libaudit1:amd64 libavahi-client3:amd64 libavahi-common-data:amd64 libavahi-common3:amd64 libbasicobjects0:amd64 libbind9-140:amd64 libblkid1:amd64 libboost-filesystem1.62.0:amd64 libboost-iostreams1.62.0:amd64 libboost-system1.62.0:amd64 libbsd0:amd64 libbz2-1.0:amd64 libc-ares2:amd64 libc-bin libc-l10n libc6:amd64 libcap-ng0:amd64 libcap2-bin libcap2:amd64 libclass-isa-perl libcollection4:amd64 libcomerr2:amd64 libcrack2:amd64 libcryptsetup4:amd64 libcups2:amd64 libcurl3-gnutls:amd64 libcurl3:amd64 libcwidget3v5:amd64 libdb5.3:amd64 libdbus-1-3:amd64 libdebconfclient0:amd64 libdevmapper1.02.1:amd64 libdhash1:amd64 libdns-export162 libdns162:amd64 libdpkg-perl libdrm2:amd64 libedit2:amd64 libelf1:amd64 liberror-perl libestr0 libev4 libevent-2.0-5:amd64 libexpat1:amd64 libfas" + +temp_dir=$(mktemp -d -t xymon-alert-XXXXXX.tmp) +service_list="${temp_dir}/services.error.list" +# }}} + +[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${BBHOSTNAME} — ${BBSVCNAME} error" + +# Temp dir for this alert +if [ ! -d "${temp_dir}" ] ; then + mkdir -p -- "${temp_dir}" + [ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${temp_dir} — created." +fi + +# Get the list of processes with an error +echo "${BBALPHAMSG}" | grep -E "&(red|yellow)" | cut -d" " -f2 | tr '[:upper:]' '[:lower:]' > "${service_list}" + +# If any error on a process +if [ -s "${service_list}" ]; then + [ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : process list — Some processes seems to be in error." + while IFS= read -r process; do + [ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : while process loop — ${process} service need to be restarted." + done < "${service_list}" +else + [ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : process list — No error on any process." +fi + +rm -rf -- "${temp_dir}" + +exit 0