Start a xymon script to restart a service in error
This commit is contained in:
parent
6da1d04fb1
commit
76b628c04d
|
|
@ -0,0 +1,93 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# Vars {{{
|
||||||
|
DEBUG=0
|
||||||
|
|
||||||
|
BBCOLORLEVEL="red"
|
||||||
|
BBHOSTNAME="deploy.ipr.univ-rennes1.fr"
|
||||||
|
BBSVCNAME="procs"
|
||||||
|
BBALPHAMSG="deploy.ipr.univ-rennes1.fr:procs red [729848]
|
||||||
|
red Fri Aug 10 15:35:26 CEST 2018 - Processes NOT ok
|
||||||
|
&green journald (found 1, req. 1 or more)
|
||||||
|
&green logind (found 1, req. 1 or more)
|
||||||
|
&green CRON (found 1, req. 1 or more)
|
||||||
|
&yellow ATD (found 0, req. 1 or more)
|
||||||
|
&green MTA-Stretch (found 2, req. 1 or more)
|
||||||
|
&green SSHD (found 3, req. 1 or more)
|
||||||
|
&green SSSD (found 1, req. 1 or more)
|
||||||
|
&red Fail2Ban (found 0, req. 1 or more)
|
||||||
|
&green tftpd (found 1, req. between 1 and 1)
|
||||||
|
&green ARPwatch (found 1, req. between 1 and 1)
|
||||||
|
|
||||||
|
PID PPID USER STARTED S PRI %CPU TIME %MEM RSZ VSZ CMD
|
||||||
|
1 0 root Aug 02 S 19 0.0 00:00:03 0.5 5492 217412 /sbin/init
|
||||||
|
43 1 root Aug 02 S 19 0.0 00:00:06 2.8 30068 124220 /lib/systemd/systemd-journald
|
||||||
|
89 1 root Aug 02 S 19 0.0 00:00:00 0.1 1500 22560 /sbin/dhclient -4 -v -pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases -I -df /var/lib/dhcp/dhclient6.eth0.leases eth0
|
||||||
|
121 1 daemon Aug 02 S 19 0.0 00:00:00 0.1 1636 27968 /usr/sbin/atd -f -l 9.6 -b 98
|
||||||
|
125 1 root Aug 02 S 19 0.0 00:00:01 0.1 1724 321836 /usr/sbin/rsyslogd -n
|
||||||
|
126 1 root Aug 02 S 19 0.0 00:00:00 0.1 1980 29668 /usr/sbin/cron -f
|
||||||
|
137 1 root Aug 02 S 19 0.0 00:00:03 0.3 3328 171944 /usr/sbin/sssd -i -f
|
||||||
|
284 137 root Aug 02 S 19 0.0 00:00:07 0.7 7768 212360 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_be --domain ur1 --uid 0 --gid 0 --debug-to-files
|
||||||
|
308 137 root Aug 02 S 19 0.0 00:00:09 0.3 4028 168376 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_nss --uid 0 --gid 0 --debug-to-files
|
||||||
|
309 137 root Aug 02 S 19 0.0 00:00:03 0.3 3628 147912 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_pam --uid 0 --gid 0 --debug-to-files
|
||||||
|
310 137 root Aug 02 S 19 0.0 00:00:03 0.2 2200 141356 \\_ /usr/lib/x86_64-linux-gnu/sssd/sssd_autofs --uid 0 --gid 0 --debug-to-files
|
||||||
|
198 1 root Aug 02 S 19 0.0 00:00:00 0.2 2288 71988 /usr/sbin/sshd -D
|
||||||
|
594703 198 root 15:34:47 S 19 0.0 00:00:00 0.6 7280 134176 \\_ sshd: jegardai [priv]
|
||||||
|
594918 594703 jegardai 15:34:48 S 19 0.0 00:00:00 0.3 3980 134176 \\_ sshd: jegardai@pts/2
|
||||||
|
594919 594918 jegardai 15:34:48 S 19 0.0 00:00:00 0.6 6316 62044 \\_ zsh
|
||||||
|
204 1 root Aug 02 S 19 0.0 00:00:00 0.0 892 17076 /usr/sbin/in.tftpd --listen --user tftp --address [::]:69 --secure --port-range 2070:2076 /srv/tftp
|
||||||
|
331 1 root Aug 02 S 19 0.0 00:00:00 0.1 1468 14316 /sbin/agetty -o -p -- \\u --noclear --keep-baud console 115200,38400,9600 linux
|
||||||
|
332 1 root Aug 02 S 19 0.0 00:00:00 0.1 1576 14316 /sbin/agetty -o -p -- \\u --noclear --keep-baud tty1 115200,38400,9600 linux
|
||||||
|
1322 1 root Aug 02 S 19 0.0 00:00:00 0.1 1780 68928 /lib/systemd/systemd --user
|
||||||
|
1324 1322 root Aug 02 S 19 0.0 00:00:00 0.0 96 111572 \\_ (sd-pam)
|
||||||
|
1643 1 arpwatch Aug 02 S 19 0.0 00:00:01 0.3 4084 36084 /usr/sbin/arpwatch -u arpwatch -N -p
|
||||||
|
2105 1 root Aug 02 S 19 0.0 00:00:03 0.3 3484 83252 /usr/lib/postfix/sbin/master -w
|
||||||
|
2107 2105 postfix Aug 02 S 19 0.0 00:00:00 0.5 5704 95656 \\_ qmgr -l -t unix -u
|
||||||
|
4516 2105 postfix Aug 02 S 19 0.0 00:00:00 0.6 6396 99844 \\_ tlsmgr -l -t unix -u -c
|
||||||
|
592131 2105 postfix 14:14:06 S 19 0.0 00:00:00 0.6 6548 95480 \\_ pickup -l -t unix -u -c
|
||||||
|
594683 2105 postfix 15:32:49 S 19 0.0 00:00:00 0.6 6420 95480 \\_ showq -t unix -u -c
|
||||||
|
2206 1 message+ Aug 02 S 19 0.0 00:00:00 0.1 1068 67636 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
|
||||||
|
2207 1 root Aug 02 S 19 0.0 00:00:00 0.3 3808 73508 /lib/systemd/systemd-logind
|
||||||
|
7896 1 jegardai Aug 02 S 19 0.0 00:00:00 0.1 1820 68932 /lib/systemd/systemd --user
|
||||||
|
7898 7896 jegardai Aug 02 S 19 0.0 00:00:00 0.0 256 132260 \\_ (sd-pam)
|
||||||
|
163996 1 netdata Aug 03 S 19 0.8 01:30:54 7.2 75652 229332 /usr/sbin/netdata -D
|
||||||
|
164014 163996 netdata Aug 03 S 19 0.4 00:46:51 0.7 8136 113012 \\_ /usr/bin/python /usr/lib/x86_64-linux-gnu/netdata/plugins.d/python.d.plugin 1
|
||||||
|
587208 163996 netdata 11:51:34 R 19 0.3 00:00:37 0.2 2632 75012 \\_ /usr/lib/x86_64-linux-gnu/netdata/plugins.d/apps.plugin 1
|
||||||
|
594189 163996 netdata 15:17:36 S 19 0.0 00:00:01 0.2 2688 9696 \\_ bash /usr/lib/x86_64-linux-gnu/netdata/plugins.d/tc-qos-helper.sh 1
|
||||||
|
594627 1 xymon 15:30:50 S 19 0.0 00:00:00 0.0 724 4292 sh -c vmstat 300 2 1>/var/lib/xymon/tmp/xymon_vmstat.deploy.ipr.univ-rennes1.fr.594578 2>&1; mv /var/lib/xymon/tmp/xymon_vmstat.deploy.ipr.univ-rennes1.fr.594578 /var/lib/xymon/tmp/xymon_vmstat.deploy.ipr.univ-rennes1.fr
|
||||||
|
594629 594627 xymon 15:30:50 S 19 0.0 00:00:00 0.1 1376 24900 \\_ vmstat 300 2
|
||||||
|
595038 1 xymon 15:35:26 S 19 0.0 00:00:00 0.1 1572 4272 /usr/lib/xymon/client/bin/xymonlaunch --config=/etc/xymon/clientlaunch.cfg --log=/var/log/xymon/clientlaunch.log --pidfile=/var/run/xymon/clientlaunch.pid
|
||||||
|
595043 595038 xymon 15:35:26 S 19 0.0 00:00:00 0.1 1432 4292 \\_ /bin/sh /usr/lib/xymon/client/bin/xymonclient.sh
|
||||||
|
595065 595043 xymon 15:35:26 S 19 0.0 00:00:00 0.1 1584 4292 | \\_ /bin/sh /usr/lib/xymon/client/bin/xymonclient-linux.sh
|
||||||
|
595107 595065 xymon 15:35:26 R 19 0.0 00:00:00 0.2 2784 44404 | \\_ ps -Aww f -o pid,ppid,user,start,state,pri,pcpu,time:12,pmem,rsz:10,vsz:10,cmd
|
||||||
|
595044 595038 xymon 15:35:26 S 19 0.0 00:00:00 0.4 5188 18300 \\_ /usr/bin/perl -w /usr/lib/xymon/client/ext/apt
|
||||||
|
595081 595044 xymon 15:35:26 R 19 0.0 00:00:00 3.9 40956 70040 \\_ apt-cache policy acl adduser apt apt-listchanges apt-transport-https apt-utils aptitude aptitude-common arpwatch at base-files base-passwd bash bash-completion bind9-host binutils bsd-mailx bsdmainutils bsdutils bzip2 ca-certificates coreutils cpio cpp cpp-6 cracklib-runtime cron curl dash dbus dctrl-tools debconf debconf-i18n debian-archive-keyring debian-faq debian-goodies debianutils debsecan debsums dh-python diffutils dirmngr distro-info-data dmidecode dmsetup doc-debian dpkg e2fslibs:amd64 e2fsprogs ed etckeeper fail2ban file findutils fontconfig-config fonts-dejavu-core fonts-font-awesome fping gcc-6-base:amd64 gettext-base git git-man gnupg gnupg-agent gnutls-bin gpgv grep groff-base gzip hobbit-plugins hostname htop iftop ifupdown init init-system-helpers iproute2 iputils-ping isc-dhcp-client isc-dhcp-common kmod krb5-locales less libacl1:amd64 libapparmor1:amd64 libapt-inst2.0:amd64 libapt-pkg5.0:amd64 libasprintf0v5:amd64 libassuan0:amd64 libattr1:amd64 libaudit-common libaudit1:amd64 libavahi-client3:amd64 libavahi-common-data:amd64 libavahi-common3:amd64 libbasicobjects0:amd64 libbind9-140:amd64 libblkid1:amd64 libboost-filesystem1.62.0:amd64 libboost-iostreams1.62.0:amd64 libboost-system1.62.0:amd64 libbsd0:amd64 libbz2-1.0:amd64 libc-ares2:amd64 libc-bin libc-l10n libc6:amd64 libcap-ng0:amd64 libcap2-bin libcap2:amd64 libclass-isa-perl libcollection4:amd64 libcomerr2:amd64 libcrack2:amd64 libcryptsetup4:amd64 libcups2:amd64 libcurl3-gnutls:amd64 libcurl3:amd64 libcwidget3v5:amd64 libdb5.3:amd64 libdbus-1-3:amd64 libdebconfclient0:amd64 libdevmapper1.02.1:amd64 libdhash1:amd64 libdns-export162 libdns162:amd64 libdpkg-perl libdrm2:amd64 libedit2:amd64 libelf1:amd64 liberror-perl libestr0 libev4 libevent-2.0-5:amd64 libexpat1:amd64 libfas"
|
||||||
|
|
||||||
|
temp_dir=$(mktemp -d -t xymon-alert-XXXXXX.tmp)
|
||||||
|
service_list="${temp_dir}/services.error.list"
|
||||||
|
# }}}
|
||||||
|
|
||||||
|
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${BBHOSTNAME} — ${BBSVCNAME} error"
|
||||||
|
|
||||||
|
# Temp dir for this alert
|
||||||
|
if [ ! -d "${temp_dir}" ] ; then
|
||||||
|
mkdir -p -- "${temp_dir}"
|
||||||
|
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : ${temp_dir} — created."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get the list of processes with an error
|
||||||
|
echo "${BBALPHAMSG}" | grep -E "&(red|yellow)" | cut -d" " -f2 | tr '[:upper:]' '[:lower:]' > "${service_list}"
|
||||||
|
|
||||||
|
# If any error on a process
|
||||||
|
if [ -s "${service_list}" ]; then
|
||||||
|
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : process list — Some processes seems to be in error."
|
||||||
|
while IFS= read -r process; do
|
||||||
|
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : while process loop — ${process} service need to be restarted."
|
||||||
|
done < "${service_list}"
|
||||||
|
else
|
||||||
|
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6s\e[m\n' "DEBUG : process list — No error on any process."
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -rf -- "${temp_dir}"
|
||||||
|
|
||||||
|
exit 0
|
||||||
Loading…
Reference in New Issue