Monitor your GNU/Linux system

Configuration

• sysstat : 12.0.3
• iperf : 3.6
• ocfs2-tools : 1.8.5

Install tools

root@host:~# apt-get install iperf3 sysstat

• Good to know, the Linux Performance Observability Tools schema, thanks to Brendan D. Gregg

Code

Bandwidth

#! /bin/bash
####TEST BW I
echo "Download Windows XP SP3 to test the bandwidth >> /var/log/monitor.log
wget -O /dev/null 'http://www.download.windowsupdate.com/msdownload/update/software/dflt/2008/04/windowsxp-kb936929-sp3-x86-enu_c81472f7eeea2eca421e116cd4c03e2300ebfde4.exe' 2>&1 | grep -E "saved" >> /var/log/monitor.log

####TEST BW II
HOSTS="bouygues.testdebit.info ping.online.net speedtest.serverius.net"

FunctionRandomHost() #Function to randomly change iperf host
{
        HOST=$(for i in $HOSTS; do echo $i; done | sort -R | head -n 1)
        echo "$HOST"
}

echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : BANDWIDTH TEST II" >> /var/log/monitor.log
HOST=$(FunctionRandomHost)
echo "$HOST"
echo "IPERF from "$HOST"" >> /var/log/monitor.log
echo "iperf3 -c "$HOST" -w 200K -i 3 -f M"
if [ "$HOST" == "speedtest.serverius.net" ]; then
        iperf3 -c "$HOST" -p 5002 -w 200K -i 3 -f M | grep -E "sender" | awk '{system("echo -n $(date \"+%Y-%m-%d %H:%M:%S\")"); print " upload : " $7" Mo/s"}' >> /var/log/monitor.log
        iperf3 -c "$HOST" -p 5002 -w 200K -i 3 -f M -R | grep -E "receiver" | awk '{system("echo -n $(date \"+%Y-%m-%d %H:%M:%S\")"); print " download : " $7" Mo/s"}' >> /var/log/monitor.log
else
        iperf3 -c "$HOST" -w 200K -i 3 -f M | grep -E "sender" | awk '{system("echo -n $(date \"+%Y-%m-%d %H:%M:%S\")"); print " upload : " $7" Mo/s"}' >> /var/log/monitor.log
        iperf3 -c "$HOST" -w 200K -i 3 -f M -R | grep -E "receiver" | awk '{system("echo -n $(date \"+%Y-%m-%d %H:%M:%S\")"); print " download : " $7" Mo/s"}' >> /var/log/monitor.log
fi

Network

#! /bin/bash
####NETWORK TEST
echo "$(date "+%Y-%m-%d %H:%M:%S") : NETWORK INTERFACES TEST" >> /var/log/monitor.log
sar -n DEV 1 8 >> /var/log/monitor.log

Hard Disk

#! /bin/bash
####DISK TEST
echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : DISK TEST - Write speed to /data/" >> /var/log/monitor.log
dd if=/dev/zero of=/data/test1.img bs=1G count=1 oflag=dsync 2>> /var/log/monitor.log
rm /data/test1.img

echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : DISK TEST - Latency of /data/" >> /var/log/monitor.log
dd if=/dev/zero of=/data/test1.img bs=512 count=1000 oflag=dsync 2>> /var/log/monitor.log
rm /data/test1.img

####DISK TEST II
echo -e "\n-------------IOSTAT1-------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : TEST DISK II" >> /var/log/monitor.log
iostat -x 90 1 >> /var/log/monitor.log

CPU Load

#! /bin/bash
####CPU TEST
echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : CPU TEST" >> /var/log/monitor.log
sar 2 5 >> /var/log/monitor.log

####CPU TEST II
echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : CPU TEST II" >> /var/log/monitor.log
top -n 2 -d 2 -b >> /var/log/monitor.log

Memory

#! /bin/bash
####MEMORY TEST
echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : TEST MEMOIRE" >> /var/log/monitor.log
sar -r 1 5 >> /var/log/monitor.log

Web server

#! /bin/bash

####WEB TEST
echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : WEB TEST INTERNE" >> /var/log/monitor.log

for i in www.google.fr orange.fr sfr.fr free.fr bouygues.fr
do
        echo -e "$(date "+%Y-%m-%d %H:%M:%S") : test de "$i" : $(curl -so /dev/null -w '%{time_total}\n' "$i")\n--" >> /var/log/monitor.log &
done

wait

####WEB TEST INTERNE II
echo -e "\n---------------------------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : WEB TEST II" >> /var/log/monitor.log
ab -k -c 200 -n 2000 http://shebangthedolphins.net/checkiffileexists.php >> /var/log/monitor.log

Latency

#! /bin/bash
####PING TEST
echo -e "\n-------------PING-------------" >> /var/log/monitor.log
echo "$(date "+%Y-%m-%d %H:%M:%S") : TEST PING" >> /var/log/monitor.log
ping -q -c 10 192.168.0.1 | grep -v PING | sed '/^$/d' >> /var/log/monitor.log &
ping -q -c 10 google.fr | grep -v PING | sed '/^$/d' >> /var/log/monitor.log &
ping -q -c 10 192.168.0.210 | grep -v PING | sed '/^$/d' >> /var/log/monitor.log &
ping -q -c 10 192.168.1.20 | grep -v PING | sed '/^$/d' >> /var/log/monitor.log &
wait

Log Processing

Works for ext4 file system only

#! /bin/bash
SDE=$(/bin/date --date='2 days ago' +%s) #two days epoch
INO=$(stat -c %i /var/log/monitor.log) #get inode number of /var/log/monitor.log file
DEV=/dev/sda1 #device where /var has been mounted
CRE=$(/bin/date --date="$(/sbin/debugfs -R 'stat <"'"$INO"'">' $DEV 2>/dev/null | grep 'crtime:' | sed 's/.*-- //')" +%s) #epoch time of last /var/log/monitor.log modification

A=6
B=7

if [ "$SDE" -gt "$CRE" ]; then
        while [ "$A" -ge 1 ]; do
                mv /var/log/monitor."$A".gz /var/log/monitor."$B".gz
                ((A-=1)) #ou A=$((A-1))
                ((B-=1))
        done

        gzip -c /var/log/monitor.log > /var/log/monitor.1.gz
        rm /var/log/monitor.log
fi