Hi All,
Sometimes, we have a situation to read recent logs from the log file which continuously growing from the current logs; source can be anything like system logs, application logs, database logs, device logs etc etc...
My Problem- Need to read all recent slow queries of mongo (nosql database) logs and notified when it breaches threshold (400ms).
Solution- It can be achieved by using various scripting/programming language like node.js or other scripting language, I opt 'BASH script.
#!/bin/bash
## Please update the value according to your system/environment.
## set variable
logfilepath=/var/log/mongo # please change the path as per the environment
name=mongod.log # mongo log file name
logrecord=<path_to_capture_current_script_log> # for recording current script log
MaxLLN_file=<file_to_hold_max_LLN_value> # for maintaing Max Last Line Number
slow_query_log_record=<logged_only_slow_queries>
tmpfile=<temp_file_path>
mailnotification=<notification_filename_which we have to monitored/display/forward>
## Set variable for mail
to='abc@xyz.com'
from='Sender:xxx'
sub='<subject_of_the_mail>'
## No need to change/update anything here. Please change cautiously,if needed.
## If file exists having start line number then save it into a file.
if [ -f $MaxLLN_file ];then
echo -e "[`date +%F_%T`] $MaxLLN_file File exist, max LLN value is copied" >> $logrecord
startlineno=`cat $MaxLLN_file`
else
# If file is not exist
echo -e "[`date +%F_%T`] $MaxLLN_file doesn't exist so startlineno counter start from 1" >> $logrecord
startlineno=1
echo 1 > $MaxLLN_file
fi
while true
do
startlineno=`cat $MaxLLN_file`
totallineno=`wc -l $logfilepath/$name|awk '{print $1}'`
if [ $startlineno -le $totallineno ];then
echo -e "[`date +%F_%T`] Slow log search start from between $startlineno untill $totallineno line number" >> $logrecord
slowqeries_count=`sed -n "$startlineno","$totallineno"p $logfilepath/$name | grep -c 'ms'`
if [ $slowqeries_count -ge 1 ];then
echo -e "[`date +%F_%T`] $slowqeries_count slow queries found between $startlineno and $totallineno line number" >> $logrecord
sed -n "$startlineno","$totallineno"p $logfilepath/$name | grep 'ms' > $tmpfile
`cat $tmpfile >> $slow_query_log_record`
echo -e "Dear All,\n\nBelow are the Slow query logs of Mongo Server `hostname` ( taking greater than or equal to 400ms)\n\n" > $mailnotification
cat $tmpfile | while read line
do
if [ `echo $line | gawk '{print $NF-400}'` -ge 0 ];then echo $line >> $mailnotification
fi
done
echo -e "\n\n Thanks & Best Regards,\nAnurag Bisht" >> $mailnotification
echo "Subject: $sub" | cat - $mailnotification | sendmail -F"$from" -t "$to"
echo > $mailnotification;
else
echo -e "[`date +%F_%T`] $slowqeries_count Slow queries found between $startlineno and $totallineno line number" >> $logrecord
fi
totallineno=`expr $totallineno + 1`
echo $totallineno > $MaxLLN_file
echo -e "[`date +%F_%T`] New search will start from `cat $MaxLLN_file` value" >> $logrecord
echo -e "---------------------------------" >> $logrecord
fi
done