How to analyze logs
1. Get logs and send them to vendor
An archive file will be generated containing all logs.
CODE
/etc/veridiumid/scripts/getLogs.sh
## or for a specific period
/etc/veridiumid/scripts/getLogs.sh 20230115 20230116 2. In case that a specific error is investigated, all logs can be tailed with one command.
CODE
tail -f /var/log/veridiumid/*/*log /var/log/veridiumid/*/*out
## in case that error in tomcat are checked
LOG=/var/log/veridiumid/tomcat/*
## in case that errors in any logfile are checked
LOG=/var/log/veridiumid/*/*
## get a few lines for all errors, to look faster through the log
## this is searching only active logs, not compressed
grep -A4 -B4 -e ERROR -e FATAL ${LOG} | less
3. Get error statistics, useful to see if there are new errors
CODE
## in case that error in tomcat are checked
LOG=/var/log/veridiumid/tomcat/*
## in case that errors in any logfile are checked
LOG=/var/log/veridiumid/*/*
## please set the period for the logs
## results are group by hour
find $LOG -newermt "2023-12-20 00:00:00" \! -newermt "2023-12-21 00:00:00" -exec zgrep -e ERROR -e FATAL {} /dev/null \; | grep -v "localhost_access" | awk -F' ' '{print $1,substr($2,0,2),$6,$7}' | sort | uniq -c
## please set the period for the logs,
## results are group by minute
find /var/log/veridiumid/*/* -newermt "2023-12-20 00:00:00" \! -newermt "2023-12-21 00:00:00" -exec zgrep -e ERROR -e FATAL {} /dev/null \; | grep -v "localhost_access" | awk -F' ' '{print $1,substr($2,0,5),$6,$7}' | sort | uniq -c
4. In order to check response time for a specific application (in access log)
CODE
## please set the LOG parameter for a specific date
LOG=/var/log/veridiumid/selfservice/*access_log*2023-12-20*
LOG=/var/log/veridiumid/tomcat/*access_log*2023-12-20*
LOG=/var/log/veridiumid/fido/*access_log*2023-12-20*
LOG=/var/log/veridiumid/websecadmin/*access_log*2023-12-20*
## or all the access logs
LOG=/var/log/veridiumid/*/*access_log*2023-12-20*
## response time every hour
echo "DATE Responce_time NumberOfRequests"
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 12 ) {print $6":"$8":"$9":"$11} else {print $5":"$7":"$8":"$10} }' | awk -F':' '{sum[$1":"$2] += $NF; count[$1":"$2] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort
## response time every minute
echo "DATE Responce_time NumberOfRequests"
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 12 ) {print $6":"$8":"$9":"$11} else {print $5":"$7":"$8":"$10} }' | awk -F':' '{sum[$1":"$2":"$3] += $NF; count[$1":"$2":"$3] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort
## responses by resonse code, every hour
echo "DATE Responce_time NumberOfRequests"
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 12 ) {print $6":"$8":"$9":"$11} else {print $5":"$7":"$8":"$10} }' | awk -F':' '{sum[$(NF-1)":"$1":"$2] += $NF; count[$(NF-1)":"$1":"$2] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort
## responses time by API
echo "DATE Responce_time NumberOfRequests"
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 12 ) {print $6":"$8":"$9":"$11} else {print $5":"$7":"$8":"$10} }' | awk -F':' '{sum[$5":"$1":"$2] += $NF; count[$5":"$1":"$2] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort
5. Haproxy response time:
CODE
LOG=/opt/veridiumid/haproxy/logs/haproxy.log
## check if there are any queues on TCP or HTTPS - if there are queues, than it should not be 0/0
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 20 ) {print $7":"$8":"$9":"$17} else {print $7":"$8":"$9":"$NF} }' | awk -F':' '{count[$1":"$2":"$5":"$6":"$NF] += 1} END { for ( key in count ) { print key, count[key] } }' | sort | grep -v "0/0"
## only for HTTP requets, calculate response time per hour:
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 20 ) {print $7":"$10":"$11":"$(NF-2)} }' | awk -F':' '{split($5, arr, "/"); sum[$1":"$2] += arr[1]; count[$1":"$2] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort
## only for HTTP requets, calculate response time per minute:
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 20 ) {print $7":"$10":"$11":"$(NF-2)} }' | awk -F':' '{split($5, arr, "/"); sum[$1":"$2":"$3] += arr[1]; count[$1":"$2":"$3] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort
## only for HTTP requets, calculate response time per hour, per API:
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 20 ) {print $7":"$10":"$11":"$(NF-2)} }' | awk -F':' '{split($5, arr, "/"); sum[$NF":"$1":"$2] += arr[1]; count[$NF":"$1":"$2] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort
## only for HTTP requets, calculate response time per hour, per API, per response status:
sed -E ':A;s/("[^ "]+) ([^"]*")/\1_\2/;tA' ${LOG} | awk -F' ' '{if ( NF > 20 ) {print $7":"$10":"$11":"$(NF-2)} }' | awk -F':' '{split($5, arr, "/"); sum[$(NF-1)":"$NF":"$1":"$2] += arr[1]; count[$(NF-1)":"$NF":"$1":"$2] += 1 } END { for ( key in count ) { print key, sum[key] / count[key], count[key] } }' | sort