Skip to content

Instantly share code, notes, and snippets.

@kapb14

kapb14/get-rps.md

Created June 19, 2019 14:55
Show Gist options
  • Save kapb14/b32942da09d9e18b76368ef4f0570cd2 to your computer and use it in GitHub Desktop.
Save kapb14/b32942da09d9e18b76368ef4f0570cd2 to your computer and use it in GitHub Desktop.
Download ZIP
Count lines per second in access.log
Raw
get-rps.md

get-rps

Counts "Lines per second" in access.log file.

Usage

whoami@hostname ~ $ get-rps --help

 Usage: get-rps [OPTIONS] LOG_FILE
   OPTIONS are:
    --help, -h    - Print information about basic usage.
    --debug, -d   - Debug mode
    --json, -j    - Output as JSON
    --logfile, -l - Log file
    --avg   - Outputs only average RPS

Examples

basic

whoami@hostname ~ $ get-rps
"timestamp":	"19/Jun/2019:17:48"
"rps_max":	"32"
"rps_min":	"6"
"rps_avg":	16

output as JSON

whoami@hostname ~ $ get-rps --json | jq
{
  "timestamp": "19/Jun/2019:17:48",
  "rps_max": 32,
  "rps_min": 6,
  "rps_avg": 16
}

output only average (int)

whoami@hostname ~ $ get-rps --logfile /var/log/nginx/access.log --avg
15

debug output with time

whoami@hostname ~ $ time get-rps --debug
  LOG_PATH:	/var/log/nginx/access.log
	DEBUG:		true
	JSON:		false
	LOG_NAME:	access.log
	TS:		19/Jun/2019:17:50
	TOTAL_LINES:	597059
	FIRST_LINE:	596033
	LAST_LINE:	596972
	LINES_TO_TAIL:	1026
	LINES_TO_HEAD:	939
1      19/Jun/2019:17:50:00 4          4
2      19/Jun/2019:17:50:01 24         24
3      19/Jun/2019:17:50:02 21         21
4      19/Jun/2019:17:50:03 16         16
5      19/Jun/2019:17:50:04 11         11
6      19/Jun/2019:17:50:05 15         15
7      19/Jun/2019:17:50:06 25         25
8      19/Jun/2019:17:50:07 22         22
9      19/Jun/2019:17:50:08 12         12
10     19/Jun/2019:17:50:09 18         18
11     19/Jun/2019:17:50:10 18         18
12     19/Jun/2019:17:50:11 16         16
13     19/Jun/2019:17:50:12 17         17
14     19/Jun/2019:17:50:13 21         21
15     19/Jun/2019:17:50:14 5          5
16     19/Jun/2019:17:50:15 11         11
17     19/Jun/2019:17:50:16 16         16
18     19/Jun/2019:17:50:17 16         16
19     19/Jun/2019:17:50:18 9          9
20     19/Jun/2019:17:50:19 15         15
21     19/Jun/2019:17:50:20 21         21
22     19/Jun/2019:17:50:21 16         16
23     19/Jun/2019:17:50:22 16         16
24     19/Jun/2019:17:50:23 26         26
25     19/Jun/2019:17:50:24 11         11
26     19/Jun/2019:17:50:25 13         13
27     19/Jun/2019:17:50:26 18         18
28     19/Jun/2019:17:50:27 17         17
29     19/Jun/2019:17:50:28 5          5
30     19/Jun/2019:17:50:29 13         13
31     19/Jun/2019:17:50:30 8          8
32     19/Jun/2019:17:50:31 14         14
33     19/Jun/2019:17:50:32 25         25
34     19/Jun/2019:17:50:33 19         19
35     19/Jun/2019:17:50:34 21         21
36     19/Jun/2019:17:50:35 14         14
37     19/Jun/2019:17:50:36 12         12
38     19/Jun/2019:17:50:37 18         18
39     19/Jun/2019:17:50:38 21         21
40     19/Jun/2019:17:50:39 8          8
41     19/Jun/2019:17:50:40 19         19
42     19/Jun/2019:17:50:41 22         22
43     19/Jun/2019:17:50:42 11         11
44     19/Jun/2019:17:50:43 8          8
45     19/Jun/2019:17:50:44 13         13
46     19/Jun/2019:17:50:45 8          8
47     19/Jun/2019:17:50:46 14         14
48     19/Jun/2019:17:50:47 15         15
49     19/Jun/2019:17:50:48 16         16
50     19/Jun/2019:17:50:49 15         15
51     19/Jun/2019:17:50:50 11         11
52     19/Jun/2019:17:50:51 6          6
53     19/Jun/2019:17:50:52 18         18
54     19/Jun/2019:17:50:53 13         13
55     19/Jun/2019:17:50:54 23         23
56     19/Jun/2019:17:50:55 23         23
57     19/Jun/2019:17:50:56 28         28
58     19/Jun/2019:17:50:57 17         17
59     19/Jun/2019:17:50:58 12         12
60     19/Jun/2019:17:50:59 13         13
          TIMESTAMP:    19/Jun/2019:17:50
        MAXIMUM RPS:    28
        MINIMUM RPS:    4
        AVERAGE RPS:    15

real	0m0.487s
user	0m0.164s
sys	0m0.120s
Raw
get-rps.sh
#!/bin/bash
NAME=$(basename $0)
DEBUG=${DEBUG:-false}
JSON=${JSON:-false}
LOG_PROVIDED=false
AVG_ONLY=false
TS="$(date +'%d/%b/%Y:%H:%M' --date='1 minute ago')"
function main(){
LOG_PATH="${LOG_PATH:-/var/log/nginx/access.log}"
LOG_NAME="$(basename ${LOG_PATH})"
${DEBUG} && __debug_inputs
TOTAL_LINES=$(wc -l ${LOG_PATH} | awk '{print $1}')
FIRST_LINE=$(_line_numbers_by_ts ${TS} | head -n 1)
LAST_LINE=$(_line_numbers_by_ts ${TS} | tail -n 1)
LINES_TO_TAIL=$((${TOTAL_LINES}-${FIRST_LINE}))
LINES_TO_HEAD=$((${LAST_LINE}-${FIRST_LINE}))
${DEBUG} && __debug_lines
TMPFILE=$(/bin/mktemp /tmp/${NAME}.tmp.XXXXXXXXX)
trap "{ /bin/rm -f $TMPFILE ; exit 255; }" EXIT
tail -n ${LINES_TO_TAIL} ${LOG_PATH} | head -n ${LINES_TO_HEAD} > ${TMPFILE}
SECONDS_ARRAY=$(echo {00..59})
REQS=()
CNT=0
for SEC in ${SECONDS_ARRAY}; do
((++CNT))
LPS=$(_lines_per_second ${SEC} ${TMPFILE} | wc -l)
REQS[$CNT]="${LPS}"
${DEBUG} && printf '%-5s %s:%s %-10s %-15s\n' "${CNT}" "${TS}" "${SEC}" "${LPS}" "${REQS[$CNT]}"
done
${AVG_ONLY} || MAX_RPS=$(echo ${REQS[@]} | jq -s max)
${AVG_ONLY} || MIN_RPS=$(echo ${REQS[@]} | jq -s min)
AVG_RPS=$(echo ${REQS[@]} | jq -s add/length | cut -d '.' -f 1)
__print_results
}
#### MISC FUNCTIONS AND UTILS
## get line numbers for timestamp (previous minute)
function _line_numbers_by_ts(){
grep -h -n "${TS}" ${LOG_PATH} | cut -d ':' -f 1
}
## get lines count per second
function _lines_per_second(){
SECOND=${1:-\d\d}
FILE=${2:-${LOG_PATH}}
egrep "${TS}:${SECOND}" ${FILE}
}
## print inputs
function __debug_inputs(){
cat <<EOF
LOG_PATH: ${LOG_PATH}
DEBUG: ${DEBUG}
JSON: ${JSON}
LOG_NAME: ${LOG_NAME}
TS: ${TS}
EOF
}
## print variables with lines calc
function __debug_lines(){
cat <<EOF
TOTAL_LINES: ${TOTAL_LINES}
FIRST_LINE: ${FIRST_LINE}
LAST_LINE: ${LAST_LINE}
LINES_TO_TAIL: ${LINES_TO_TAIL}
LINES_TO_HEAD: ${LINES_TO_HEAD}
EOF
}
## print result
function __print_results(){
if ${DEBUG}; then
cat <<EOF
TIMESTAMP: ${TS}
MAXIMUM RPS: ${MAX_RPS}
MINIMUM RPS: ${MIN_RPS}
AVERAGE RPS: ${AVG_RPS}
EOF
else
if ${AVG_ONLY}; then
printf '%s\n' "${AVG_RPS}"
elif ${JSON}; then
printf '{"timestamp":"%s","rps_max":%s,"rps_min":%s,"rps_avg":%s}\n' "${TS}" "${MAX_RPS}" "${MIN_RPS}" "${AVG_RPS}"
else
printf '"timestamp":\t"%s"\n"rps_max":\t"%s"\n"rps_min":\t"%s"\n"rps_avg":\t%s\n' "${TS}" "${MAX_RPS}" "${MIN_RPS}" "${AVG_RPS}"
fi
fi
}
## print usage info
function __print_help(){
cat <<EOF
Usage: ${NAME} [OPTIONS] LOG_FILE
OPTIONS are:
--help, -h - Print information about basic usage.
--debug, -d - Debug mode
--json, -j - Output as JSON
--logfile, -l - Log file
--avg - Outputs only average RPS
EOF
}
##################################
for OPT in "$@"; do
${DEBUG} && echo "OPT: ${OPT}"
case $OPT in
--help|-h|-\?|help) __print_help ; exit 1 ;;
--debug|-d) DEBUG=true ;;
--json|-j) JSON=true ;;
--logfile|-l) LOG_PROVIDED=true ;;
--avg) AVG_ONLY=true ;;
*) VALUE=${OPT} ;;
esac
done
#${DEBUG} && echo "[--logfile] LOG_PROVIDED: ${LOG_PROVIDED}"
if ${LOG_PROVIDED}; then
if [[ -f ${VALUE} ]]; then
LOG_PATH=${VALUE}
else
echo "FAIL: seems ${VALUE} is not a file"
exit 1
fi
fi
#__debug_inputs
##############################
main
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment