Skip to content

Instantly share code, notes, and snippets.

@QueuingKoala

QueuingKoala/bogon-updater

Last active January 6, 2020 23:45
Show Gist options
  • Save QueuingKoala/1c201e9c9f32861f4479cab134a72ce8 to your computer and use it in GitHub Desktop.
Save QueuingKoala/1c201e9c9f32861f4479cab134a72ce8 to your computer and use it in GitHub Desktop.
Download ZIP
OpenWRT bogon-updater called from cron or on-boot
Raw
bogon-updater
#!/bin/sh
# Designed for OpenWRT and requires associated "bogons" UCI file.
#
# This is still pretty raw now and not formally published.
# Copyright 2018.
#
# Feel free to use, edit, and re-distribute under the BSD 3-clause License.
# https://opensource.org/licenses/BSD-3-Clause
. /lib/functions.sh
BIN_IPSET="/usr/sbin/ipset"
usage_exit() {
cat <<_EOH
USAGE: $(basename "$0")
[ -l | -u ]
ACTIONS:
-l: load bogons from cache file
-u: check for and perform an update if needed
_EOH
exit 0
}
die() {
log "$1"
echo "ERROR(bogon-updater): $1" >&2
exit "${2:-1}"
}
log() {
logger -t "bogon-updater" "$1"
}
parse_config() {
config_load bogons
config_foreach load_settings settings
if [ "$cache_mkdir" -eq 1 ]; then
mkdir -p "$cache_dir" || die "failed to create cache_dir"
fi
[ -d "$cache_dir" ] || die "cache_dir does not exist: $cache_dir"
TS="$cache_dir/$file_ts"
GZ="$cache_dir/$file_gz"
now=$(date +%s)
last_update=0
last_change=0
[ -r "$TS" ] && {
read last_update
read last_change
} < "$TS"
next_update=$((last_update + valid_hours * 3600))
}
get_gz_time() {
gz_time=0
if [ -r "$GZ" ]; then
gz_time=$(gzip -dc "$GZ" | awk '{print $4; exit 0;}')
[ "$gz_time" -ge 0 ] || gz_time=0
fi
}
load_settings() {
local cfg="$1"
config_get_bool enabled "$cfg" enabled 1
[ "$enabled" -eq 1 ] || exit 0
config_get_bool cache_mkdir "$cfg" cache_mkdir 1
config_get cache_dir "$cfg" cache_dir '/tmp/bogons'
config_get shm_dir "$cfg" shm_dir '/dev/shm'
config_get valid_hours "$cfg" valid_hours 24
config_get ipset_name "$cfg" ipset_name 'bogons'
config_get ipset_options "$cfg" ipset_options ''
config_get growth_size "$cfg" growth_size 0
[ -d "$shm_dir" ] || die "shm_dir must exist"
config_get uri_scheme "$cfg" uri_scheme
config_get uri_domain "$cfg" uri_domain
config_get uri_gz "$cfg" uri_gz
config_get uri_ts "$cfg" uri_ts
config_get uri_tries "$cfg" uri_tries 5
config_get uri_delay "$cfg" uri_delay 60
[ -n "$uri_scheme" ] || die "No uri_scheme defined"
[ -n "$uri_domain" ] || die "No uri_domain defined"
[ -n "$uri_ts" ] || die "No uri_ts defined"
[ -n "$uri_gz" ] || die "No uri_gz defined"
[ "$uri_tries" -ge 1 ] || die "uri_tries must be >= 1"
[ "$uri_delay" -ge 0 ] || die "uri_delay must be >= 0"
config_get file_gz "$cfg" file_gz "${uri_gz##*/}"
config_get file_ts "$cfg" file_ts "${uri_ts##*/}"
}
bogon_loader() {
[ -r "$GZ" ] || die "No bogons file availble to load"
n_entries=$(gzip -dc "$GZ" | wc -l)
n_entries=$((n_entries - 1 + growth_size))
runfile=$(mktemp "$shm_dir/bogon-loader.XXXXXX") \
|| die "mktemp failed to create runfile"
trap cleanup INT TERM EXIT
echo "$$" > "$runfile"
tmp_set="loader_${runfile##*.}"
"$BIN_IPSET" create "$tmp_set" 'hash:net' maxelem "$n_entries" $ipset_options \
|| die "failed to create temp set: $tmp_set"
gzip -dc "$GZ" \
| sed -E "1d;s/^/add $tmp_set /" \
| "$BIN_IPSET" restore \
|| die "bogon temp ipset restore failed"
"$BIN_IPSET" swap "$tmp_set" "$ipset_name" \
|| die "bogon ipset swap failed"
# sets $gz_time >= 0 based on comment line in $GZ
get_gz_time
log "Load successful (updated: $gz_time)"
}
update_bogons() {
if [ "$now" -lt "$next_update" ]; then
log "next update ($next_update) has not passed: nothing to do"
return 0
fi
download "$uri_scheme" "$uri_domain" "$uri_ts" "$file_ts" \
|| die "download of timestamp file failed"
this_update=0
this_change=0
{
read this_update
read this_change
} < "$TS"
# sets $gz_time >= 0 based on comment line in $GZ
get_gz_time
if [ "$this_change" -le "$gz_time" ]; then
log "Last download ($gz_time) includes current change ($this_change)"
return 0
fi
download "$uri_scheme" "$uri_domain" "$uri_gz" "$file_gz" \
|| die "download of bogons file failed"
log "Loading new bogons (updated: $this_update) (changed: $this_change)"
bogon_loader
}
download() {
local uri="${1}${2}${3}"
local file="$4"
local try_num=0
local dl_dir="$cache_dir/dl"
if [ ! -d "$dl_dir" ]; then
mkdir "$dl_dir" || {
log "could not mkdir $dl_dir"
return 1
}
fi
while [ "$try_num" -lt "$uri_tries" ]; do
if [ "$try_num" -gt 0 ]; then
log "failed download attempt $try_num (sleeping $uri_delay s)"
sleep "$uri_delay"
fi
try_num=$((try_num + 1))
rm -f "$dl_dir/$file"
wget -q -P "$dl_dir" "$uri" 2>/dev/null && break
done
rv=$?
if [ "$rv" -eq 0 ]; then
mv "$dl_dir/$file" "$cache_dir" || return 1
return 0
fi
return "$rv"
}
cleanup() {
if [ -n "$tmp_set" ]; then
"$BIN_IPSET" destroy "$tmp_set"
tmp_set=""
fi
[ -e "$runfile" ] && rm -f "$runfile"
}
INVOKE="usage_exit"
while getopts hlu opt
do
case "$opt" in
h) ;;
l) INVOKE="bogon_loader" ;;
u) INVOKE="update_bogons" ;;
?) die "Bad options"
esac
done
shift $((OPTIND - 1))
parse_config
"$INVOKE" "$@"
exit 0
Raw
bogons.init
#!/bin/sh /etc/rc.common
# Copyright 2017 Josh Cepek <josh.cepek AT usa.net>
# Licensed under the 3-clause BSD License:
#
# https://opensource.org/licenses/BSD-3-Clause
NAME=bogons
START=42
bogon_startup() {
local cfg="$1"
config_get_bool enabled "$cfg" updater_enabled 1
config_get loader_bin "$cfg" updater_path
if [ "$enabled" -ne 1 ] || [ ! -x "$loader_bin" ]; then
return 0
fi
"$loader_bin" -l
}
start() {
config_load bogons
config_foreach bogon_startup settings
}
restart() {
start
}
Raw
bogons.uci
# Bogons loader configuration
config settings
option cache_mkdir 1
option cache_dir '/tmp/bogons'
option shm_dir '/dev/shm'
option valid_hours 24
option ipset_name 'bogons'
#option growth_size 0
#option ipset_options ''
# For optional on-boot initscript operation:
#option updater_enabled 1
#option updater_path '/usr/sbin/bogon-updater'
# Where to download files from.
option uri_scheme 'https://'
option uri_domain 'my-cdn.example.net'
option uri_gz '/bogons/fullbogons-ipv4.gz'
option uri_ts '/bogons/fullbogons-ipv4-ts.txt'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment