ghost
commented
1 year ago #!/bin/bash
# Get country IP addresses from RIPE and compile them into separate ipv4 and ipv6 plain lists
## See https://stat.ripe.net/docs/data_api
# Depends on jq - JSON processor.
## On Debian and derivatives, if you are missing jq then install it using this command:
## apt install jq
myversion=0.25.1
me=$(basename "$0")
ripe_url="https://stat.ripe.net/data/country-resource-list/data.json?v4_format=prefix&resource="
#### FUNCTIONS
usage() {
cat <<EOF
Usage: $me -c country [more options]
Options:
-c tld : tld/country code
-o output dir : Output directory (default: /tmp)
-S size : minimum ipv4 list file size to trust the data (default = 5000)
-s size : minimum ipv6 list file size to trust the data (default = 2000)
-d : Debug
-h : This help
EOF
}
die() {
echo "$@" 1>&2
exit 1
}
warn() {
er="$@"
echo "$er" 1>&2
errors+=("$er")
}
#### PARSE ARGUMENTS
while getopts "c:o:S:s:dh" opt; do
case $opt in
c) country=$OPTARG;;
o) out_dir=$OPTARG;;
S) min_size_ipv4=$OPTARG;;
s) min_size_ipv6=$OPTARG;;
d) debug=true;;
h) usage;;
esac
done
shift $((OPTIND -1))
min_size_ipv4=${min_size_ipv4:-5000} # default is 5000 bytes
min_size_ipv6=${min_size_ipv6:-2000} # default is 2000 bytes
out_dir=${out_dir:-/tmp}
if [ -z "$country" ]; then
usage
exit 1
fi
echo ""
url="$ripe_url$country"
# use curl or wget, depending on which one we find
curl_or_wget=$(if hash curl 2>/dev/null; then echo "curl -s"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi);
if [ -z "$curl_or_wget" ]; then
echo "Error: Neither curl nor wget found. Cannot download data." >&2
exit 1
fi
# check we have jq
if ! hash jq; then
echo "Error: Cannot find the jq Json processor. Install it with 'apt install jq' or similar" >&2
exit 1
fi
tmpfile=$(mktemp "/tmp/ripe-$country-XXXX.json")
[ $debug ] && echo "Debug: Trying: $curl_or_wget '$url'" >&2
$curl_or_wget "$url" > $tmpfile
rv=$?
if [ $rv -gt 0 ]; then
echo "Error $rv trying to run $curl_or_wget $url" >&2
exit $rv
else
echo "Successfully fetched $url"
fi
status=$(jq -r '.status' $tmpfile)
if [ ! "$status" = "ok" ]; then
ripe_msg=$(jq -r -c '.messages' $tmpfile)
echo "Error: RIPE replied with status = '$status'." >&2
echo "The requested url was '$url'" >&2
echo "and the messages in their reply were: '$ripe_msg'" >&2
echo "The full response is in $tmpfile" >&2
exit 1
fi
ripe_db_time=$(jq -r '.data.query_time' $tmpfile)
[ $debug ] && echo "Debug: ripe_db_time=$ripe_db_time" >&2
date_new=$(date -d "$ripe_db_time" +%s)
family="ipv4"
for family in ipv4 ipv6; do
filename="${family}_$country"
out_list="$out_dir/$filename"
if [ "$family" = "ipv6" ]; then min_size="$min_size_ipv6"; else min_size="$min_size_ipv4"; fi
if [ -f "$out_list" ]; then
date_old=$(stat --printf "%Y" "$out_list")
else
date_old=0
fi
if [ ! $date_new -gt $date_old ]; then
echo "Note: Not updating $filename because data querytime is the same as existing list time: $date_old ($(date --date=@$date_old))." >&2
fi
if [ $date_new -gt $date_old ]; then
[ $debug ] && echo "Debug: Querytime is newer than of old list $filename, updating..."
jq -r ".data.resources.$family | .[]" "$tmpfile" > "$out_list.new"
touch -d "$ripe_db_time" "$out_list.new"
### Check for minimum size before updating our list
size=$(stat --printf %s "$out_list.new")
[ $debug ] && echo "Debug: $filename size: $size"
[ $debug ] && echo "Debug: min_size for $family: $min_size"
if [ "$size" -gt "$min_size" ]; then
echo "Updating $out_list"
mv "$out_list.new" "$out_list"
else
echo "Error: fetched $filename size of $size bytes not greater than minimum $min_size bytes. Probably a download error. Not updating $filename." >&2
dt=$(date +%F_%H%M%S)
echo "Removing previous bad lists for $filename" >&2
rm "$out_list"*.bad
echo "Saving downloaded list for reference as $out_list-$dt.bad" >&2
mv "$out_list.new" "$out_list-$dt.bad"
fi
fi
done
[ $debug ] && echo "Debug: Removing $tmpfile"
rm $tmpfile
echo ""
Hi, first of all - thank you for making this script public! I'll be using it on my server. Second, I modified it to solve a bug, and to improve the notifications it provides to the user as well. [and some minor improvements to the code] [including adding code that deletes old .bad files when creating a new one, in order to prevent situation where something in the script breaks down the road and bad files start to pile up and take up a lot of storage space] [I also bumped the version to 0.25.1]
The bug was that querytime check was prior to and outside of the "for" loop, however the file length check was inside the loop. That created an edge case where on the first attempt the script fetches a partially corrupted list, and then the user gets stuck with it until the querytime changes. I simply inserted the querytime check into the loop and made it specific to family (ipv4 or ipv6).
I also changed the code to have minimum valid list size specific for each family, since ipv4 lists can be expected to be quite longer than ipv6 lists. For my country, the ipv6 list is about 2200 bytes while ipv4 list is about 13300 bytes. I set default value for min_size_ipv4 to 5000 bytes and for min_size_ipv6 to 2000 bytes.
I'm not a professional dev so I won't be submitting pull requests. Also I may have introduced new bugs (although I did test my code and it seems to be working fine). But I wanted to share the complete modified script, in the hope that it will be useful.
I'll post it in first comment.