Move my scripts to scripts/

scripts/extract-scanner-ips.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+
+set -eux
+
+grep -v -F ":" digitalocean_announced_ips.txt | xargs -I {} sh -c "echo '{} # $(dig +short -x {})'" > digitalocean_announced_ips_with_reverse.txt
+
+
+# Fetch all reverse DNS addresses
+# ns3.digitalocean.com = 198.41.222.173
+cat digitalocean_announced_ips_full.txt | xargs -n 1 -P 40 dig @198.41.222.173 +short +time=5 +tries=10 -x > digitalocean_announced_ips_full_reverse.txt
+
+# Test command
+# cat digitalocean_announced_reverse_dns.txt | grep -v -F "ip6.arpa" | sed 's/.in-addr.arpa//'  | awk -F. '{print $3"." $2"."$1}' | sort | less
+
+# A sample
+#cat digitalocean_announced_ips_full.txt | xargs -n 1 -P 40 dig @198.41.222.173 +short +time=5 +tries=10 -x > digitalocean_announced_ips_full_reverse.txt
+
+cat digitalocean_announced_ips_full.txt | xargs -P 40  -I {} sh -c 'set -eu;rev="$(dig @198.41.222.173 +short +time=5 +tries=10 -x {})";echo "{} # $rev";' > digitalocean_announced_ips_full_reverse_better.txt
+
+
+grep -F -x -v -f digitalocean_announced_ips_full_reverse_better_only_ips.txt digitalocean_announced_ips_full.txt
+sed -i 's/ # $//' digitalocean_announced_ips_full_reverse_better.txt
+sort digitalocean_announced_ips_full_reverse_better.txt > digitalocean_announced_ips_full_reverse_better2.txt
+mv digitalocean_announced_ips_full_reverse_better2.txt digitalocean_announced_ips_full_reverse_better.txt
+
+diff -u digitalocean_announced_ips_full_reverse_better_only_ips.txt digitalocean_announced_ips_full.txt | delta
+cut -d ' ' -f 1 digitalocean_announced_ips_full_reverse_better.txt > digitalocean_announced_ips_full_reverse_better_only_ips.txt
+
+# Find all results
+grep -F "stretchoid" digitalocean_announced_ips_full_reverse_better.txt | cut -d " " -f 3 | sort
+
+
+# Find all ranges
+grep -F "stretchoid" digitalocean_announced_ips_full_reverse_better.txt | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq
+
+
+# Make a list of search keys
+grep -F "stretchoid" digitalocean_announced_ips_full_reverse_better.txt | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq > found_ranges.txt
+
+# Find all ranges to re-scan
+cat found_ranges.txt | xargs -I {} grep -F "{}" digitalocean_announced_ips.txt | sort
+
+# Compare with debian-scripts
+grep -F "add stretchoid" stretchoid.ipset | cut -d ' ' -f 3 | cut -d '.' -f -3 | sort | uniq > found_ranges.txt
+cat found_ranges.txt | xargs -I {} grep -F "{}" digitalocean_announced_ips.txt | sort > stretchoid_ranges_debian_scripts.txt
+
+
+# Re scan
+dig -4 +noauthority +noadditional +nostats -x 107.170.202.77 @1.0.0.1
+
+cat stretchoid_ranges.txt | xargs -n1 prips | uniq | sort -V > stretchoid_possible_ips.txt
+cat binaryedge_ranges.txt | xargs -n1 prips | uniq | sort -V > binaryedge_digitalocean_possible_ips.txt
+
+# With failure handling
+cat stretchoid_digitalocean_possible_ips.txt | xargs -P 50 -I {} bash -c 'set -eu;rev="$(dig @9.9.9.9 +short +time=1 +tries=1 -x {})"; if [[ "$rev" == *";;"* ]]; then sleep 1; rev="$(dig @8.8.8.8 +short +time=1 +tries=1 -x {})"; fi; echo "{} # $rev";' 1> stretchoid_revisions/v5.txt
+
+grep -F "stretchoid" stretchoid_revisions/v5.txt | sort > stretchoid_revisions/v5.sorted.txt
+mv stretchoid_revisions/v5.sorted.txt stretchoid_revisions/v5.txt
+
+# Reverse the file
+awk -F'#' '{print $2" # "$1}' OFS=, "stretchoid_revisions/v5.txt" | awk '{$1=$1;print}' | sort > stretchoid_revisions/v5-reversed.txt
+
+# Build the count per name per ip
+cat stretchoid_revisions/v*-reversed.txt | sort | uniq -c > stretchoid_revisions/count-reversed.txt
+# Same but sorted not by name but by count
+cat stretchoid_revisions/v*-reversed.txt | sort | uniq -c | sort > stretchoid_revisions/count-reversed.txt
+
+# Generate the list of full IPs of stretchoid
+cat stretchoid_revisions/v*-reversed.txt | sort | uniq | awk -F'#' '{print $2" # "$1}' OFS='#' | awk '{$1=$1;print}' > ../stretchoid.txt
+
+
+
+# Find missing CIDRs from the IPS found in the revisions using the announced prefix list
+cat binaryedge_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq | cut -d ' ' -f 3 | sort -V | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq | xargs -I {} grep -E "^{}\.0" digitalocean_announced_ips_simpler.txt | sort -V -t# | uniq | cut -d ' ' -f 3 | sort -V | uniq | grep -v -F -f ./binaryedge_ranges.txt
+
+cat binaryedge-full-possible-names_with_ips_clean_ips.txt | sort -V | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq | xargs -I {} grep -E "^{}\.0" digitalocean_announced_ips_simpler.txt | sort -V -t# | uniq | cut -d ' ' -f 3 | sort -V | uniq | grep -v -F -f ./binaryedge_ranges.txt

scripts/make-aws-cloudfront-range.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -eu
+
+# See: https://stackoverflow.com/a/69768584/5155484
+curl -f -s -# https://ip-ranges.amazonaws.com/ip-ranges.json | jq -r '.prefixes[] | select(.service == "CLOUDFRONT") | .ip_prefix' | sort -V > cloudfront-ips.txt
+curl -f -s -# https://ip-ranges.amazonaws.com/ip-ranges.json | jq -r '.ipv6_prefixes[] | select(.service == "CLOUDFRONT") | .ipv6_prefix' | sort -V >> cloudfront-ips.txt
+
+# Does not seem up to date: 06-2023
+#curl https://d7uri8nf7uskq.cloudfront.net/tools/list-cloudfront-ips | jq -r '.CLOUDFRONT_GLOBAL_IP_LIST  | join("\n")' | sort > cloudfront-ips.txt

scripts/make-binaryedge.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+set -eux
+
+############################################################################################
+#                                 Information                                              #
+# The program dns-ptr-resolver can be installed from cargo: cargo install dns-ptr-resolver #
+# See: https://github.com/wdes/dns-ptr-resolver                                            #
+############################################################################################
+
+REV="v-$(date --iso-8601=seconds)"
+
+cd ./data/
+
+if [ ! -d ./binaryedge_revisions/ ]; then
+    mkdir ./binaryedge_revisions
+fi
+
+if [ ! -d ./reverse_revisions/ ]; then
+    mkdir ./reverse_revisions
+fi
+
+curl -A "https://github.com/wdes/security" https://api.binaryedge.io/v1/minions | jq -r '.scanners[]' | sed '/^$/d' | sort -V > ./binaryedge_api_ips.txt
+curl -A "https://github.com/wdes/security" https://api.binaryedge.io/v1/minions-ipv6 | jq -r '.scanners[]' | sed '/^$/d' | sort -V >> ./binaryedge_api_ips.txt
+
+doRev () {
+    dns-ptr-resolver $PWD/$1 1> binaryedge_revisions/$REV.txt
+
+    grep -F "binaryedge" binaryedge_revisions/$REV.txt | sort -V > binaryedge_revisions/$REV.sorted.txt
+    grep -v -F "binaryedge" binaryedge_revisions/$REV.txt | sort -V > reverse_revisions/$REV.sorted.txt
+    mv binaryedge_revisions/$REV.sorted.txt binaryedge_revisions/$REV.txt
+    mv reverse_revisions/$REV.sorted.txt reverse_revisions/$REV.txt
+
+    # Reverse the file
+    awk -F'#' '{print $2" # "$1}' OFS=, "binaryedge_revisions/$REV.txt" | awk '{$1=$1;print}' | sort > binaryedge_revisions/$REV-reversed.txt
+
+    # Sort by name and reverse the list to build the list of all possible IPs
+    cat binaryedge_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq | awk -F'#' '{print "# "$1" \n "$2}' OFS='#' | awk '{$1=$1;print}' > ../binaryedge.txt
+
+    grep -F '#' ../binaryedge.txt | cut -d ' ' -f 2 | sort | cut -d. -f-1 | rev | cut -d '-' -f2- | rev | sort | uniq -c | sort > ./binaryedge-chunk-counts.txt
+}
+
+doRev "binaryedge_api_ips.txt"
+doRev "binaryedge_digitalocean_possible_ips.txt"
+
+# Search for false positives
+# dns-ptr-resolver ../binaryedge.txt | grep -v -F "binaryedge.com"

scripts/make-stretchoid.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -eux
+
+############################################################################################
+#                                 Information                                              #
+# The program dns-ptr-resolver can be installed from cargo: cargo install dns-ptr-resolver #
+# See: https://github.com/wdes/dns-ptr-resolver                                            #
+############################################################################################
+
+REV="v-$(date --iso-8601=seconds)"
+
+cd ./data/
+
+if [ ! -d ./stretchoid_revisions/ ]; then
+    mkdir ./stretchoid_revisions
+fi
+
+if [ ! -d ./reverse_revisions/ ]; then
+    mkdir ./reverse_revisions
+fi
+
+dns-ptr-resolver $PWD/stretchoid_possible_ips.txt 1> stretchoid_revisions/$REV.txt
+
+grep -F "stretchoid" stretchoid_revisions/$REV.txt | sort -V > stretchoid_revisions/$REV.sorted.txt
+grep -v -F "stretchoid" stretchoid_revisions/$REV.txt | sort -V > reverse_revisions/$REV.sorted.txt
+mv stretchoid_revisions/$REV.sorted.txt stretchoid_revisions/$REV.txt
+mv reverse_revisions/$REV.sorted.txt reverse_revisions/$REV.txt
+
+# Reverse the file
+awk -F'#' '{print $2" # "$1}' OFS=, "stretchoid_revisions/$REV.txt" | awk '{$1=$1;print}' | sort > stretchoid_revisions/$REV-reversed.txt
+
+# Sort by name and reverse the list to build the list of all possible IPs
+cat stretchoid_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq | awk -F'#' '{print "# "$1" \n "$2}' OFS='#' | awk '{$1=$1;print}' > ../stretchoid.txt
+
+grep -F '#' ../stretchoid.txt | cut -d- -f2 | grep -P '^[0-9]{3,}+' | sort | uniq -c | sort > ./stretchoid-chunk-counts.txt
+
+# Search for false positives
+# dns-ptr-resolver ../stretchoid.txt | grep -v -F "stretchoid.com"

scripts/update-announced-ips.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+set -eu
+
+if [ -z "${1:-}" ]; then
+    echo 'Missing the user agent as a first argument'
+    exit 1;
+fi
+
+UA="$1"
+
+set -x
+
+cd ./data/
+
+# Fetch digitalocean declared IPs
+curl https://digitalocean.com/geo/google.csv -s -L -# -o - | cut -d ',' -f 1 | sort | uniq > digitalocean_ips.txt
+
+# List all routable routes for the AS 14061
+curl https://bgp.tools/table.txt -A "$UA" -s | grep -e ' 14061$' | wc -l
+
+# Fetch all announced IPs
+curl https://bgp.tools/table.txt -A "$UA" -s | grep -e ' 14061$' | cut -d ' ' -f 1 | sort | uniq > digitalocean_announced_ips.txt
+
+# Compare the declared IPs and announced IPs
+diff -u digitalocean_ips.txt digitalocean_announced_ips.txt > digitalocean_ips_vs_announced_ips.diff
+
+# Generate the full IP list to check PTRs
+grep -v -F ":" digitalocean_announced_ips.txt | xargs -n1 prips > digitalocean_announced_ips_full.txt
+
+# Make all digitalocean announced IPS simpler /24 networks
+grep -v -F ":" digitalocean_announced_ips.txt | xargs -I {} sh -c "prips -i 256 '{}' | awk '\$0=\"\"\$0\" # {}\"'" > digitalocean_announced_ips_simpler.txt
+
+# Some test command to get all declared reverse DNS objects at RIPE
+#curl 'https://apps.db.ripe.net/db-web-ui/api/rest/fulltextsearch/select?format=json&rows=10000&q=(nserver:(digitalocean.com))%20AND%20(object-type:domain)' -H 'Accept: application/json' -A "$UA" | jq -r '.result.docs | map(.doc.strs) | .[] | map(select(.str.name=="domain")) | map(.str.value) | .[] ' > digitalocean_announced_reverse_dns.txt
+
+# Build IPs CIDRs with found reverse DNS servers
+grep -v -F ":" digitalocean_announced_ips.txt | xargs -I {} sh -c "cidr="$(echo '{}' | cut -d '/' -f 1)"; dig +nocomments -x \$cidr | grep -v -F ';' | grep -e '.*\.in-addr\.arpa\.' | echo '{}'" > digitalocean_announced_ips_with_reverse.txt