Add more data from the binaryedge API

Ref: https://github.com/wravoc/authlog-threats/issues/3
This commit is contained in:
2023-08-13 21:38:01 +02:00
parent 24df6cdefa
commit b9fc5e9aa5
8 changed files with 24550 additions and 147 deletions

File diff suppressed because it is too large Load Diff

View File

@ -1,3 +1,85 @@
108 prod-boron-ams3
108 prod-boron-lon1
108 prod-boron-nyc1
108 prod-boron-nyc3
108 prod-boron-sgp1
10 prod-barium-fra1
10 prod-barium-sfo2
10 prod-barium-sfo3
10 prod-barium-tor1
10 prod-beryllium-ap-northeast
10 prod-beryllium-ap-south
10 prod-beryllium-ap-southeast
10 prod-beryllium-ap-west
10 prod-beryllium-ca-central
10 prod-beryllium-eu-central
10 prod-beryllium-eu-west
10 prod-beryllium-us-central
10 prod-beryllium-us-east
10 prod-beryllium-us-southeast
10 prod-beryllium-us-west
10 prod-jerry-se-clients-ams3
10 prod-jerry-se-clients-lon1
10 prod-jerry-se-clients-nyc1
10 prod-jerry-se-clients-sfo2
10 prod-jerry-se-clients-tor1
10 prod-jerry-se-endor-lon1
10 prod-jerry-se-endor-syd1
10 prod-jerry-se-hoth-ap-southeast
10 prod-jerry-se-hoth-eu-west
10 prod-jerry-se-jakku-lon1
10 prod-jerry-se-jakku-syd1
10 prod-jerry-se-kamino-lon1
10 prod-jerry-se-kamino-syd1
10 prod-jerry-se-kashyyyk-lon1
10 prod-jerry-se-kashyyyk-syd1
10 prod-jerry-se-managers-eu-west
10 prod-magnesium-lon1
10 prod-magnesium-nyc1
10 prod-magnesium-nyc3
10 prod-magnesium-sgp1
10 prod-manganese-ams3
10 prod-manganese-fra1
10 prod-manganese-lon1
10 prod-manganese-nyc1
10 prod-manganese-nyc3
10 prod-manganese-sfo2
10 prod-manganese-sfo3
10 prod-manganese-sgp1
10 prod-manganese-syd1
10 prod-manganese-tor1
10 prod-silicon-fra1
10 prod-silicon-sfo2
10 prod-silicon-sfo3
10 prod-silicon-tor1
110 prod-boron-fra1
110 prod-boron-sfo2
110 prod-boron-sfo3
110 prod-boron-tor1
120 prod-jerry-se-scanners-ams3
120 prod-jerry-se-scanners-lon1
120 prod-jerry-se-scanners-nyc1
120 prod-jerry-se-scanners-sfo2
120 prod-jerry-se-scanners-tor1
12 prod-jerry-se-jakku-tor1
12 prod-jerry-se-kamino-tor1
12 prod-jerry-se-kashyyyk-tor1
12 prod-magnesium-ams3
12 prod-magnesium-fra1
12 prod-magnesium-sfo2
12 prod-magnesium-sfo3
12 prod-magnesium-tor1
15 prod-jerry-se-endor-ca-central
16 prod-jerry-se-naboo-nyc1
16 prod-jerry-se-naboo-sfo2
16 prod-sulfur-nyc1
16 prod-sulfur-nyc3
16 prod-sulfur-sfo2
16 prod-sulfur-sfo3
16 prod-sulfur-tor1
180 prod-jerry-se-hoth-nyc1
180 prod-jerry-se-hoth-sfo2
18 jerry-se-do-na-east-unbounds
1 dev-barium-fra1 1 dev-barium-fra1
1 dev-barium-sfo2 1 dev-barium-sfo2
1 dev-barium-sfo3 1 dev-barium-sfo3
@ -6,55 +88,108 @@
1 dev-beryllium-lon1 1 dev-beryllium-lon1
1 dev-mercury-sfo2 1 dev-mercury-sfo2
1 dev-mercury-sfo3 1 dev-mercury-sfo3
1 prod-jerry-se-naboo-lon1
1 prod-jerry-se-non-allowlisted-lon1 1 prod-jerry-se-non-allowlisted-lon1
1 prod-jerry-se-utils-lon1 20 jerry-se-do-eu-central-unbounds
1 prod-mendelevium-sfo3 20 jerry-se-do-eu-west-unbounds
20 jerry-se-do-na-central-unbounds
20 jerry-se-do-na-west-unbounds
20 prod-beryllium-ams3
20 prod-beryllium-fra1
20 prod-beryllium-lon1
20 prod-beryllium-nyc1
20 prod-beryllium-nyc3
20 prod-beryllium-sfo2
20 prod-beryllium-sfo3
20 prod-beryllium-sgp1
20 prod-beryllium-syd1
20 prod-beryllium-tor1
20 prod-jerry-se-e16b-ams3
20 prod-jerry-se-e16b-lon1
20 prod-jerry-se-e16b-nyc1
20 prod-jerry-se-e16b-sfo2
20 prod-jerry-se-e16b-tor1
20 prod-jerry-se-hoth-lon1
20 prod-jerry-se-hoth-syd1
20 prod-jerry-se-managers-tor1
22 prod-jerry-se-endor-us-central
22 prod-jerry-se-endor-us-east
22 prod-jerry-se-endor-us-west
23 prod-jerry-se-endor-us-southeast
28 prod-jerry-se-scanners-ap-south
29 prod-jerry-se-scanners-ap-west
2 do-prod-eu-central-proxys-1102 2 do-prod-eu-central-proxys-1102
2 do-prod-eu-west-proxys-1102 2 do-prod-eu-west-proxys-1102
2 do-prod-us-east-proxys-1102 2 do-prod-us-east-proxys-1102
2 do-prod-us-north-proxys-1102 2 do-prod-us-north-proxys-1102
2 do-prod-us-west-proxys-1102 2 do-prod-us-west-proxys-1102
2 prod-mendelevium-fra1 2 prod-jerry-se-naboo-ca-central
2 prod-mendelevium-lon1 2 prod-jerry-se-naboo-lon1
2 prod-mendelevium-nyc1 2 prod-jerry-se-naboo-syd1
2 prod-mendelevium-nyc3 2 prod-jerry-se-utils-lon1
2 prod-mendelevium-sfo2 2 prod-mendelevium-us-central
2 prod-mendelevium-sgp1 2 prod-mendelevium-us-east
2 prod-mendelevium-tor1 2 prod-mendelevium-us-southeast
2 prod-mercury-ams3 2 prod-mendelevium-us-west
2 prod-mercury-fra1 30 prod-jerry-se-endor-tor1
2 prod-mercury-lon1 36 prod-jerry-se-hoth-ca-central
2 prod-mercury-nyc1 3 prod-jerry-se-jakku-us-central
2 prod-mercury-nyc3 3 prod-jerry-se-jakku-us-east
2 prod-mercury-sgp1 3 prod-jerry-se-jakku-us-southeast
2 prod-mercury-tor1 3 prod-jerry-se-jakku-us-west
3 prod-barium-nyc1 3 prod-jerry-se-kamino-us-central
3 prod-barium-sgp1 3 prod-jerry-se-kamino-us-east
3 prod-jerry-se-jakku-nyc1 3 prod-jerry-se-kamino-us-southeast
3 prod-jerry-se-jakku-sfo2 3 prod-jerry-se-kamino-us-west
3 prod-jerry-se-kamino-nyc1 3 prod-jerry-se-kashyyyk-us-central
3 prod-jerry-se-kamino-sfo2 3 prod-jerry-se-kashyyyk-us-east
3 prod-jerry-se-kashyyyk-nyc1 3 prod-jerry-se-kashyyyk-us-southeast
3 prod-jerry-se-kashyyyk-sfo2 3 prod-jerry-se-kashyyyk-us-west
3 prod-jerry-se-naboo-tor1 3 prod-mendelevium-ap-northeast
3 prod-mendelevium-ams3 3 prod-mendelevium-ap-south
3 prod-mercury-sfo2 3 prod-mendelevium-ap-southeast
3 prod-mercury-sfo3 3 prod-mendelevium-ap-west
3 prod-mendelevium-ca-central
3 prod-mendelevium-eu-central
3 prod-mendelevium-eu-west
40 prod-jerry-se-managers-lon1
46 prod-jerry-se-endor-nyc1
46 prod-jerry-se-endor-sfo2
4 dev-boron-ams3 4 dev-boron-ams3
4 dev-boron-lon1 4 dev-boron-lon1
4 dev-boron-nyc1 4 dev-boron-nyc1
4 dev-boron-nyc3 4 dev-boron-nyc3
4 dev-boron-sgp1 4 dev-boron-sgp1
4 prod-barium-ams3 4 prod-mendelevium-fra1
4 prod-barium-lon1 4 prod-mendelevium-lon1
4 prod-barium-nyc3 4 prod-mendelevium-nyc1
4 prod-jerry-se-clients-nyc1 4 prod-mendelevium-nyc3
4 prod-silicon-ams3 4 prod-mendelevium-sfo2
4 prod-silicon-lon1 4 prod-mendelevium-sfo3
4 prod-silicon-nyc1 4 prod-mendelevium-sgp1
4 prod-silicon-nyc3 4 prod-mendelevium-syd1
4 prod-silicon-sgp1 4 prod-mendelevium-tor1
4 prod-mercury-ams3
4 prod-mercury-fra1
4 prod-mercury-lon1
4 prod-mercury-nyc1
4 prod-mercury-nyc3
4 prod-mercury-sgp1
4 prod-mercury-tor1
50 prod-meitnerium-ams3
50 prod-meitnerium-fra1
50 prod-meitnerium-lon1
50 prod-meitnerium-nyc1
50 prod-meitnerium-nyc3
50 prod-meitnerium-sfo2
50 prod-meitnerium-sgp1
50 prod-meitnerium-syd1
50 prod-meitnerium-tor1
51 prod-meitnerium-sfo3
53 prod-jerry-se-scanners-ca-central
55 prod-jerry-se-scanners-us-east
55 prod-jerry-se-scanners-us-west
59 prod-jerry-se-scanners-eu-central
59 prod-jerry-se-scanners-eu-west
5 dev-boron-fra1 5 dev-boron-fra1
5 dev-boron-sfo2 5 dev-boron-sfo2
5 dev-boron-sfo3 5 dev-boron-sfo3
@ -68,100 +203,50 @@
5 dev-meitnerium-sfo3 5 dev-meitnerium-sfo3
5 dev-meitnerium-sgp1 5 dev-meitnerium-sgp1
5 dev-meitnerium-tor1 5 dev-meitnerium-tor1
5 prod-barium-fra1 5 prod-jerry-se-endor-ap-southeast
5 prod-barium-sfo2 5 prod-jerry-se-endor-eu-west
5 prod-barium-sfo3 5 prod-jerry-se-jakku-ap-southeast
5 prod-barium-tor1 5 prod-jerry-se-jakku-eu-west
5 prod-jerry-se-clients-ams3 5 prod-jerry-se-kamino-ap-southeast
5 prod-jerry-se-clients-lon1 5 prod-jerry-se-kamino-eu-west
5 prod-jerry-se-clients-sfo2 5 prod-jerry-se-kashyyyk-ap-southeast
5 prod-jerry-se-clients-tor1 5 prod-jerry-se-kashyyyk-eu-west
5 prod-jerry-se-endor-lon1 6 prod-jerry-se-clients-ca-central
5 prod-jerry-se-jakku-lon1 6 prod-jerry-se-clients-eu-central
5 prod-jerry-se-kamino-lon1 6 prod-jerry-se-clients-eu-west
5 prod-jerry-se-kashyyyk-lon1 6 prod-jerry-se-clients-us-east
5 prod-magnesium-lon1 6 prod-jerry-se-clients-us-west
5 prod-magnesium-nyc1 6 prod-jerry-se-jakku-ca-central
5 prod-magnesium-nyc3 6 prod-jerry-se-jakku-nyc1
5 prod-magnesium-sgp1 6 prod-jerry-se-jakku-sfo2
5 prod-manganese-ams3 6 prod-jerry-se-kamino-ca-central
5 prod-manganese-fra1 6 prod-jerry-se-kamino-nyc1
5 prod-manganese-lon1 6 prod-jerry-se-kamino-sfo2
5 prod-manganese-nyc1 6 prod-jerry-se-kashyyyk-ca-central
5 prod-manganese-nyc3 6 prod-jerry-se-kashyyyk-nyc1
5 prod-manganese-sfo2 6 prod-jerry-se-kashyyyk-sfo2
5 prod-manganese-sfo3 6 prod-jerry-se-naboo-tor1
5 prod-manganese-sgp1 6 prod-mendelevium-ams3
5 prod-manganese-tor1 6 prod-mercury-sfo2
5 prod-silicon-fra1 6 prod-mercury-sfo3
5 prod-silicon-sfo2 72 prod-jerry-se-hoth-tor1
5 prod-silicon-sfo3 7 prod-jerry-se-naboo-us-central
5 prod-silicon-tor1 7 prod-jerry-se-naboo-us-east
6 prod-jerry-se-jakku-tor1 7 prod-jerry-se-naboo-us-west
6 prod-jerry-se-kamino-tor1 80 prod-jerry-se-scanners-blr1
6 prod-jerry-se-kashyyyk-tor1 80 prod-jerry-se-scanners-sgp1
6 prod-magnesium-ams3 89 prod-jerry-se-hoth-us-east
6 prod-magnesium-fra1 89 prod-jerry-se-hoth-us-west
6 prod-magnesium-sfo2 8 prod-barium-ams3
6 prod-magnesium-sfo3 8 prod-barium-lon1
6 prod-magnesium-tor1 8 prod-barium-nyc1
7 prod-beryllium-nyc3 8 prod-barium-nyc3
7 prod-beryllium-sfo3 8 prod-barium-sgp1
8 prod-jerry-se-naboo-nyc1 8 prod-jerry-se-naboo-us-southeast
8 prod-jerry-se-naboo-sfo2 8 prod-silicon-ams3
8 prod-sulfur-nyc1 8 prod-silicon-lon1
8 prod-sulfur-nyc3 8 prod-silicon-nyc1
8 prod-sulfur-sfo2 8 prod-silicon-nyc3
8 prod-sulfur-sfo3 8 prod-silicon-sgp1
8 prod-sulfur-tor1 90 prod-jerry-se-hoth-us-central
9 prod-beryllium-fra1 90 prod-jerry-se-hoth-us-southeast
9 prod-beryllium-sgp1
10 prod-beryllium-ams3
10 prod-beryllium-lon1
10 prod-beryllium-nyc1
10 prod-beryllium-sfo2
10 prod-beryllium-tor1
10 prod-jerry-se-e16b-ams3
10 prod-jerry-se-e16b-lon1
10 prod-jerry-se-e16b-nyc1
10 prod-jerry-se-e16b-sfo2
10 prod-jerry-se-e16b-tor1
10 prod-jerry-se-hoth-lon1
10 prod-jerry-se-managers-tor1
15 prod-jerry-se-endor-tor1
18 jerry-se-do-na-east-unbounds
20 jerry-se-do-eu-central-unbounds
20 jerry-se-do-eu-west-unbounds
20 jerry-se-do-na-central-unbounds
20 jerry-se-do-na-west-unbounds
20 prod-jerry-se-managers-lon1
21 prod-meitnerium-nyc3
23 prod-jerry-se-endor-nyc1
23 prod-jerry-se-endor-sfo2
23 prod-meitnerium-fra1
23 prod-meitnerium-nyc1
23 prod-meitnerium-sgp1
25 prod-meitnerium-ams3
25 prod-meitnerium-lon1
25 prod-meitnerium-sfo2
25 prod-meitnerium-sfo3
25 prod-meitnerium-tor1
36 prod-jerry-se-hoth-tor1
37 prod-boron-fra1
39 prod-jerry-se-scanners-blr1
39 prod-jerry-se-scanners-sgp1
43 prod-boron-nyc3
44 prod-boron-sgp1
48 prod-boron-nyc1
50 prod-boron-sfo3
52 prod-boron-lon1
53 prod-boron-ams3
55 prod-boron-sfo2
55 prod-boron-tor1
60 prod-jerry-se-scanners-ams3
60 prod-jerry-se-scanners-lon1
60 prod-jerry-se-scanners-nyc1
60 prod-jerry-se-scanners-sfo2
60 prod-jerry-se-scanners-tor1
90 prod-jerry-se-hoth-nyc1
90 prod-jerry-se-hoth-sfo2

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -2,6 +2,12 @@
set -eux set -eux
############################################################################################
# Information #
# The program dns-ptr-resolver can be installed from cargo: cargo install dns-ptr-resolver #
# See: https://github.com/wdes/dns-ptr-resolver #
############################################################################################
REV="v-$(date --iso-8601=seconds)" REV="v-$(date --iso-8601=seconds)"
cd ./digitalocean/ cd ./digitalocean/
@ -14,20 +20,28 @@ if [ ! -d ./reverse_revisions/ ]; then
mkdir ./reverse_revisions mkdir ./reverse_revisions
fi fi
dns-ptr-resolver $PWD/binaryedge_digitalocean_possible_ips.txt 1> binaryedge_revisions/$REV.txt curl -A "https://github.com/datacenters-network/security" https://api.binaryedge.io/v1/minions | jq -r '.scanners[]' | sed '/^$/d' | sort -V > ./binaryedge_api_ips.txt
curl -A "https://github.com/datacenters-network/security" https://api.binaryedge.io/v1/minions-ipv6 | jq -r '.scanners[]' | sed '/^$/d' | sort -V >> ./binaryedge_api_ips.txt
grep -F "binaryedge" binaryedge_revisions/$REV.txt | sort -V > binaryedge_revisions/$REV.sorted.txt doRev () {
grep -v -F "binaryedge" binaryedge_revisions/$REV.txt | sort -V > reverse_revisions/$REV.sorted.txt dns-ptr-resolver $PWD/$1 1> binaryedge_revisions/$REV.txt
mv binaryedge_revisions/$REV.sorted.txt binaryedge_revisions/$REV.txt
mv reverse_revisions/$REV.sorted.txt reverse_revisions/$REV.txt
# Reverse the file grep -F "binaryedge" binaryedge_revisions/$REV.txt | sort -V > binaryedge_revisions/$REV.sorted.txt
awk -F'#' '{print $2" # "$1}' OFS=, "binaryedge_revisions/$REV.txt" | awk '{$1=$1;print}' | sort > binaryedge_revisions/$REV-reversed.txt grep -v -F "binaryedge" binaryedge_revisions/$REV.txt | sort -V > reverse_revisions/$REV.sorted.txt
mv binaryedge_revisions/$REV.sorted.txt binaryedge_revisions/$REV.txt
mv reverse_revisions/$REV.sorted.txt reverse_revisions/$REV.txt
# Sort by name and reverse the list to build the list of all possible IPs # Reverse the file
cat binaryedge_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq | awk -F'#' '{print "# "$1" \n "$2}' OFS='#' | awk '{$1=$1;print}' > ../binaryedge.txt awk -F'#' '{print $2" # "$1}' OFS=, "binaryedge_revisions/$REV.txt" | awk '{$1=$1;print}' | sort > binaryedge_revisions/$REV-reversed.txt
grep -F '#' ../binaryedge.txt | cut -d ' ' -f 2 | sort | cut -d. -f-1 | rev | cut -d '-' -f2- | rev | sort | uniq -c | sort > ./binaryedge-chunk-counts.txt # Sort by name and reverse the list to build the list of all possible IPs
cat binaryedge_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq | awk -F'#' '{print "# "$1" \n "$2}' OFS='#' | awk '{$1=$1;print}' > ../binaryedge.txt
grep -F '#' ../binaryedge.txt | cut -d ' ' -f 2 | sort | cut -d. -f-1 | rev | cut -d '-' -f2- | rev | sort | uniq -c | sort > ./binaryedge-chunk-counts.txt
}
doRev "binaryedge_api_ips.txt"
doRev "binaryedge_digitalocean_possible_ips.txt"
# Search for false positives # Search for false positives
# cat ../binaryedge.txt | cut -d '#' -f 1 | xargs -P 50 -I {} bash -c 'set -eu;rev="$(dig @9.9.9.9 +short +time=1 +tries=1 -x {})"; if [[ "$rev" == *";;"* ]]; then sleep 1; rev="$(dig @8.8.8.8 +short +time=1 +tries=1 -x {})"; fi; echo "{} # $rev";' | grep -v -F "binaryedge.com" # dns-ptr-resolver ../binaryedge.txt | grep -v -F "binaryedge.com"

View File

@ -2,6 +2,12 @@
set -eux set -eux
############################################################################################
# Information #
# The program dns-ptr-resolver can be installed from cargo: cargo install dns-ptr-resolver #
# See: https://github.com/wdes/dns-ptr-resolver #
############################################################################################
REV="v-$(date --iso-8601=seconds)" REV="v-$(date --iso-8601=seconds)"
cd ./digitalocean/ cd ./digitalocean/
@ -30,4 +36,4 @@ cat stretchoid_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq
grep -F '#' ../stretchoid.txt | cut -d- -f2 | grep -P '^[0-9]{3,}+' | sort | uniq -c | sort > ./stretchoid-chunk-counts.txt grep -F '#' ../stretchoid.txt | cut -d- -f2 | grep -P '^[0-9]{3,}+' | sort | uniq -c | sort > ./stretchoid-chunk-counts.txt
# Search for false positives # Search for false positives
# cat ../stretchoid.txt | cut -d '#' -f 1 | xargs -P 50 -I {} bash -c 'set -eu;rev="$(dig @9.9.9.9 +short +time=1 +tries=1 -x {})"; if [[ "$rev" == *";;"* ]]; then sleep 1; rev="$(dig @8.8.8.8 +short +time=1 +tries=1 -x {})"; fi; echo "{} # $rev";' | grep -v -F "stretchoid.com" # dns-ptr-resolver ../stretchoid.txt | grep -v -F "stretchoid.com"