Document how to release

README.md

@@ -6,28 +6,57 @@
 
 - `https://security.wdes.eu/scanners/stretchoid.txt` (List of all known stretchoid IPs)
 - `https://security.wdes.eu/scanners/binaryedge.txt` (List of all known binaryedge IPs)
-- `https://security.wdes.eu/scanners/shadowserver.txt` (List of all known shadowserver IPs)
+- `https://security.wdes.eu/scanners/shadowserver.txt` (List of all known shadowserver (shadowserver.org) IPs)
-- `https://security.wdes.eu/scanners/censys.txt` (List of all IPs declared by censys scanner on their [FAQ](https://docs.censys.com/docs/opt-out-of-data-collection)
+- `https://security.wdes.eu/scanners/censys.txt` (List of all IPs declared by censys scanner (censys.io) on their [FAQ](https://docs.censys.com/docs/opt-out-of-data-collection)
 - `https://security.wdes.eu/scanners/internet-measurement.com.txt` (List of all IPs declared by internet-measurement.com on [their website](https://internet-measurement.com/#ips))
+- `https://security.wdes.eu/scanners/anssi.txt` (List of all IPs declared by CERT-FR/ANSSI on [their website](https://www.cert.ssi.gouv.fr/scans/))
 
 ### Collections (by vendor)
 
 - `https://security.wdes.eu/collections/wdes/bad-networks.txt` (List of some hand picked bad networks)
 - `https://security.wdes.eu/collections/wdes/bad-ips.txt` (List of some hand picked bad IPs that caused harm/attacks/scans to mail servers)
 
-- `https://security.wdes.eu/collections/microsoft/email-servers.txt` (List of the Microsoft IPs for it's mail servers)
+- `https://security.wdes.eu/collections/microsoft/email-servers.txt` (List of the [Microsoft IPs for it's mail servers](https://learn.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide#exchange-online))
 - `https://security.wdes.eu/collections/amazon/cloudfront-ips.txt` (List of AWS CloudFront IPs)
 
 ## Other similar projects
 
 - https://github.com/szepeviktor/debian-server-tools/tree/master/security/myattackers-ipsets/ipset
-- https://github.com/wravoc/authlog-threats/blob/main/scanners
+- https://github.com/wravoc/authlog-threats/blob/main/scanners (not updated in 2025 since two years)
 - https://github.com/stamparm/maltrail/blob/master/trails/static/mass_scanner.txt
 
-## Bad actors to handle
+## TODO list
 
-- scan-*.shadowserver.org example: scan-37-1d.shadowserver.org
+### Features
-- *.scan.bufferover.run example: bogota.scan.bufferover.run
+
-- security.criminalip.com
+- https://security.wdes.eu/scanners/<scanner>.txt#commented -> Output with the name
-- zl-ams-nl-gr1-wk102b.internet-census.org
+
-- optout.scanopticon.com
+### Scanners
+
+  - *.scan.bufferover.run example: bogota.scan.bufferover.run
+  - optout.scanopticon.com
+  - pinger*.netsec.colostate.edu
+  - pinger-*.ant.isi.edu
+  - researchscanner*.eecs.berkeley.edu
+  - researchscan*.eecs.umich.edu
+  - researchscan*.comsys.rwth-aachen.de
+  - scanners.labs.rapid7.com, scanner*.labs.rapid7.com
+  - openresolverproject.org, opensnmpproject.org, openntpproject.org, openssdpproject.org, openresolverproject.org
+  - probe*.projectblindferret.com
+  - shodan.io, *.census.shodan.io, census*.shodan.io
+  - kudelskisecurity.com
+  - riskiq.com
+  - scan.sba-research.org, scanning.sba-research.org
+  - 5thcolumn.net
+  - internet-census.org, ca-san-*-*-*.internet-census.org, Example: zl-ams-nl-gr1-wk102b.internet-census.org
+  - internettl.org
+  - netsystemsresearch.com
+  - tequilaboomboom.club
+  - scanner.openportstats.com
+  - outspoken.ca
+  - phenome.ca
+  - ltx71
+  - Greenbone
+  - leakix.net
+  - [45.83.64.0/22 is ALPHASTRIKE-RESEARCH](https://www.alphastrike.io/en/log4j/)
+  - security.criminalip.com

data/scanners/anssi.txt

@@ -0,0 +1,8 @@
+92.154.95.236
+185.50.66.1
+54.38.103.0
+54.38.103.1
+137.74.246.152
+147.135.160.230
+80.13.153.140
+217.108.23.109

data/scanners/binaryedge_ranges.txt

@@ -1,472 +0,0 @@
-104.131.0.0/18
-104.131.64.0/18
-104.236.0.0/18
-104.236.192.0/18
-104.248.112.0/20
-104.248.128.0/20
-104.248.144.0/20
-104.248.16.0/20
-104.248.160.0/20
-104.248.176.0/20
-104.248.192.0/20
-104.248.208.0/20
-104.248.224.0/20
-104.248.48.0/20
-104.248.64.0/20
-104.248.80.0/20
-128.199.0.0/20
-128.199.128.0/18
-128.199.16.0/20
-128.199.192.0/18
-128.199.32.0/19
-128.199.64.0/18
-134.122.0.0/20
-134.122.112.0/20
-134.122.16.0/20
-134.122.32.0/20
-134.122.48.0/20
-134.122.64.0/20
-134.122.80.0/20
-134.122.96.0/20
-134.209.0.0/20
-134.209.112.0/20
-134.209.144.0/20
-134.209.16.0/20
-134.209.160.0/20
-134.209.176.0/20
-134.209.192.0/20
-134.209.208.0/20
-134.209.224.0/20
-134.209.48.0/20
-134.209.64.0/20
-134.209.80.0/20
-134.209.96.0/20
-137.184.0.0/20
-137.184.112.0/20
-137.184.128.0/20
-137.184.144.0/20
-137.184.16.0/20
-137.184.160.0/20
-137.184.176.0/20
-137.184.192.0/20
-137.184.208.0/20
-137.184.224.0/20
-137.184.240.0/22
-137.184.244.0/22
-137.184.248.0/22
-137.184.252.0/24
-137.184.254.0/24
-137.184.32.0/20
-137.184.48.0/20
-137.184.64.0/20
-137.184.80.0/20
-138.197.0.0/20
-138.197.112.0/20
-138.197.128.0/20
-138.197.144.0/20
-138.197.16.0/20
-138.197.160.0/20
-138.197.192.0/20
-138.197.208.0/20
-138.197.80.0/20
-138.197.96.0/20
-138.68.0.0/20
-138.68.128.0/20
-138.68.144.0/20
-138.68.16.0/20
-138.68.160.0/20
-138.68.176.0/20
-138.68.224.0/20
-138.68.240.0/20
-138.68.40.0/21
-138.68.48.0/20
-138.68.80.0/20
-139.59.0.0/20
-139.59.112.0/20
-139.59.128.0/19
-139.59.160.0/20
-139.59.176.0/20
-139.59.240.0/20
-139.59.32.0/20
-139.59.52.0/22
-139.59.56.0/21
-139.59.64.0/20
-139.59.80.0/20
-139.59.96.0/20
-142.93.0.0/20
-142.93.112.0/20
-142.93.144.0/20
-142.93.16.0/20
-142.93.160.0/20
-142.93.176.0/20
-142.93.208.0/20
-142.93.224.0/20
-142.93.240.0/20
-142.93.32.0/20
-142.93.64.0/20
-142.93.80.0/20
-143.110.128.0/20
-143.110.160.0/20
-143.110.176.0/20
-143.110.192.0/20
-143.110.208.0/20
-143.110.224.0/20
-143.110.240.0/20
-143.198.0.0/20
-143.198.112.0/20
-143.198.128.0/20
-143.198.144.0/20
-143.198.16.0/20
-143.198.160.0/20
-143.198.176.0/20
-143.198.192.0/20
-143.198.208.0/20
-143.198.224.0/20
-143.198.240.0/22
-143.198.244.0/22
-143.198.248.0/22
-143.198.32.0/20
-143.198.48.0/20
-143.198.64.0/20
-143.198.80.0/20
-143.198.96.0/20
-143.244.144.0/20
-143.244.160.0/20
-143.244.176.0/20
-144.126.208.0/20
-144.126.224.0/20
-146.190.112.0/20
-146.190.16.0/20
-146.190.160.0/20
-146.190.200.0/22
-146.190.204.0/22
-146.190.208.0/20
-146.190.224.0/20
-146.190.240.0/20
-146.190.32.0/19
-146.190.64.0/20
-146.190.80.0/20
-146.190.96.0/20
-147.182.128.0/20
-147.182.144.0/20
-147.182.160.0/20
-147.182.176.0/20
-147.182.192.0/20
-147.182.208.0/20
-147.182.240.0/20
-157.230.0.0/20
-157.230.112.0/20
-157.230.128.0/20
-157.230.144.0/20
-157.230.16.0/20
-157.230.160.0/20
-157.230.176.0/20
-157.230.192.0/22
-157.230.196.0/22
-157.230.208.0/20
-157.230.224.0/20
-157.230.240.0/20
-157.230.32.0/20
-157.230.48.0/20
-157.230.64.0/22
-157.230.68.0/22
-157.230.80.0/20
-157.230.96.0/20
-157.245.0.0/20
-157.245.112.0/20
-157.245.128.0/20
-157.245.144.0/20
-157.245.160.0/20
-157.245.176.0/20
-157.245.192.0/20
-157.245.208.0/20
-157.245.224.0/20
-157.245.240.0/20
-157.245.32.0/20
-157.245.48.0/20
-157.245.64.0/20
-157.245.80.0/20
-159.203.0.0/20
-159.203.112.0/20
-159.203.128.0/20
-159.203.144.0/22
-159.203.148.0/22
-159.203.152.0/22
-159.203.156.0/22
-159.203.16.0/20
-159.203.160.0/20
-159.203.176.0/20
-159.203.192.0/20
-159.203.208.0/20
-159.203.224.0/20
-159.203.240.0/20
-159.203.32.0/20
-159.203.48.0/22
-159.203.56.0/21
-159.203.64.0/20
-159.203.80.0/20
-159.203.96.0/20
-159.223.0.0/20
-159.223.112.0/20
-159.223.128.0/20
-159.223.144.0/20
-159.223.160.0/19
-159.223.192.0/20
-159.223.208.0/20
-159.223.224.0/20
-159.223.240.0/22
-159.223.244.0/22
-159.223.248.0/22
-159.223.32.0/20
-159.223.48.0/20
-159.223.64.0/20
-159.223.96.0/20
-159.65.0.0/20
-159.65.128.0/20
-159.65.144.0/20
-159.65.16.0/20
-159.65.192.0/20
-159.65.212.0/22
-159.65.216.0/21
-159.65.224.0/20
-159.65.48.0/20
-159.65.64.0/20
-159.65.80.0/20
-159.65.96.0/20
-159.89.112.0/20
-159.89.128.0/20
-159.89.144.0/20
-159.89.16.0/20
-159.89.160.0/20
-159.89.224.0/20
-159.89.252.0/22
-159.89.32.0/20
-159.89.48.0/21
-159.89.60.0/24
-159.89.61.0/24
-159.89.62.0/24
-159.89.63.0/24
-159.89.64.0/20
-159.89.96.0/20
-161.35.0.0/20
-161.35.112.0/20
-161.35.128.0/20
-161.35.144.0/20
-161.35.160.0/20
-161.35.176.0/20
-161.35.192.0/20
-161.35.224.0/20
-161.35.32.0/20
-161.35.48.0/20
-161.35.64.0/20
-161.35.80.0/20
-161.35.96.0/20
-162.243.160.0/20
-164.90.128.0/20
-164.90.192.0/20
-164.90.208.0/20
-164.90.224.0/20
-164.92.128.0/20
-164.92.144.0/20
-164.92.160.0/20
-164.92.176.0/20
-164.92.208.0/20
-164.92.224.0/20
-164.92.64.0/19
-164.92.96.0/19
-165.22.112.0/20
-165.22.128.0/20
-165.22.144.0/20
-165.22.16.0/20
-165.22.160.0/20
-165.22.176.0/20
-165.22.192.0/20
-165.22.208.0/20
-165.22.224.0/20
-165.22.240.0/20
-165.22.32.0/20
-165.22.48.0/20
-165.22.64.0/20
-165.22.80.0/20
-165.22.96.0/20
-165.227.0.0/20
-165.227.112.0/20
-165.227.128.0/20
-165.227.16.0/20
-165.227.160.0/20
-165.227.176.0/20
-165.227.192.0/20
-165.227.224.0/20
-165.227.252.0/22
-165.227.32.0/20
-165.227.48.0/20
-165.227.64.0/20
-165.227.80.0/20
-165.227.96.0/20
-165.232.112.0/20
-165.232.128.0/20
-165.232.144.0/20
-165.232.160.0/20
-165.232.32.0/20
-165.232.48.0/20
-165.232.80.0/20
-165.232.96.0/20
-167.172.112.0/20
-167.172.144.0/20
-167.172.16.0/20
-167.172.192.0/20
-167.172.208.0/20
-167.172.224.0/20
-167.172.240.0/20
-167.172.32.0/20
-167.172.48.0/20
-167.71.0.0/20
-167.71.112.0/20
-167.71.128.0/20
-167.71.144.0/20
-167.71.16.0/20
-167.71.176.0/20
-167.71.192.0/20
-167.71.208.0/20
-167.71.224.0/20
-167.71.240.0/20
-167.71.32.0/20
-167.71.48.0/20
-167.71.64.0/20
-167.71.80.0/20
-167.71.96.0/20
-167.99.0.0/20
-167.99.112.0/20
-167.99.128.0/20
-167.99.144.0/20
-167.99.160.0/20
-167.99.176.0/20
-167.99.192.0/20
-167.99.208.0/20
-167.99.224.0/20
-167.99.240.0/20
-167.99.32.0/20
-167.99.48.0/20
-167.99.64.0/20
-167.99.80.0/20
-167.99.96.0/20
-174.138.0.0/20
-174.138.100.0/22
-174.138.104.0/22
-174.138.108.0/22
-174.138.120.0/22
-174.138.124.0/22
-174.138.32.0/20
-174.138.64.0/20
-174.138.80.0/20
-178.128.0.0/20
-178.128.112.0/20
-178.128.144.0/20
-178.128.160.0/20
-178.128.176.0/20
-178.128.192.0/20
-178.128.208.0/20
-178.128.224.0/20
-178.128.240.0/20
-178.128.32.0/20
-178.128.48.0/20
-178.128.64.0/20
-178.128.80.0/20
-178.128.96.0/20
-178.62.0.0/18
-178.62.128.0/18
-178.62.192.0/18
-178.62.64.0/18
-188.166.0.0/18
-188.166.128.0/22
-188.166.132.0/22
-188.166.136.0/22
-188.166.140.0/22
-188.166.144.0/20
-188.166.160.0/21
-188.166.168.0/21
-188.166.176.0/20
-188.166.192.0/22
-188.166.196.0/22
-188.166.208.0/20
-188.166.224.0/20
-188.166.240.0/20
-188.166.64.0/18
-192.241.128.0/19
-192.34.56.0/21
-192.34.57.0/24
-192.34.58.0/24
-192.81.216.0/22
-198.199.64.0/20
-198.211.112.0/22
-204.48.16.0/20
-206.189.0.0/20
-206.189.112.0/20
-206.189.128.0/20
-206.189.144.0/20
-206.189.16.0/20
-206.189.160.0/20
-206.189.176.0/20
-206.189.192.0/20
-206.189.208.0/20
-206.189.32.0/20
-206.189.64.0/20
-206.189.96.0/20
-206.81.16.0/20
-207.154.192.0/20
-208.68.39.0/24
-209.38.192.0/19
-209.38.224.0/19
-209.97.128.0/20
-209.97.176.0/20
-24.144.96.0/19
-24.199.112.0/20
-24.199.80.0/20
-24.199.96.0/20
-45.55.32.0/19
-45.55.64.0/19
-46.101.0.0/18
-46.101.124.0/22
-46.101.128.0/17
-46.101.72.0/21
-46.101.80.0/20
-46.101.96.0/19
-64.225.0.0/20
-64.225.112.0/20
-64.225.16.0/20
-64.225.32.0/20
-64.225.48.0/20
-64.225.64.0/20
-64.225.96.0/20
-64.226.112.0/20
-64.226.64.0/20
-64.226.80.0/20
-64.226.96.0/20
-64.227.0.0/20
-64.227.128.0/19
-64.227.16.0/20
-64.227.160.0/20
-64.227.176.0/20
-64.227.32.0/20
-64.227.48.0/20
-64.227.64.0/20
-64.227.80.0/20
-64.227.96.0/20
-67.205.128.0/20
-67.205.144.0/20
-67.205.160.0/20
-67.205.176.0/20
-67.207.80.0/20
-68.183.0.0/20
-68.183.112.0/20
-68.183.128.0/20
-68.183.144.0/20
-68.183.16.0/20
-68.183.160.0/20
-68.183.192.0/20
-68.183.252.0/22
-68.183.32.0/20
-68.183.64.0/20
-68.183.80.0/20
-68.183.96.0/20
-69.55.49.0/24

data/scanners/digitalocean_ranges.txt

@@ -1,234 +0,0 @@
-37.139.31.0/24
-104.131.64.0/18
-104.131.144.0/20
-45.55.0.0/24
-45.55.1.0/24
-45.55.2.0/24
-45.55.3.0/24
-45.55.4.0/24
-45.55.6.0/24
-45.55.7.0/24
-45.55.8.0/24
-45.55.9.0/24
-45.55.10.0/24
-45.55.11.0/24
-45.55.12.0/24
-45.55.13.0/24
-45.55.14.0/24
-45.55.16.0/24
-45.55.17.0/24
-45.55.18.0/24
-45.55.19.0/24
-45.55.21.0/24
-45.55.22.0/24
-45.55.26.0/24
-45.55.28.0/24
-45.55.29.0/24
-45.55.30.0/24
-103.78.132.0/24
-104.131.128.0/24
-104.131.133.0/24
-104.131.134.0/24
-104.131.135.0/24
-104.131.137.0/24
-104.131.138.0/24
-104.131.141.0/24
-104.131.144.0/24
-104.131.145.0/24
-104.131.146.0/24
-104.131.156.0/24
-104.236.128.0/24
-104.236.149.0/24
-104.236.150.0/24
-104.236.152.0/24
-104.236.154.0/24
-104.236.155.0/24
-104.236.157.0/24
-104.236.159.0/24
-104.236.162.0/24
-104.236.163.0/24
-104.236.164.0/24
-104.236.165.0/24
-104.236.168.0/24
-104.236.169.0/24
-104.236.170.0/24
-104.236.172.0/24
-104.236.175.0/24
-104.236.176.0/24
-104.236.177.0/24
-104.236.180.0/24
-104.236.182.0/24
-104.236.183.0/24
-104.236.184.0/24
-104.236.185.0/24
-104.236.186.0/24
-104.236.187.0/24
-104.236.191.0/24
-107.170.192.0/24
-107.170.193.0/24
-107.170.194.0/24
-107.170.195.0/24
-107.170.196.0/24
-107.170.197.0/24
-107.170.198.0/24
-107.170.199.0/24
-107.170.200.0/24
-107.170.201.0/24
-107.170.202.0/24
-107.170.203.0/24
-107.170.204.0/24
-107.170.207.0/24
-107.170.208.0/24
-107.170.209.0/24
-107.170.210.0/24
-107.170.211.0/24
-107.170.212.0/24
-107.170.213.0/24
-107.170.215.0/24
-107.170.216.0/24
-107.170.218.0/24
-107.170.219.0/24
-107.170.224.0/24
-107.170.225.0/24
-107.170.226.0/24
-107.170.227.0/24
-107.170.228.0/24
-107.170.229.0/24
-107.170.230.0/24
-107.170.231.0/24
-107.170.232.0/24
-107.170.233.0/24
-107.170.234.0/24
-107.170.235.0/24
-107.170.236.0/24
-107.170.237.0/24
-107.170.238.0/24
-107.170.239.0/24
-107.170.240.0/24
-107.170.241.0/24
-107.170.242.0/24
-107.170.243.0/24
-107.170.244.0/24
-107.170.245.0/24
-107.170.246.0/24
-107.170.247.0/24
-107.170.248.0/24
-107.170.249.0/24
-107.170.250.0/24
-107.170.251.0/24
-107.170.252.0/24
-107.170.253.0/24
-107.170.254.0/24
-107.170.255.0/24
-138.68.208.0/24
-159.203.192.0/24
-159.203.197.0/24
-159.203.201.0/24
-159.203.208.0/24
-159.203.224.0/24
-159.203.240.0/24
-159.203.252.0/24
-159.203.255.0/24
-162.243.128.0/24
-162.243.129.0/24
-162.243.130.0/24
-162.243.131.0/24
-162.243.132.0/24
-162.243.133.0/24
-162.243.134.0/24
-162.243.135.0/24
-162.243.136.0/24
-162.243.137.0/24
-162.243.138.0/24
-162.243.139.0/24
-162.243.140.0/24
-162.243.141.0/24
-162.243.142.0/24
-162.243.143.0/24
-162.243.144.0/24
-162.243.145.0/24
-162.243.146.0/24
-162.243.147.0/24
-162.243.148.0/24
-162.243.149.0/24
-162.243.150.0/24
-162.243.151.0/24
-162.243.152.0/24
-162.243.157.0/24
-162.243.158.0/24
-192.241.192.0/24
-192.241.193.0/24
-192.241.194.0/24
-192.241.195.0/24
-192.241.196.0/24
-192.241.197.0/24
-192.241.198.0/24
-192.241.199.0/24
-192.241.200.0/24
-192.241.201.0/24
-192.241.202.0/24
-192.241.203.0/24
-192.241.204.0/24
-192.241.205.0/24
-192.241.206.0/24
-192.241.207.0/24
-192.241.208.0/24
-192.241.209.0/24
-192.241.210.0/24
-192.241.211.0/24
-192.241.212.0/24
-192.241.213.0/24
-192.241.214.0/24
-192.241.215.0/24
-192.241.216.0/24
-192.241.217.0/24
-192.241.218.0/24
-192.241.219.0/24
-192.241.220.0/24
-192.241.221.0/24
-192.241.222.0/24
-192.241.223.0/24
-192.241.224.0/24
-192.241.225.0/24
-192.241.226.0/24
-192.241.227.0/24
-192.241.228.0/24
-192.241.229.0/24
-192.241.230.0/24
-192.241.231.0/24
-192.241.232.0/24
-192.241.233.0/24
-192.241.234.0/24
-192.241.235.0/24
-192.241.236.0/24
-192.241.237.0/24
-192.241.238.0/24
-192.241.239.0/24
-192.243.128.0/24
-198.199.92.0/24
-198.199.93.0/24
-198.199.94.0/24
-198.199.95.0/24
-198.199.96.0/24
-198.199.97.0/24
-198.199.98.0/24
-198.199.100.0/24
-198.199.101.0/24
-198.199.102.0/24
-198.199.103.0/24
-198.199.104.0/24
-198.199.105.0/24
-198.199.106.0/24
-198.199.107.0/24
-198.199.108.0/24
-198.199.109.0/24
-198.199.110.0/24
-198.199.111.0/24
-198.199.112.0/24
-198.199.113.0/24
-198.199.114.0/24
-198.199.115.0/24
-198.199.116.0/24
-198.199.117.0/24
-198.199.118.0/24
-198.199.119.0/24

scripts/extract-scanner-ips.sh

@@ -1,76 +0,0 @@
-#!/bin/sh
-
-set -eux
-
-grep -v -F ":" digitalocean_announced_ips.txt | xargs -I {} sh -c "echo '{} # $(dig +short -x {})'" > digitalocean_announced_ips_with_reverse.txt
-
-
-# Fetch all reverse DNS addresses
-# ns3.digitalocean.com = 198.41.222.173
-cat digitalocean_announced_ips_full.txt | xargs -n 1 -P 40 dig @198.41.222.173 +short +time=5 +tries=10 -x > digitalocean_announced_ips_full_reverse.txt
-
-# Test command
-# cat digitalocean_announced_reverse_dns.txt | grep -v -F "ip6.arpa" | sed 's/.in-addr.arpa//'  | awk -F. '{print $3"." $2"."$1}' | sort | less
-
-# A sample
-#cat digitalocean_announced_ips_full.txt | xargs -n 1 -P 40 dig @198.41.222.173 +short +time=5 +tries=10 -x > digitalocean_announced_ips_full_reverse.txt
-
-cat digitalocean_announced_ips_full.txt | xargs -P 40  -I {} sh -c 'set -eu;rev="$(dig @198.41.222.173 +short +time=5 +tries=10 -x {})";echo "{} # $rev";' > digitalocean_announced_ips_full_reverse_better.txt
-
-
-grep -F -x -v -f digitalocean_announced_ips_full_reverse_better_only_ips.txt digitalocean_announced_ips_full.txt
-sed -i 's/ # $//' digitalocean_announced_ips_full_reverse_better.txt
-sort digitalocean_announced_ips_full_reverse_better.txt > digitalocean_announced_ips_full_reverse_better2.txt
-mv digitalocean_announced_ips_full_reverse_better2.txt digitalocean_announced_ips_full_reverse_better.txt
-
-diff -u digitalocean_announced_ips_full_reverse_better_only_ips.txt digitalocean_announced_ips_full.txt | delta
-cut -d ' ' -f 1 digitalocean_announced_ips_full_reverse_better.txt > digitalocean_announced_ips_full_reverse_better_only_ips.txt
-
-# Find all results
-grep -F "stretchoid" digitalocean_announced_ips_full_reverse_better.txt | cut -d " " -f 3 | sort
-
-
-# Find all ranges
-grep -F "stretchoid" digitalocean_announced_ips_full_reverse_better.txt | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq
-
-
-# Make a list of search keys
-grep -F "stretchoid" digitalocean_announced_ips_full_reverse_better.txt | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq > found_ranges.txt
-
-# Find all ranges to re-scan
-cat found_ranges.txt | xargs -I {} grep -F "{}" digitalocean_announced_ips.txt | sort
-
-# Compare with debian-scripts
-grep -F "add stretchoid" stretchoid.ipset | cut -d ' ' -f 3 | cut -d '.' -f -3 | sort | uniq > found_ranges.txt
-cat found_ranges.txt | xargs -I {} grep -F "{}" digitalocean_announced_ips.txt | sort > stretchoid_ranges_debian_scripts.txt
-
-
-# Re scan
-dig -4 +noauthority +noadditional +nostats -x 107.170.202.77 @1.0.0.1
-
-cat stretchoid_ranges.txt | xargs -n1 prips | uniq | sort -V > stretchoid_possible_ips.txt
-cat binaryedge_ranges.txt | xargs -n1 prips | uniq | sort -V > binaryedge_digitalocean_possible_ips.txt
-
-# With failure handling
-cat stretchoid_digitalocean_possible_ips.txt | xargs -P 50 -I {} bash -c 'set -eu;rev="$(dig @9.9.9.9 +short +time=1 +tries=1 -x {})"; if [[ "$rev" == *";;"* ]]; then sleep 1; rev="$(dig @8.8.8.8 +short +time=1 +tries=1 -x {})"; fi; echo "{} # $rev";' 1> stretchoid_revisions/v5.txt
-
-grep -F "stretchoid" stretchoid_revisions/v5.txt | sort > stretchoid_revisions/v5.sorted.txt
-mv stretchoid_revisions/v5.sorted.txt stretchoid_revisions/v5.txt
-
-# Reverse the file
-awk -F'#' '{print $2" # "$1}' OFS=, "stretchoid_revisions/v5.txt" | awk '{$1=$1;print}' | sort > stretchoid_revisions/v5-reversed.txt
-
-# Build the count per name per ip
-cat stretchoid_revisions/v*-reversed.txt | sort | uniq -c > stretchoid_revisions/count-reversed.txt
-# Same but sorted not by name but by count
-cat stretchoid_revisions/v*-reversed.txt | sort | uniq -c | sort > stretchoid_revisions/count-reversed.txt
-
-# Generate the list of full IPs of stretchoid
-cat stretchoid_revisions/v*-reversed.txt | sort | uniq | awk -F'#' '{print $2" # "$1}' OFS='#' | awk '{$1=$1;print}' > ../stretchoid.txt
-
-
-
-# Find missing CIDRs from the IPS found in the revisions using the announced prefix list
-cat binaryedge_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq | cut -d ' ' -f 3 | sort -V | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq | xargs -I {} grep -E "^{}\.0" digitalocean_announced_ips_simpler.txt | sort -V -t# | uniq | cut -d ' ' -f 3 | sort -V | uniq | grep -v -F -f ./binaryedge_ranges.txt
-
-cat binaryedge-full-possible-names_with_ips_clean_ips.txt | sort -V | cut -d " " -f 1 | cut -d '.' -f -3 | sort | uniq | xargs -I {} grep -E "^{}\.0" digitalocean_announced_ips_simpler.txt | sort -V -t# | uniq | cut -d ' ' -f 3 | sort -V | uniq | grep -v -F -f ./binaryedge_ranges.txt

scripts/make-stretchoid.sh

@@ -1,39 +0,0 @@
-#!/bin/sh
-
-set -eux
-
-############################################################################################
-#                                 Information                                              #
-# The program dns-ptr-resolver can be installed from cargo: cargo install dns-ptr-resolver #
-# See: https://github.com/wdes/dns-ptr-resolver                                            #
-############################################################################################
-
-REV="v-$(date --iso-8601=seconds)"
-
-cd ./data/
-
-if [ ! -d ./stretchoid_revisions/ ]; then
-    mkdir ./stretchoid_revisions
-fi
-
-if [ ! -d ./reverse_revisions/ ]; then
-    mkdir ./reverse_revisions
-fi
-
-dns-ptr-resolver $PWD/stretchoid_possible_ips.txt 1> stretchoid_revisions/$REV.txt
-
-grep -F "stretchoid" stretchoid_revisions/$REV.txt | sort -V > stretchoid_revisions/$REV.sorted.txt
-grep -v -F "stretchoid" stretchoid_revisions/$REV.txt | sort -V > reverse_revisions/$REV.sorted.txt
-mv stretchoid_revisions/$REV.sorted.txt stretchoid_revisions/$REV.txt
-mv reverse_revisions/$REV.sorted.txt reverse_revisions/$REV.txt
-
-# Reverse the file
-awk -F'#' '{print $2" # "$1}' OFS=, "stretchoid_revisions/$REV.txt" | awk '{$1=$1;print}' | sort > stretchoid_revisions/$REV-reversed.txt
-
-# Sort by name and reverse the list to build the list of all possible IPs
-cat stretchoid_revisions/v*-reversed.txt | LC_ALL=C.UTF-8 sort -t "-" -n | uniq | awk -F'#' '{print "# "$1" \n "$2}' OFS='#' | awk '{$1=$1;print}' > ../stretchoid.txt
-
-grep -F '#' ../stretchoid.txt | cut -d- -f2 | grep -P '^[0-9]{3,}+' | sort | uniq -c | sort > ./stretchoid-chunk-counts.txt
-
-# Search for false positives
-# dns-ptr-resolver ../stretchoid.txt | grep -v -F "stretchoid.com"

snow-scanner/.gitignore

@@ -5,7 +5,7 @@ target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
+#Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

snow-scanner/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snow-scanner"
-version = "1.0.0"
+version = "1.1.0"
 authors = ["William Desportes <williamdes@wdes.fr>"]
 edition = "2021"
 rust-version = "1.81.0"  # MSRV

snow-scanner/README.md

@@ -5,5 +5,39 @@ This project name is inspired by the Netflix series "The Snowpiercer"
 ## Run it
 
 ```sh
+SERVER_ADDRESS="127.0.0.1:8777" \
+DB_URL="mysql://db-user:db-pass@db-server/db-snow-scanner" \
+STATIC_DATA_DIR="$PWD/../data" \
 cargo run --release
 ```
+
+## Run in production
+
+The env file located at `/etc/snow-scanner/.env`:
+
+```env
+# Your public IP
+SERVER_ADDRESS="[2a10:ffff:ff:ff:fff::1]:80"
+DB_URL="mysql://db-user:db-pass@db-server/db-snow-scanner"
+STATIC_DATA_DIR="/usr/share/snow-scanner/data"
+# Adjust this
+ROCKET_LOG_LEVEL="debug"
+ROCKET_PROFILE="debug"
+# Setup TLS
+ROCKET_TLS='{certs="/etc/ssl/certs/cert.pem",key="/etc/ssl/private/key.pem", mutual={ca_certs="/etc/ssl/certs/cloudflare.crt",mandatory=true}}'
+```
+
+## Make a new release
+
+Pre-requirements: `cargo install cargo-deb`
+
+- Bump `Cargo.toml`
+- Commit it
+- Run `cd snow-scanner`
+- Run `cargo update`
+- Commit the changes to the lockfile
+- Run tests `cargo test`
+- Run `cargo deb`
+- Run `./debian/upload.sh target/debian/snow-scanner_1.1.0-1_amd64.deb`
+- Run `git tag -a -s -m "snow-scanner/1.1.0" snow-scanner/1.1.0`
+- Check `debdiff ../../snow-scanner_1.0.0-1_amd64.deb target/debian/snow-scanner_1.1.0-1_amd64.deb`

snow-scanner/debian/postinst

@@ -0,0 +1,21 @@
+#!/bin/sh
+# postinst script for snow-scanner
+
+set -eu
+
+if [ "$1" = "configure" ]; then
+
+	if ! getent passwd | grep -q "^snow-scanner:"; then
+		useradd --shell /bin/sh snow-scanner
+	fi
+
+	if [ ! -d /etc/snow-scanner ]; then
+		mkdir /etc/snow-scanner
+		chown snow-scanner:snow-scanner /etc/snow-scanner
+		chmod 770 /etc/snow-scanner
+	fi
+fi
+
+#DEBHELPER#
+
+exit 0

snow-scanner/debian/snow-scanner-worker.service

@@ -1,6 +1,8 @@
 [Unit]
 Description=Snow scanner worker
 After=network.target
+After=snow-scanner.service
+Requires=snow-scanner.service
 
 [Service]
 Type=simple

snow-scanner/src/event_bus.rs

@@ -6,7 +6,8 @@ use hickory_resolver::Name;
 use rocket::futures::channel::mpsc as rocket_mpsc;
 use rocket::futures::StreamExt;
 use rocket::tokio;
-use snow_scanner_worker::detection::{detect_scanner_from_name, validate_ip};
+use snow_scanner_worker::detection::validate_ip;
+use snow_scanner_worker::scanners::ScannerNode;
 
 use crate::Scanner;
 
@@ -57,9 +58,13 @@ impl EventBus {
                     return;
                 }
                 let name = Name::from_str(name.as_str()).unwrap();
-                match detect_scanner_from_name(&name) {
+                let scanner: Result<ScannerNode, String> = name.clone().try_into();
-                    Ok(Some(scanner_type)) => {
+
-                        match Scanner::find_or_new(ip, scanner_type, Some(name), db).await {
+                match scanner {
+                    Ok(scanner_type) => {
+                        match Scanner::find_or_new(ip, scanner_type.info.to_owned(), Some(name), db)
+                            .await
+                        {
                             Ok(scanner) => {
                                 let _ = scanner.save(db).await;
                             }
@@ -68,9 +73,6 @@ impl EventBus {
                             }
                         }
                     }
-                    Ok(None) => {
-                        error!("No name detected for: {:?}", name);
-                    }
 
                     Err(err) => {
                         error!("No name detected error: {:?}", err);

snow-scanner/src/main.rs

@@ -33,13 +33,16 @@ use rocket_ws::WebSocket;
 use server::Server;
 use weighted_rs::Weight;
 
-use snow_scanner_worker::detection::{
+use snow_scanner_worker::scanners::ScannerMethods;
-    detect_scanner, get_dns_client, get_dns_server_config, validate_ip,
-};
-use snow_scanner_worker::modules::{Network, WorkerMessages};
-use snow_scanner_worker::scanners::IsStatic;
-use snow_scanner_worker::scanners::Scanners;
 use snow_scanner_worker::utils::get_dns_rr;
+use snow_scanner_worker::{
+    detection::{get_dns_client, get_dns_server_config, validate_ip},
+    scanners::STRETCHOID,
+};
+use snow_scanner_worker::{
+    modules::{Network, WorkerMessages},
+    scanners::{ScannerData, ScannerNode},
+};
 
 use std::net::SocketAddr;
 use std::{
@@ -125,14 +128,14 @@ impl FromFormField<'_> for SafeIpAddr {
 
 async fn handle_ip(
     query_address: IpAddr,
-) -> Result<(IpAddr, Option<Scanners>, ResolvedResult), ()> {
+) -> Result<(IpAddr, Option<ScannerNode>, ResolvedResult), String> {
-    let ptr_result: Result<ResolvedResult, ()> = std::thread::spawn(move || {
+    let ptr_result: Result<ResolvedResult, String> = std::thread::spawn(move || {
         let mut rr_dns_servers = get_dns_rr();
         let client = get_dns_client(&get_dns_server_config(&rr_dns_servers.next().unwrap()));
         let ptr_result: ResolvedResult = if let Ok(res) = get_ptr(query_address, client) {
             res
         } else {
-            return Err(());
+            return Err("Resolving error".to_string());
         };
         Ok(ptr_result)
     })
@@ -140,17 +143,20 @@ async fn handle_ip(
     .unwrap();
 
     match ptr_result {
-        Ok(result) => match detect_scanner(&result) {
+        Ok(result) => {
-            Ok(Some(scanner_type)) => {
+            let scanner: Result<ScannerNode, String> = result.query.clone().try_into();
-                if !validate_ip(query_address) {
+
-                    error!("Invalid IP address: {query_address}");
+            match scanner {
-                    return Err(());
+                Ok(scanner_type) => {
+                    if !validate_ip(query_address) {
+                        error!("Invalid IP address: {query_address}");
+                        return Err("".to_string());
+                    }
+                    Ok((query_address, Some(scanner_type), result))
                 }
-                Ok((query_address, Some(scanner_type), result))
+                Err(err) => Err(err),
             }
-            Ok(None) => Ok((query_address, None, result)),
+        }
-            Err(err) => Err(err),
-        },
         Err(err) => Err(err),
     }
 }
@@ -243,7 +249,7 @@ async fn handle_scan(
 
                 let msg = EventBusWriterEvent::BroadcastMessage(
                     WorkerMessages::DoWorkRequest {
-                        neworks: vec![Network(cidr)],
+                        networks: vec![Network(cidr)],
                     }
                     .into(),
                 );
@@ -262,45 +268,20 @@ pub struct ReportParams {
     ip: SafeIpAddr,
 }
 
-fn reply_contents_for_scanner_found(scanner: Scanner) -> HtmlContents {
+fn reply_contents_for_scanner_found(scanner: Scanner, scanner_type: ScannerData) -> HtmlContents {
-    HtmlContents(match scanner.scanner_name {
+    HtmlContents(match scanner.last_checked_at {
-        Scanners::Binaryedge => match scanner.last_checked_at {
+        Some(date) => format!(
-            Some(date) => format!(
+            "Reported a {}! <b>{}</b> known as {} since {date}.",
-                "Reported a binaryedge ninja! <b>{}</b> known as {} since {date}.",
+            scanner_type.funny_name,
-                scanner.ip,
+            scanner.ip,
-                scanner.ip_ptr.unwrap_or("".to_string())
+            scanner.ip_ptr.unwrap_or("".to_string())
-            ),
+        ),
-            None => format!(
+        None => format!(
-                "Reported a binaryedge ninja! <b>{}</b> known as {}.",
+            "Reported a {}! <b>{}</b> known as {}.",
-                scanner.ip,
+            scanner_type.funny_name,
-                scanner.ip_ptr.unwrap_or("".to_string())
+            scanner.ip,
-            ),
+            scanner.ip_ptr.unwrap_or("".to_string())
-        },
+        ),
-        Scanners::Stretchoid => match scanner.last_checked_at {
-            Some(date) => format!(
-                "Reported a stretchoid agent! <b>{}</b> known as {} since {date}.",
-                scanner.ip,
-                scanner.ip_ptr.unwrap_or("".to_string())
-            ),
-            None => format!(
-                "Reported a stretchoid agent! <b>{}</b> known as {}.",
-                scanner.ip,
-                scanner.ip_ptr.unwrap_or("".to_string())
-            ),
-        },
-        Scanners::Shadowserver => match scanner.last_checked_at {
-            Some(date) => format!(
-                "Reported a cloudy shadowserver ! <b>{}</b> known as {} since {date}.",
-                scanner.ip,
-                scanner.ip_ptr.unwrap_or("".to_string())
-            ),
-            None => format!(
-                "Reported a cloudy shadowserver ! <b>{}</b> known as {}.",
-                scanner.ip,
-                scanner.ip_ptr.unwrap_or("".to_string())
-            ),
-        },
-        _ => format!("Not supported"),
     })
 }
 
@@ -310,13 +291,16 @@ async fn handle_report(mut db: DbConn, form: Form<ReportParams>) -> MultiReply {
         Ok((query_address, scanner_type, result)) => match scanner_type {
             Some(scanner_type) => match Scanner::find_or_new(
                 query_address,
-                scanner_type,
+                scanner_type.info,
                 result.result.clone(),
                 &mut db,
             )
             .await
             {
-                Ok(scanner) => MultiReply::Content(reply_contents_for_scanner_found(scanner)),
+                Ok(scanner) => MultiReply::Content(reply_contents_for_scanner_found(
+                    scanner,
+                    scanner_type.info,
+                )),
                 Err(err) => MultiReply::Error(ServerError(format!(
                     "The IP {} resolved as {} could not be saved, server error: {err}.",
                     form.ip.addr,
@@ -389,7 +373,7 @@ async fn handle_get_collection(
 #[get("/scanners/<scanner_name>")]
 async fn handle_list_scanners(
     mut db: DbConn,
-    scanner_name: Scanners,
+    scanner_name: ScannerNode,
     app_configs: &State<AppConfigs>,
 ) -> MultiReply {
     let static_data_dir: String = app_configs.static_data_dir.clone();
@@ -397,13 +381,11 @@ async fn handle_list_scanners(
         let mut path: PathBuf = PathBuf::new();
         path.push(static_data_dir);
         path.push("scanners");
-        path.push(match scanner_name {
+        path.push(
-            Scanners::Stretchoid | Scanners::Binaryedge | Scanners::Shadowserver => {
+            scanner_name
-                panic!("This should not happen")
+                .static_file_name()
-            }
+                .expect("Static files should have a static file name"),
-            Scanners::Censys => "censys.txt".to_string(),
+        );
-            Scanners::InternetMeasurement => "internet-measurement.com.txt".to_string(),
-        });
 
         return match NamedFile::open(path).await {
             Ok(file) => MultiReply::FileContents(file),
@@ -411,7 +393,7 @@ async fn handle_list_scanners(
         };
     }
 
-    let scanners_list = match Scanner::list_names(scanner_name, &mut db).await {
+    let scanners_list = match Scanner::list_names(scanner_name.info, &mut db).await {
         Ok(data) => Ok(data),
         Err(err) => Err(err),
     };
@@ -537,7 +519,7 @@ async fn report_counts<'a>(rocket: Rocket<rocket::Build>) -> Rocket<rocket::Buil
             span_error!("failed to connect to MySQL database" => error!("{e}"));
             panic!("aborting launch");
         });
-    match Scanner::list_names(Scanners::Stretchoid, &mut DbConnection(conn)).await {
+    match Scanner::list_names(STRETCHOID, &mut DbConnection(conn)).await {
         Ok(d) => info!("Found {} Stretchoid scanners", d.len()),
         Err(err) => error!("Unable to fetch Stretchoid scanners: {err}"),
     }

snow-scanner/src/models.rs

@@ -1,9 +1,10 @@
 use std::net::IpAddr;
 
-use crate::{DbConn, Scanners};
+use crate::DbConn;
 use chrono::{NaiveDateTime, Utc};
 use hickory_resolver::Name;
 use rocket_db_pools::diesel::{dsl::insert_into, prelude::*, result::Error as DieselError};
+use snow_scanner_worker::scanners::ScannerData;
 
 use crate::schema::scan_tasks::dsl::scan_tasks;
 use crate::schema::scanners::dsl::scanners;
@@ -14,7 +15,7 @@ use crate::schema::scanners::dsl::scanners;
 pub struct Scanner {
     pub ip: String,
     pub ip_type: u8,
-    pub scanner_name: Scanners,
+    pub scanner_name: String,
     pub ip_ptr: Option<String>,
     pub created_at: NaiveDateTime,
     pub updated_at: Option<NaiveDateTime>,
@@ -25,7 +26,7 @@ pub struct Scanner {
 impl Scanner {
     pub async fn find_or_new(
         query_address: IpAddr,
-        scanner_name: Scanners,
+        scanner_data: ScannerData<'static>,
         ptr: Option<Name>,
         conn: &mut DbConn,
     ) -> Result<Scanner, DieselError> {
@@ -45,7 +46,7 @@ impl Scanner {
             Scanner {
                 ip: query_address.to_string(),
                 ip_type: ip_type,
-                scanner_name: scanner_name.clone(),
+                scanner_name: scanner_data.value.to_string(),
                 ip_ptr: match ptr {
                     Some(ptr) => Some(ptr.to_string()),
                     None => None,
@@ -79,15 +80,16 @@ impl Scanner {
     }
 
     pub async fn list_names(
-        scanner_name: Scanners,
+        scanner_data: ScannerData<'static>,
         conn: &mut DbConn,
     ) -> Result<Vec<String>, DieselError> {
         use crate::schema::scanners;
         use crate::schema::scanners::ip;
+        use crate::schema::scanners::scanner_name;
 
         scanners
             .select(ip)
-            .filter(scanners::scanner_name.eq(scanner_name.to_string()))
+            .filter(scanner_name.eq(scanner_data.value))
             .order((scanners::ip_type.desc(), scanners::created_at.desc()))
             .load::<String>(conn)
             .await

snow-scanner/src/server.rs

@@ -271,7 +271,7 @@ impl<'a> Worker<'a> {
                 Ok(())
             }
             WorkerMessages::GetWorkRequest {} => {
-                worker_reply = Some(WorkerMessages::DoWorkRequest { neworks: vec![] });
+                worker_reply = Some(WorkerMessages::DoWorkRequest { networks: vec![] });
                 Ok(())
             }
             WorkerMessages::DoWorkRequest { .. } | WorkerMessages::Invalid { .. } => {

snow-scanner/src/worker/detection.rs

@@ -1,12 +1,8 @@
 use std::net::IpAddr;
-use std::str::FromStr;
 use std::time::Duration;
 
-use crate::scanners::Scanners;
-use dns_ptr_resolver::ResolvedResult;
-
 use hickory_resolver::config::{NameServerConfigGroup, ResolverConfig, ResolverOpts};
-use hickory_resolver::{Name, Resolver};
+use hickory_resolver::Resolver;
 
 use crate::ip_addr::is_global_hardcoded;
 
@@ -33,68 +29,3 @@ pub fn validate_ip(ip: IpAddr) -> bool {
     }
     return is_global_hardcoded(ip);
 }
-
-pub fn detect_scanner(ptr_result: &ResolvedResult) -> Result<Option<Scanners>, ()> {
-    match &ptr_result.result {
-        Some(name) => detect_scanner_from_name(&name),
-        None => Ok(None),
-    }
-}
-
-pub fn detect_scanner_from_name(name: &Name) -> Result<Option<Scanners>, ()> {
-    match name {
-        ref name
-            if name
-                .trim_to(2)
-                .eq_case(&Name::from_str("binaryedge.ninja.").expect("Should parse")) =>
-        {
-            Ok(Some(Scanners::Binaryedge))
-        }
-        ref name
-            if name
-                .trim_to(2)
-                .eq_case(&Name::from_str("stretchoid.com.").expect("Should parse")) =>
-        {
-            Ok(Some(Scanners::Stretchoid))
-        }
-        ref name
-            if name
-                .trim_to(2)
-                .eq_case(&Name::from_str("shadowserver.org.").expect("Should parse")) =>
-        {
-            Ok(Some(Scanners::Shadowserver))
-        }
-        &_ => Ok(None),
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_detect_scanner_from_name() {
-        let ptr = Name::from_str("scan-47e.shadowserver.org.").unwrap();
-
-        assert_eq!(
-            detect_scanner_from_name(&ptr).unwrap(),
-            Some(Scanners::Shadowserver)
-        );
-    }
-
-    #[test]
-    fn test_detect_scanner() {
-        let cname_ptr = Name::from_str("111.0-24.197.62.64.in-addr.arpa.").unwrap();
-        let ptr = Name::from_str("scan-47e.shadowserver.org.").unwrap();
-
-        assert_eq!(
-            detect_scanner(&ResolvedResult {
-                query: cname_ptr,
-                result: Some(ptr),
-                error: None
-            })
-            .unwrap(),
-            Some(Scanners::Shadowserver)
-        );
-    }
-}

snow-scanner/src/worker/modules.rs

@@ -15,7 +15,7 @@ pub enum WorkerMessages {
     #[serde(rename = "get_work")]
     GetWorkRequest {},
     #[serde(rename = "do_work")]
-    DoWorkRequest { neworks: Vec<Network> },
+    DoWorkRequest { networks: Vec<Network> },
     #[serde(rename = "scanner_found")]
     ScannerFoundResponse { name: String, address: IpAddr },
     #[serde(rename = "")]
@@ -95,25 +95,25 @@ mod tests {
 
     #[test]
     fn deserialize_do_work_empty() {
-        let data = "{\"type\":\"do_work\",\"request\":{\"neworks\":[]}}";
+        let data = "{\"type\":\"do_work\",\"request\":{\"networks\":[]}}";
         let result: WorkerMessages = data.to_string().into();
         assert_eq!(
             result,
             WorkerMessages::DoWorkRequest {
-                neworks: [].to_vec()
+                networks: [].to_vec()
             }
         );
     }
 
     #[test]
     fn deserialize_do_work() {
-        let data = "{\"type\":\"do_work\",\"request\":{\"neworks\":[\"127.0.0.0/31\"]}}";
+        let data = "{\"type\":\"do_work\",\"request\":{\"networks\":[\"127.0.0.0/31\"]}}";
         let result: WorkerMessages = data.to_string().into();
         let cidr: IpCidr = IpCidr::from_str("127.0.0.0/31").unwrap();
         assert_eq!(
             result,
             WorkerMessages::DoWorkRequest {
-                neworks: [Network(cidr)].to_vec()
+                networks: [Network(cidr)].to_vec()
             }
         );
     }

snow-scanner/src/worker/scanners.rs

@@ -5,112 +5,150 @@ use diesel::mysql::MysqlValue;
 use diesel::serialize;
 use diesel::serialize::IsNull;
 use diesel::sql_types::Text;
+use hickory_resolver::Name;
 use rocket::request::FromParam;
+use std::str::FromStr;
 
 use serde::{Deserialize, Deserializer};
-use std::fmt;
 use std::io::Write;
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ScannerData<'a> {
+    pub static_file_name: Option<&'a str>,
+    pub funny_name: &'a str,
+    pub display_name: &'a str,
+    pub value: &'a str,
+    pub dns_prefix: Option<&'a str>,
+}
+
+pub const STRETCHOID: ScannerData = ScannerData {
+    static_file_name: None,
+    funny_name: "stretchoid agent",
+    display_name: "stretchoid",
+    value: "stretchoid",
+    dns_prefix: Some("stretchoid.com."),
+};
+pub const BINARYEDGE: ScannerData = ScannerData {
+    static_file_name: None,
+    funny_name: "binaryedge ninja",
+    display_name: "binaryedge",
+    value: "binaryedge",
+    dns_prefix: Some("binaryedge.ninja."),
+};
+
+pub const SHADOWSERVER: ScannerData = ScannerData {
+    static_file_name: None,
+    funny_name: "cloudy shadowserver",
+    display_name: "shadowserver",
+    value: "shadowserver",
+    dns_prefix: Some("shadowserver.org."),
+};
+
+pub fn get_scanners() -> Vec<ScannerData<'static>> {
+    vec![
+        STRETCHOID,
+        BINARYEDGE,
+        SHADOWSERVER,
+        ScannerData {
+            static_file_name: Some("censys.txt"),
+            funny_name: "Censys node",
+            display_name: "censys",
+            value: "censys",
+            dns_prefix: None,
+        },
+        ScannerData {
+            static_file_name: Some("internet-measurement.com.txt"),
+            funny_name: "internet measurement probe",
+            display_name: "internet-measurement.com",
+            value: "internet-measurement.com",
+            dns_prefix: None,
+        },
+        ScannerData {
+            static_file_name: Some("anssi.txt"),
+            funny_name: "French ANSSI probe",
+            display_name: "anssi",
+            value: "anssi",
+            dns_prefix: None,
+        },
+    ]
+}
+
+pub type ScannerNode = ScannersWrapper<ScannerData<'static>>;
+
 #[derive(Debug, Clone, Copy, FromSqlRow, PartialEq)]
-pub enum Scanners {
+pub struct ScannersWrapper<ScannerData> {
-    Stretchoid,
+    pub info: ScannerData,
-    Binaryedge,
-    Shadowserver,
-    Censys,
-    InternetMeasurement,
 }
 
-pub trait IsStatic {
+pub trait ScannerMethods {
     fn is_static(self: &Self) -> bool;
+    fn static_file_name(self: &Self) -> Option<&str>;
+    fn funny_name(self: &Self) -> &str;
 }
 
-impl IsStatic for Scanners {
+impl ScannerMethods for ScannerNode {
     fn is_static(self: &Self) -> bool {
-        match self {
+        self.static_file_name().is_some()
-            Scanners::Censys => true,
+    }
-            Scanners::InternetMeasurement => true,
+
-            _ => false,
+    fn static_file_name(self: &Self) -> Option<&str> {
-        }
+        self.info.static_file_name
+    }
+
+    fn funny_name(self: &Self) -> &str {
+        self.info.funny_name
     }
 }
 
-impl FromParam<'_> for Scanners {
+impl FromParam<'_> for ScannerNode {
     type Error = String;
 
     fn from_param(param: &'_ str) -> Result<Self, Self::Error> {
-        match param {
+        param.try_into()
-            "stretchoid" => Ok(Scanners::Stretchoid),
-            "binaryedge" => Ok(Scanners::Binaryedge),
-            "shadowserver" => Ok(Scanners::Shadowserver),
-            "stretchoid.txt" => Ok(Scanners::Stretchoid),
-            "binaryedge.txt" => Ok(Scanners::Binaryedge),
-            "shadowserver.txt" => Ok(Scanners::Shadowserver),
-            "censys.txt" => Ok(Scanners::Censys),
-            "internet-measurement.com.txt" => Ok(Scanners::InternetMeasurement),
-            v => Err(format!("Unknown value: {v}")),
-        }
     }
 }
 
-impl<'de> Deserialize<'de> for Scanners {
+impl<'de> Deserialize<'de> for ScannerNode {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
         let s = <Vec<String>>::deserialize(deserializer)?;
         let k: &str = s[0].as_str();
-        match k {
+        match k.try_into() {
-            "stretchoid" => Ok(Scanners::Stretchoid),
+            Ok(scanners) => Ok(scanners),
-            "binaryedge" => Ok(Scanners::Binaryedge),
+            Err(v) => Err(serde::de::Error::custom(format!("Unknown value: {}", v))),
-            "shadowserver" => Ok(Scanners::Shadowserver),
-            "stretchoid.txt" => Ok(Scanners::Stretchoid),
-            "binaryedge.txt" => Ok(Scanners::Binaryedge),
-            "shadowserver.txt" => Ok(Scanners::Shadowserver),
-            "censys.txt" => Ok(Scanners::Censys),
-            "internet-measurement.com.txt" => Ok(Scanners::InternetMeasurement),
-            v => Err(serde::de::Error::custom(format!(
-                "Unknown value: {}",
-                v.to_string()
-            ))),
         }
     }
 }
 
-impl fmt::Display for Scanners {
+impl ToString for ScannerNode {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn to_string(&self) -> String {
-        write!(
+        let res: &str = (*self).into();
-            f,
+        res.to_string()
-            "{}",
-            match self {
-                Self::Stretchoid => "stretchoid",
-                Self::Binaryedge => "binaryedge",
-                Self::Censys => "censys",
-                Self::InternetMeasurement => "internet-measurement.com",
-                Self::Shadowserver => "shadowserver",
-            }
-        )
     }
 }
 
-impl serialize::ToSql<Text, Mysql> for Scanners {
+impl Into<&str> for ScannerNode {
+    fn into(self) -> &'static str {
+        self.info.display_name
+    }
+}
+
+impl serialize::ToSql<Text, Mysql> for ScannerNode {
     fn to_sql(&self, out: &mut serialize::Output<Mysql>) -> serialize::Result {
-        match *self {
+        let res: &str = (*self).into();
-            Self::Stretchoid => out.write_all(b"stretchoid")?,
+        out.write_all(res.as_bytes())?;
-            Self::Binaryedge => out.write_all(b"binaryedge")?,
-            Self::Censys => out.write_all(b"censys")?,
-            Self::InternetMeasurement => out.write_all(b"internet-measurement.com")?,
-            Self::Shadowserver => out.write_all(b"shadowserver")?,
-        };
 
         Ok(IsNull::No)
     }
 }
 
-impl deserialize::FromSql<Text, Mysql> for Scanners {
+impl deserialize::FromSql<Text, Mysql> for ScannerNode {
     fn from_sql(bytes: MysqlValue) -> deserialize::Result<Self> {
         let value = <String as deserialize::FromSql<Text, Mysql>>::from_sql(bytes)?;
         let value = &value as &str;
-        let value: Result<Scanners, String> = value.try_into();
+        let value: Result<ScannerNode, String> = value.try_into();
         match value {
             Ok(d) => Ok(d),
             Err(err) => Err(err.into()),
@@ -118,16 +156,54 @@ impl deserialize::FromSql<Text, Mysql> for Scanners {
     }
 }
 
-impl TryInto<Scanners> for &str {
+// Used for FromSql & FromParam & Deserialize
+impl TryInto<ScannerNode> for &str {
     type Error = String;
 
-    fn try_into(self) -> Result<Scanners, Self::Error> {
+    fn try_into(self) -> Result<ScannerNode, Self::Error> {
-        match self {
+        let value: String = self.replace(".txt", "").as_str().to_string();
-            "stretchoid" => Ok(Scanners::Stretchoid),
+        match get_scanners()
-            "binaryedge" => Ok(Scanners::Binaryedge),
+            .iter()
-            "internet-measurement.com" => Ok(Scanners::InternetMeasurement),
+            .find(|scanner| scanner.value.eq(&value))
-            "shadowserver" => Ok(Scanners::Shadowserver),
+        {
-            value => Err(format!("Invalid value: {value}")),
+            Some(scanner) => Ok(ScannersWrapper { info: *scanner }),
+            None => Err(format!("Invalid value: {value}")),
         }
     }
 }
+
+// Used by the DNS logic
+impl TryInto<ScannerNode> for Name {
+    type Error = String;
+
+    fn try_into(self) -> Result<ScannerNode, Self::Error> {
+        let short_name = self.trim_to(2);
+        match get_scanners()
+            .iter()
+            .filter(|scanner| scanner.dns_prefix.is_some())
+            .find(|scanner| {
+                short_name.eq_case(
+                    &Name::from_str(scanner.dns_prefix.expect("Should have a DNS prefix"))
+                        .expect("Should parse"),
+                )
+            }) {
+            Some(scanner) => Ok(ScannersWrapper { info: *scanner }),
+            None => Err(format!("Invalid hostname: {self}")),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::str::FromStr;
+
+    #[test]
+    fn test_detect_scanner_from_name() {
+        let ptr = Name::from_str("scan-47e.shadowserver.org.").unwrap();
+
+        let res: Result<ScannerNode, String> = ptr.try_into();
+
+        assert_eq!(res.unwrap().info, SHADOWSERVER);
+    }
+}

snow-scanner/src/worker/utils.rs

@@ -16,9 +16,9 @@ pub fn get_dns_rr() -> RoundrobinWeight<Vec<IpAddr>> {
         IpAddr::from_str("9.9.9.10").unwrap(),
         IpAddr::from_str("2.56.220.2").unwrap(), // G-Core DNS
         IpAddr::from_str("95.85.95.85").unwrap(), // G-Core DNS
-        IpAddr::from_str("193.110.81.0").unwrap(),   // dns0.eu	AS50902
+        IpAddr::from_str("193.110.81.0").unwrap(), // dns0.eu	AS50902
-        IpAddr::from_str("185.253.5.0").unwrap(),    // dns0.eu	AS50902
+        IpAddr::from_str("185.253.5.0").unwrap(), // dns0.eu	AS50902
-        IpAddr::from_str("74.82.42.42").unwrap(),    // Hurricane Electric	[AS6939]
+        IpAddr::from_str("74.82.42.42").unwrap(), // Hurricane Electric	[AS6939]
     ];
 
     let mut rr: RoundrobinWeight<Vec<IpAddr>> = RoundrobinWeight::new();

snow-scanner/src/worker/worker.rs

@@ -2,10 +2,9 @@ use std::{env, net::IpAddr};
 
 use chrono::{Duration, NaiveDateTime, Utc};
 use cidr::IpCidr;
-use detection::detect_scanner;
 use dns_ptr_resolver::{get_ptr, ResolvedResult};
 use log2::*;
-use scanners::Scanners;
+use scanners::ScannerNode;
 use tungstenite::stream::MaybeTlsStream;
 use tungstenite::{connect, Error, Message, WebSocket};
 use weighted_rs::Weight;
@@ -160,14 +159,16 @@ impl Worker {
         for addr in addresses {
             let client = get_dns_client(&get_dns_server_config(&rr_dns_servers.next().unwrap()));
             match get_ptr(addr, client) {
-                Ok(result) => match detect_scanner(&result) {
+                Ok(result) => {
-                    Ok(Some(scanner_name)) => {
+                    let scanner: Result<ScannerNode, String> = result.query.clone().try_into();
-                        self.report_detection(scanner_name, addr, result);
-                    }
-                    Ok(None) => {}
 
-                    Err(err) => error!("Error detecting for {addr}: {:?}", err),
+                    match scanner {
-                },
+                        Ok(scanner_name) => {
+                            self.report_detection(scanner_name, addr, result);
+                        }
+                        Err(err) => error!("Error detecting for {addr}: {:?}", err),
+                    }
+                }
                 Err(_) => {
                     //debug!("Error processing {addr}: {err}")
                 }
@@ -180,7 +181,12 @@ impl Worker {
         }
     }
 
-    fn report_detection(&mut self, scanner_name: Scanners, addr: IpAddr, result: ResolvedResult) {
+    fn report_detection(
+        &mut self,
+        scanner_name: ScannerNode,
+        addr: IpAddr,
+        result: ResolvedResult,
+    ) {
         info!("Detected {:?} for {addr}", scanner_name);
         let request = WorkerMessages::ScannerFoundResponse {
             name: result.result.unwrap().to_string(),
@@ -195,9 +201,9 @@ impl Worker {
 
     pub fn receive_request(&mut self, server_request: WorkerMessages) -> &Worker {
         match server_request {
-            WorkerMessages::DoWorkRequest { neworks } => {
+            WorkerMessages::DoWorkRequest { networks } => {
-                info!("Work request received for neworks: {:?}", neworks);
+                info!("Work request received for networks: {:?}", networks);
-                for cidr in neworks {
+                for cidr in networks {
                     let cidr = cidr.0;
                     self.work_on_cidr(cidr);
                 }