Swap out blocked from France DNS servers

.gitea/workflows/build-lists.yml

@@ -11,79 +11,6 @@ on:
         - cron: "30 0 */5 * *"
 
 jobs:
-    build-scanners-list:
-        name: Build scanners list
-        environment:
-            name: sudo-bot
-        runs-on: ubuntu-latest
-        strategy:
-          fail-fast: false
-          matrix:
-            type: ["stretchoid", "binaryedge"]
-        steps:
-            - name: Checkout
-              uses: actions/checkout@v4
-            - name: Cache cargo binaries
-              uses: actions/cache@v4
-              id: cache-dns-ptr-resolver
-              with:
-                  path: ~/.cargo/bin/dns-ptr-resolver
-                  key: ${{ runner.os }}-cargo-bin-dns-ptr-resolver-1.1.0
-            - name: Set up toolchain
-              if: steps.cache-dns-ptr-resolver.outputs.cache-hit != 'true'
-              uses: actions-rs/toolchain@v1
-              with:
-                  profile: minimal
-                  toolchain: 1.67
-                  override: true
-            - name: Install dns-ptr-resolver
-              if: steps.cache-dns-ptr-resolver.outputs.cache-hit != 'true'
-              run: cargo install dns-ptr-resolver@1.1.0
-            - name: Build the ${{ matrix.type }} list
-              run: ./make-${{ matrix.type }}.sh
-            - name: Post the summary
-              run: |
-                git add -A
-                printf '### Diff\n```diff\n%s\n```\n' "$(git diff --staged)" >> $GITHUB_STEP_SUMMARY
-            - name: Extract secrets
-              run: |
-                  printf '%s' "${{ secrets.GH_APP_JWT_PRIV_PEM_CONTENTS }}" > ${HOME}/.secret_jwt.pem
-                  printf '%s' "${{ secrets.GPG_PRIVATE_KEY }}" > ${HOME}/.private-key.asc
-            - uses: actions/setup-node@v4
-              with:
-                  node-version: 18
-            - name: Get yarn cache directory path
-              id: yarn-cache-dir-path
-              run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
-            - name: yarn cache
-              uses: actions/cache@v4
-              with:
-                  path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-                  key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-                  restore-keys: |
-                      ${{ runner.os }}-yarn-
-            - name: Install sudo-bot
-              run: yarn global add sudo-bot
-            - name: Run sudo-bot
-              run: |
-                  sudo-bot --verbose \
-                    --jwt-file="${HOME}/.secret_jwt.pem" \
-                    --gh-app-id='17453' \
-                    --installation-id="${{ secrets.INSTALLATION_ID }}" \
-                    --repository-slug='wdes/security' \
-                    --target-branch='main' \
-                    --assign='williamdes' \
-                    --commit-author-email='sudo-bot@wdes.fr' \
-                    --commit-author-name='Sudo Bot' \
-                    --gpg-private-key-file="${HOME}/.private-key.asc" \
-                    --template="$GITHUB_WORKSPACE/.github/sudo-bot-template.js" \
-                    --gpg-private-key-passphrase="${{ secrets.GPG_PASSPHRASE }}"
-            - name: Purge secrets
-              if: always()
-              run: |
-                  rm -v ${HOME}/.secret_jwt.pem
-                  rm -v ${HOME}/.private-key.asc
-
     build-aws-cloudfront:
         runs-on: ubuntu-latest
         steps:

README.md

@@ -6,6 +6,7 @@
 
 - `https://security.wdes.eu/scanners/stretchoid.txt` (List of all known stretchoid IPs)
 - `https://security.wdes.eu/scanners/binaryedge.txt` (List of all known binaryedge IPs)
+- `https://security.wdes.eu/scanners/shadowserver.txt` (List of all known shadowserver IPs)
 - `https://security.wdes.eu/scanners/censys.txt` (List of all IPs declared by censys scanner on their [FAQ](https://support.censys.io/hc/en-us/articles/360043177092-Opt-Out-of-Data-Collection)
 - `https://security.wdes.eu/scanners/internet-measurement.com.txt` (List of all IPs declared by internet-measurement.com on [their website](https://internet-measurement.com/#ips))
 

snow-scanner/Cargo.toml

@@ -44,20 +44,35 @@ members = [
 unstable = []
 
 [dependencies]
-rocket = { git = "https://github.com/rwf2/Rocket/", rev = "3bf9ef02d6e803fe9f753777f5a829dda6d2453d"}
-rocket_ws = { git = "https://github.com/rwf2/Rocket/", rev = "3bf9ef02d6e803fe9f753777f5a829dda6d2453d"}
 rocket_db_pools = { git = "https://github.com/rwf2/Rocket/", rev = "3bf9ef02d6e803fe9f753777f5a829dda6d2453d", default-features = false, features = ["diesel_mysql"] }
+snow-scanner-worker = {path = "./src/worker"}
+diesel.workspace = true
+dns-ptr-resolver.workspace = true
+hickory-resolver.workspace = true
+uuid.workspace = true
+rocket.workspace = true
+rocket_ws.workspace = true
+ws.workspace = true
+chrono.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+cidr.workspace = true
+weighted-rs.workspace = true
 
+[workspace.dependencies]
 # mariadb-dev on Alpine
 # "mysqlclient-src" "mysql_backend"
 diesel = { version = "^2", default-features = false, features = ["mysql", "chrono", "uuid"] }
-
 ws = { package = "rocket_ws", version = "0.1.1" }
 
 dns-ptr-resolver = {git = "https://github.com/wdes/dns-ptr-resolver.git"}
 hickory-resolver = { version = "0.24.1", default-features = false, features = ["tokio-runtime", "dns-over-h3", "dns-over-https", "dns-over-quic"]}
+
+rocket = { git = "https://github.com/rwf2/Rocket/", rev = "3bf9ef02d6e803fe9f753777f5a829dda6d2453d"}
+rocket_ws = { git = "https://github.com/rwf2/Rocket/", rev = "3bf9ef02d6e803fe9f753777f5a829dda6d2453d"}
 chrono = "0.4.38"
 uuid = { version = "1.10.0", default-features = false, features = ["v7", "serde", "std"] }
 cidr = "0.3.0"
 serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "1.0.128"
+weighted-rs = "0.1.3"

snow-scanner/src/event_bus.rs

@@ -1,13 +1,12 @@
 use std::{net::IpAddr, str::FromStr};
 
-use crate::{
+use crate::{DbConnection, SnowDb};
-    worker::detection::{detect_scanner_from_name, validate_ip},
+
-    DbConnection, SnowDb,
-};
 use hickory_resolver::Name;
 use rocket::futures::channel::mpsc as rocket_mpsc;
 use rocket::futures::StreamExt;
 use rocket::tokio;
+use snow_scanner_worker::detection::{detect_scanner_from_name, validate_ip};
 
 use crate::Scanner;
 

snow-scanner/src/main.rs

@@ -7,6 +7,7 @@ use cidr::IpCidr;
 use event_bus::{EventBusSubscriber, EventBusWriter, EventBusWriterEvent};
 use rocket::{
     fairing::AdHoc,
+    form::FromFormField,
     futures::SinkExt,
     http::Status,
     request::{FromParam, FromRequest, Outcome, Request},
@@ -26,28 +27,30 @@ use rocket_db_pools::{
     Connection, Pool,
 };
 
-use crate::worker::modules::{Network, WorkerMessages};
-
-use rocket_db_pools::diesel::mysql::{Mysql, MysqlValue};
-use rocket_db_pools::diesel::serialize::IsNull;
-use rocket_db_pools::diesel::sql_types::Text;
 use rocket_db_pools::diesel::MysqlPool;
-use rocket_db_pools::diesel::{deserialize, serialize};
 use rocket_db_pools::Database;
-
 use rocket_ws::WebSocket;
 use server::Server;
-use worker::detection::{detect_scanner, get_dns_client, validate_ip, Scanners};
+use weighted_rs::Weight;
 
+use snow_scanner_worker::detection::{
+    detect_scanner, get_dns_client, get_dns_server_config, validate_ip,
+};
+use snow_scanner_worker::modules::{Network, WorkerMessages};
+use snow_scanner_worker::scanners::IsStatic;
+use snow_scanner_worker::scanners::Scanners;
+use snow_scanner_worker::utils::get_dns_rr;
+
+use std::net::SocketAddr;
 use std::{
-    env, fmt,
+    env,
+    net::IpAddr,
     ops::{Deref, DerefMut},
 };
-use std::{io::Write, net::SocketAddr};
 use std::{path::PathBuf, str::FromStr};
 use uuid::Uuid;
 
-use serde::{Deserialize, Deserializer, Serialize};
+use serde::{Deserialize, Serialize};
 
 use dns_ptr_resolver::{get_ptr, ResolvedResult};
 
@@ -55,7 +58,6 @@ pub mod event_bus;
 pub mod models;
 pub mod schema;
 pub mod server;
-pub mod worker;
 
 use crate::models::*;
 
@@ -97,116 +99,36 @@ impl<D: Database> DerefMut for DbConnection<D> {
     }
 }
 
-trait IsStatic {
+#[derive(serde::Deserialize, Clone)]
-    fn is_static(self: &Self) -> bool;
+struct SafeIpAddr {
+    pub addr: IpAddr,
 }
 
-impl IsStatic for Scanners {
+impl FromFormField<'_> for SafeIpAddr {
-    fn is_static(self: &Self) -> bool {
+    fn from_value(field: rocket::form::ValueField<'_>) -> rocket::form::Result<'_, Self> {
-        match self {
+        let ip = field.value;
-            Scanners::Censys => true,
+        let query_address = IpAddr::from_str(ip);
-            Scanners::InternetMeasurement => true,
+        match query_address {
-            _ => false,
+            Ok(ip) => {
+                if !validate_ip(ip) {
+                    return Err(rocket::form::Error::validation(format!(
+                        "Invalid IP address: {ip}"
+                    ))
+                    .into());
+                }
+                Ok(SafeIpAddr { addr: ip })
+            }
+            Err(err) => Err(rocket::form::Error::validation(format!("Invalid IP: {err}")).into()),
         }
     }
 }
 
-impl FromParam<'_> for Scanners {
+async fn handle_ip(
-    type Error = String;
+    query_address: IpAddr,
-
+) -> Result<(IpAddr, Option<Scanners>, ResolvedResult), ()> {
-    fn from_param(param: &'_ str) -> Result<Self, Self::Error> {
-        match param {
-            "stretchoid" => Ok(Scanners::Stretchoid),
-            "binaryedge" => Ok(Scanners::Binaryedge),
-            "stretchoid.txt" => Ok(Scanners::Stretchoid),
-            "binaryedge.txt" => Ok(Scanners::Binaryedge),
-            "censys.txt" => Ok(Scanners::Censys),
-            "internet-measurement.com.txt" => Ok(Scanners::InternetMeasurement),
-            v => Err(format!("Unknown value: {v}")),
-        }
-    }
-}
-
-impl<'de> Deserialize<'de> for Scanners {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = <Vec<String>>::deserialize(deserializer)?;
-        let k: &str = s[0].as_str();
-        match k {
-            "stretchoid" => Ok(Scanners::Stretchoid),
-            "binaryedge" => Ok(Scanners::Binaryedge),
-            "stretchoid.txt" => Ok(Scanners::Stretchoid),
-            "binaryedge.txt" => Ok(Scanners::Binaryedge),
-            "censys.txt" => Ok(Scanners::Censys),
-            "internet-measurement.com.txt" => Ok(Scanners::InternetMeasurement),
-            v => Err(serde::de::Error::custom(format!(
-                "Unknown value: {}",
-                v.to_string()
-            ))),
-        }
-    }
-}
-
-impl fmt::Display for Scanners {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            match self {
-                Self::Stretchoid => "stretchoid",
-                Self::Binaryedge => "binaryedge",
-                Self::Censys => "censys",
-                Self::InternetMeasurement => "internet-measurement.com",
-            }
-        )
-    }
-}
-
-impl serialize::ToSql<Text, Mysql> for Scanners {
-    fn to_sql(&self, out: &mut serialize::Output<Mysql>) -> serialize::Result {
-        match *self {
-            Self::Stretchoid => out.write_all(b"stretchoid")?,
-            Self::Binaryedge => out.write_all(b"binaryedge")?,
-            Self::Censys => out.write_all(b"censys")?,
-            Self::InternetMeasurement => out.write_all(b"internet-measurement.com")?,
-        };
-
-        Ok(IsNull::No)
-    }
-}
-
-impl deserialize::FromSql<Text, Mysql> for Scanners {
-    fn from_sql(bytes: MysqlValue) -> deserialize::Result<Self> {
-        let value = <String as deserialize::FromSql<Text, Mysql>>::from_sql(bytes)?;
-        let value = &value as &str;
-        let value: Result<Scanners, String> = value.try_into();
-        match value {
-            Ok(d) => Ok(d),
-            Err(err) => Err(err.into()),
-        }
-    }
-}
-
-impl TryInto<Scanners> for &str {
-    type Error = String;
-
-    fn try_into(self) -> Result<Scanners, Self::Error> {
-        match self {
-            "stretchoid" => Ok(Scanners::Stretchoid),
-            "binaryedge" => Ok(Scanners::Binaryedge),
-            "internet-measurement.com" => Ok(Scanners::InternetMeasurement),
-            value => Err(format!("Invalid value: {value}")),
-        }
-    }
-}
-
-async fn handle_ip(mut conn: DbConn, ip: String) -> Result<Scanner, Option<ResolvedResult>> {
-    let query_address = ip.parse().expect("To parse");
-
     let ptr_result: Result<ResolvedResult, ()> = std::thread::spawn(move || {
-        let client = get_dns_client();
+        let mut rr_dns_servers = get_dns_rr();
+        let client = get_dns_client(&get_dns_server_config(&rr_dns_servers.next().unwrap()));
         let ptr_result: ResolvedResult = if let Ok(res) = get_ptr(query_address, client) {
             res
         } else {
@@ -217,27 +139,19 @@ async fn handle_ip(mut conn: DbConn, ip: String) -> Result<Scanner, Option<Resol
     .join()
     .unwrap();
 
-    if ptr_result.is_err() {
+    match ptr_result {
-        return Err(None);
+        Ok(result) => match detect_scanner(&result) {
-    }
-
-    let result = ptr_result.unwrap();
-
-    match detect_scanner(&result) {
             Ok(Some(scanner_type)) => {
                 if !validate_ip(query_address) {
-                error!("Invalid IP address: {ip}");
+                    error!("Invalid IP address: {query_address}");
-                return Err(None);
+                    return Err(());
                 }
-            match Scanner::find_or_new(query_address, scanner_type, result.result, &mut conn).await
+                Ok((query_address, Some(scanner_type), result))
-            {
-                Ok(scanner) => Ok(scanner),
-                Err(_) => Err(None),
             }
-        }
+            Ok(None) => Ok((query_address, None, result)),
-        Ok(None) => Err(None),
+            Err(err) => Err(err),
-
+        },
-        Err(_) => Err(Some(result)),
+        Err(err) => Err(err),
     }
 }
 
@@ -276,11 +190,15 @@ enum MultiReply {
     Error(ServerError),
     #[response(status = 422)]
     FormError(PlainText),
+    #[response(status = 422)]
+    HtmlFormError(HtmlContents),
     #[response(status = 404)]
     NotFound(String),
     #[response(status = 200)]
     Content(HtmlContents),
     #[response(status = 200)]
+    TextContent(PlainText),
+    #[response(status = 200)]
     FileContents(NamedFile),
 }
 
@@ -294,9 +212,20 @@ async fn handle_scan(
         return MultiReply::FormError(PlainText("Invalid username".to_string()));
     }
 
+    let mut cidrs: Vec<IpCidr> = vec![];
+
+    for line in form.ips.lines() {
+        cidrs.push(match IpCidr::from_str(line.trim()) {
+            Ok(data) => data,
+            Err(err) => {
+                return MultiReply::FormError(PlainText(format!("Invalid value: {line}: {err}")))
+            }
+        });
+    }
+
     let task_group_id: Uuid = Uuid::now_v7();
 
-    for cidr in form.ips.lines() {
+    for cidr in cidrs {
         let scan_task = ScanTask {
             task_group_id: task_group_id.to_string(),
             cidr: cidr.to_string(),
@@ -311,11 +240,10 @@ async fn handle_scan(
         match scan_task.save(&mut db).await {
             Ok(_) => {
                 info!("Added {}", cidr.to_string());
-                let net = IpCidr::from_str(cidr).unwrap();
 
                 let msg = EventBusWriterEvent::BroadcastMessage(
                     WorkerMessages::DoWorkRequest {
-                        neworks: vec![Network(net)],
+                        neworks: vec![Network(cidr)],
                     }
                     .into(),
                 );
@@ -329,15 +257,13 @@ async fn handle_scan(
     MultiReply::Content(HtmlContents(format!("New task added: {} !", task_group_id)))
 }
 
-#[derive(FromForm, Serialize, Deserialize)]
+#[derive(FromForm, Deserialize)]
 pub struct ReportParams {
-    ip: String,
+    ip: SafeIpAddr,
 }
 
-#[post("/report", data = "<form>")]
+fn reply_contents_for_scanner_found(scanner: Scanner) -> HtmlContents {
-async fn handle_report(db: DbConn, form: Form<ReportParams>) -> HtmlContents {
+    HtmlContents(match scanner.scanner_name {
-    match handle_ip(db, form.ip.clone()).await {
-        Ok(scanner) => HtmlContents(match scanner.scanner_name {
         Scanners::Binaryedge => match scanner.last_checked_at {
             Some(date) => format!(
                 "Reported a binaryedge ninja! <b>{}</b> known as {} since {date}.",
@@ -362,17 +288,58 @@ async fn handle_report(db: DbConn, form: Form<ReportParams>) -> HtmlContents {
                 scanner.ip_ptr.unwrap_or("".to_string())
             ),
         },
+        Scanners::Shadowserver => match scanner.last_checked_at {
+            Some(date) => format!(
+                "Reported a cloudy shadowserver ! <b>{}</b> known as {} since {date}.",
+                scanner.ip,
+                scanner.ip_ptr.unwrap_or("".to_string())
+            ),
+            None => format!(
+                "Reported a cloudy shadowserver ! <b>{}</b> known as {}.",
+                scanner.ip,
+                scanner.ip_ptr.unwrap_or("".to_string())
+            ),
+        },
         _ => format!("Not supported"),
-        }),
+    })
-
-        Err(ptr_result) => HtmlContents(format!(
-            "The IP <b>{}</a> resolved as {:?} did not match known scanners patterns.",
-            form.ip,
-            match ptr_result {
-                Some(res) => res.result,
-                None => None,
 }
-        )),
+
+#[post("/report", data = "<form>")]
+async fn handle_report(mut db: DbConn, form: Form<ReportParams>) -> MultiReply {
+    match handle_ip(form.ip.addr).await {
+        Ok((query_address, scanner_type, result)) => match scanner_type {
+            Some(scanner_type) => match Scanner::find_or_new(
+                query_address,
+                scanner_type,
+                result.result.clone(),
+                &mut db,
+            )
+            .await
+            {
+                Ok(scanner) => MultiReply::Content(reply_contents_for_scanner_found(scanner)),
+                Err(err) => MultiReply::Error(ServerError(format!(
+                    "The IP {} resolved as {} could not be saved, server error: {err}.",
+                    form.ip.addr,
+                    match result.result {
+                        Some(res) => res.to_string(),
+                        None => "No value".to_string(),
+                    }
+                ))),
+            },
+            None => MultiReply::HtmlFormError(HtmlContents(format!(
+                "The IP <b>{}</a> resolved as {:?} did not match known scanners patterns.",
+                form.ip.addr,
+                match result.result {
+                    Some(res) => res.to_string(),
+                    None => "No value".to_string(),
+                }
+            ))),
+        },
+
+        Err(_) => MultiReply::Error(ServerError(format!(
+            "The IP <b>{}</a> did encounter en error at resolve time.",
+            form.ip.addr
+        ))),
     }
 }
 
@@ -431,8 +398,9 @@ async fn handle_list_scanners(
         path.push(static_data_dir);
         path.push("scanners");
         path.push(match scanner_name {
-            Scanners::Stretchoid |
+            Scanners::Stretchoid | Scanners::Binaryedge | Scanners::Shadowserver => {
-            Scanners::Binaryedge => panic!("This should not happen"),
+                panic!("This should not happen")
+            }
             Scanners::Censys => "censys.txt".to_string(),
             Scanners::InternetMeasurement => "internet-measurement.com.txt".to_string(),
         });
@@ -449,7 +417,7 @@ async fn handle_list_scanners(
     };
 
     if let Ok(scanners) = scanners_list {
-        MultiReply::Content(HtmlContents(scanners.join("\n")))
+        MultiReply::TextContent(PlainText(scanners.join("\n")))
     } else {
         MultiReply::Error(ServerError("Unable to list scanners".to_string()))
     }
@@ -664,3 +632,35 @@ async fn main() -> Result<(), rocket::Error> {
         .await;
     Ok(())
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use hickory_resolver::{
+        config::{NameServerConfigGroup, ResolverConfig, ResolverOpts},
+        Name, Resolver,
+    };
+    use std::time::Duration;
+
+    #[test]
+    fn test_get_ptr() {
+        let server = NameServerConfigGroup::google();
+        let config = ResolverConfig::from_parts(None, vec![], server);
+        let mut options = ResolverOpts::default();
+        options.timeout = Duration::from_secs(5);
+        options.attempts = 1; // One try
+
+        let resolver = Resolver::new(config, options).unwrap();
+
+        let query_address = "8.8.8.8".parse().expect("To parse");
+
+        assert_eq!(
+            get_ptr(query_address, resolver).unwrap(),
+            ResolvedResult {
+                query: Name::from_str_relaxed("8.8.8.8.in-addr.arpa.").unwrap(),
+                result: Some(Name::from_str_relaxed("dns.google.").unwrap()),
+                error: None,
+            }
+        );
+    }
+}

snow-scanner/src/models.rs

@@ -28,12 +28,12 @@ impl Scanner {
         scanner_name: Scanners,
         ptr: Option<Name>,
         conn: &mut DbConn,
-    ) -> Result<Scanner, ()> {
+    ) -> Result<Scanner, DieselError> {
         let ip_type = if query_address.is_ipv6() { 6 } else { 4 };
         let scanner_row_result = Scanner::find(query_address.to_string(), ip_type, conn).await;
         let scanner_row = match scanner_row_result {
             Ok(scanner_row) => scanner_row,
-            Err(_) => return Err(()),
+            Err(err) => return Err(err),
         };
 
         let scanner = if let Some(mut scanner) = scanner_row {
@@ -58,7 +58,7 @@ impl Scanner {
         };
         match scanner.save(conn).await {
             Ok(scanner) => Ok(scanner),
-            Err(_) => Err(()),
+            Err(err) => Err(err),
         }
     }
 

snow-scanner/src/server.rs

@@ -2,11 +2,9 @@ use rocket::futures::{stream::Next, SinkExt, StreamExt};
 use rocket_ws::{frame::CloseFrame, Message};
 use std::pin::Pin;
 
-use crate::{
+use crate::event_bus::{EventBusEvent, EventBusWriter, EventBusWriterEvent};
-    event_bus::{EventBusEvent, EventBusWriter, EventBusWriterEvent},
-    worker::modules::WorkerMessages,
-};
 use rocket::futures::channel::mpsc as rocket_mpsc;
+use snow_scanner_worker::modules::WorkerMessages;
 
 pub struct Server {}
 

snow-scanner/src/worker/Cargo.toml

@@ -10,17 +10,25 @@ description = "The CLI to run a snow-scanner worker"
 name = "snow-scanner-worker"
 path = "worker.rs"
 
+[lib]
+name = "snow_scanner_worker"
+path = "mod.rs"
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 tungstenite = { version = "0.24.0", default-features = true, features = ["native-tls"] }
-rocket_ws = { git = "https://github.com/rwf2/Rocket/", rev = "3bf9ef02d6e803fe9f753777f5a829dda6d2453d", default-features = true}
+rocket.workspace = true
+rocket_ws.workspace = true
 log2 = "0.1.11"
-diesel = { version = "2", default-features = false, features = [] }
+diesel.workspace = true
-dns-ptr-resolver = {git = "https://github.com/wdes/dns-ptr-resolver.git"}
+dns-ptr-resolver.workspace = true
-hickory-resolver = { version = "0.24.1", default-features = false, features = ["tokio-runtime", "dns-over-h3", "dns-over-https", "dns-over-quic"]}
+hickory-resolver.workspace = true
-chrono = "0.4.38"
+chrono.workspace = true
-uuid = { version = "1.10.0", default-features = false, features = ["v7", "serde", "std"] }
+uuid.workspace = true
-cidr = "0.2.2"
+cidr.workspace = true
-serde = "1.0.210"
+serde.workspace = true
-serde_json = "1.0.128"
+serde_json.workspace = true
+weighted-rs.workspace = true
+rayon = "1.10.0"
+rand = "0.8.5"

snow-scanner/src/worker/detection.rs

@@ -2,32 +2,23 @@ use std::net::IpAddr;
 use std::str::FromStr;
 use std::time::Duration;
 
-use diesel::deserialize::FromSqlRow;
+use crate::scanners::Scanners;
 use dns_ptr_resolver::ResolvedResult;
 
 use hickory_resolver::config::{NameServerConfigGroup, ResolverConfig, ResolverOpts};
 use hickory_resolver::{Name, Resolver};
 
-use crate::worker::ip_addr::is_global_hardcoded;
+use crate::ip_addr::is_global_hardcoded;
 
-#[derive(Debug, Clone, Copy, FromSqlRow)]
+pub fn get_dns_server_config(server_ips: &Vec<IpAddr>) -> NameServerConfigGroup {
-pub enum Scanners {
+    NameServerConfigGroup::from_ips_clear(
-    Stretchoid,
+        server_ips, 53, // Port 53
-    Binaryedge,
+        true,
-    Censys,
+    )
-    InternetMeasurement,
 }
 
-pub fn get_dns_client() -> Resolver {
+pub fn get_dns_client(server: &NameServerConfigGroup) -> Resolver {
-    let server_ip = "1.1.1.1";
+    let config = ResolverConfig::from_parts(None, vec![], server.clone());
-
-    let server = NameServerConfigGroup::from_ips_clear(
-        &[IpAddr::from_str(server_ip).unwrap()],
-        53, // Port 53
-        true,
-    );
-
-    let config = ResolverConfig::from_parts(None, vec![], server);
     let mut options = ResolverOpts::default();
     options.timeout = Duration::from_secs(5);
     options.attempts = 1; // One try
@@ -66,6 +57,44 @@ pub fn detect_scanner_from_name(name: &Name) -> Result<Option<Scanners>, ()> {
         {
             Ok(Some(Scanners::Stretchoid))
         }
+        ref name
+            if name
+                .trim_to(2)
+                .eq_case(&Name::from_str("shadowserver.org.").expect("Should parse")) =>
+        {
+            Ok(Some(Scanners::Shadowserver))
+        }
         &_ => Ok(None),
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_detect_scanner_from_name() {
+        let ptr = Name::from_str("scan-47e.shadowserver.org.").unwrap();
+
+        assert_eq!(
+            detect_scanner_from_name(&ptr).unwrap(),
+            Some(Scanners::Shadowserver)
+        );
+    }
+
+    #[test]
+    fn test_detect_scanner() {
+        let cname_ptr = Name::from_str("111.0-24.197.62.64.in-addr.arpa.").unwrap();
+        let ptr = Name::from_str("scan-47e.shadowserver.org.").unwrap();
+
+        assert_eq!(
+            detect_scanner(&ResolvedResult {
+                query: cname_ptr,
+                result: Some(ptr),
+                error: None
+            })
+            .unwrap(),
+            Some(Scanners::Shadowserver)
+        );
+    }
+}

snow-scanner/src/worker/mod.rs

@@ -1,3 +1,5 @@
 pub mod detection;
 pub mod ip_addr;
 pub mod modules;
+pub mod scanners;
+pub mod utils;

snow-scanner/src/worker/scanners.rs

@@ -0,0 +1,133 @@
+use diesel::deserialize;
+use diesel::deserialize::FromSqlRow;
+use diesel::mysql::Mysql;
+use diesel::mysql::MysqlValue;
+use diesel::serialize;
+use diesel::serialize::IsNull;
+use diesel::sql_types::Text;
+use rocket::request::FromParam;
+
+use serde::{Deserialize, Deserializer};
+use std::fmt;
+use std::io::Write;
+
+#[derive(Debug, Clone, Copy, FromSqlRow, PartialEq)]
+pub enum Scanners {
+    Stretchoid,
+    Binaryedge,
+    Shadowserver,
+    Censys,
+    InternetMeasurement,
+}
+
+pub trait IsStatic {
+    fn is_static(self: &Self) -> bool;
+}
+
+impl IsStatic for Scanners {
+    fn is_static(self: &Self) -> bool {
+        match self {
+            Scanners::Censys => true,
+            Scanners::InternetMeasurement => true,
+            _ => false,
+        }
+    }
+}
+
+impl FromParam<'_> for Scanners {
+    type Error = String;
+
+    fn from_param(param: &'_ str) -> Result<Self, Self::Error> {
+        match param {
+            "stretchoid" => Ok(Scanners::Stretchoid),
+            "binaryedge" => Ok(Scanners::Binaryedge),
+            "shadowserver" => Ok(Scanners::Shadowserver),
+            "stretchoid.txt" => Ok(Scanners::Stretchoid),
+            "binaryedge.txt" => Ok(Scanners::Binaryedge),
+            "shadowserver.txt" => Ok(Scanners::Shadowserver),
+            "censys.txt" => Ok(Scanners::Censys),
+            "internet-measurement.com.txt" => Ok(Scanners::InternetMeasurement),
+            v => Err(format!("Unknown value: {v}")),
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for Scanners {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = <Vec<String>>::deserialize(deserializer)?;
+        let k: &str = s[0].as_str();
+        match k {
+            "stretchoid" => Ok(Scanners::Stretchoid),
+            "binaryedge" => Ok(Scanners::Binaryedge),
+            "shadowserver" => Ok(Scanners::Shadowserver),
+            "stretchoid.txt" => Ok(Scanners::Stretchoid),
+            "binaryedge.txt" => Ok(Scanners::Binaryedge),
+            "shadowserver.txt" => Ok(Scanners::Shadowserver),
+            "censys.txt" => Ok(Scanners::Censys),
+            "internet-measurement.com.txt" => Ok(Scanners::InternetMeasurement),
+            v => Err(serde::de::Error::custom(format!(
+                "Unknown value: {}",
+                v.to_string()
+            ))),
+        }
+    }
+}
+
+impl fmt::Display for Scanners {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "{}",
+            match self {
+                Self::Stretchoid => "stretchoid",
+                Self::Binaryedge => "binaryedge",
+                Self::Censys => "censys",
+                Self::InternetMeasurement => "internet-measurement.com",
+                Self::Shadowserver => "shadowserver",
+            }
+        )
+    }
+}
+
+impl serialize::ToSql<Text, Mysql> for Scanners {
+    fn to_sql(&self, out: &mut serialize::Output<Mysql>) -> serialize::Result {
+        match *self {
+            Self::Stretchoid => out.write_all(b"stretchoid")?,
+            Self::Binaryedge => out.write_all(b"binaryedge")?,
+            Self::Censys => out.write_all(b"censys")?,
+            Self::InternetMeasurement => out.write_all(b"internet-measurement.com")?,
+            Self::Shadowserver => out.write_all(b"shadowserver")?,
+        };
+
+        Ok(IsNull::No)
+    }
+}
+
+impl deserialize::FromSql<Text, Mysql> for Scanners {
+    fn from_sql(bytes: MysqlValue) -> deserialize::Result<Self> {
+        let value = <String as deserialize::FromSql<Text, Mysql>>::from_sql(bytes)?;
+        let value = &value as &str;
+        let value: Result<Scanners, String> = value.try_into();
+        match value {
+            Ok(d) => Ok(d),
+            Err(err) => Err(err.into()),
+        }
+    }
+}
+
+impl TryInto<Scanners> for &str {
+    type Error = String;
+
+    fn try_into(self) -> Result<Scanners, Self::Error> {
+        match self {
+            "stretchoid" => Ok(Scanners::Stretchoid),
+            "binaryedge" => Ok(Scanners::Binaryedge),
+            "internet-measurement.com" => Ok(Scanners::InternetMeasurement),
+            "shadowserver" => Ok(Scanners::Shadowserver),
+            value => Err(format!("Invalid value: {value}")),
+        }
+    }
+}

snow-scanner/src/worker/utils.rs

@@ -0,0 +1,33 @@
+use rand::seq::SliceRandom;
+use rand::thread_rng;
+use std::net::IpAddr;
+use weighted_rs::{RoundrobinWeight, Weight};
+
+pub fn get_dns_rr() -> RoundrobinWeight<Vec<IpAddr>> {
+    use std::str::FromStr;
+
+    // https://gist.github.com/mutin-sa/5dcbd35ee436eb629db7872581093bc5
+    let dns_servers: Vec<IpAddr> = vec![
+        IpAddr::from_str("1.1.1.1").unwrap(),
+        IpAddr::from_str("1.0.0.1").unwrap(),
+        IpAddr::from_str("8.8.8.8").unwrap(),
+        IpAddr::from_str("8.8.4.4").unwrap(),
+        IpAddr::from_str("9.9.9.9").unwrap(),
+        IpAddr::from_str("9.9.9.10").unwrap(),
+        IpAddr::from_str("2.56.220.2").unwrap(), // G-Core DNS
+        IpAddr::from_str("95.85.95.85").unwrap(), // G-Core DNS
+        IpAddr::from_str("193.110.81.0").unwrap(),   // dns0.eu	AS50902
+        IpAddr::from_str("185.253.5.0").unwrap(),    // dns0.eu	AS50902
+        IpAddr::from_str("74.82.42.42").unwrap(),    // Hurricane Electric	[AS6939]
+    ];
+
+    let mut rr: RoundrobinWeight<Vec<IpAddr>> = RoundrobinWeight::new();
+    // For each entry in the list we create a lot of two DNS servers to use
+    for _ in &dns_servers {
+        let mut client_servers = dns_servers.clone();
+        client_servers.shuffle(&mut thread_rng());
+        client_servers.truncate(2);
+        rr.add(client_servers, 1);
+    }
+    rr
+}

snow-scanner/src/worker/worker.rs

@@ -1,18 +1,24 @@
 use std::{env, net::IpAddr};
 
 use chrono::{Duration, NaiveDateTime, Utc};
+use cidr::IpCidr;
 use detection::detect_scanner;
-use dns_ptr_resolver::get_ptr;
+use dns_ptr_resolver::{get_ptr, ResolvedResult};
 use log2::*;
+use scanners::Scanners;
 use tungstenite::stream::MaybeTlsStream;
 use tungstenite::{connect, Error, Message, WebSocket};
+use weighted_rs::Weight;
 
 pub mod detection;
 pub mod ip_addr;
 pub mod modules;
+pub mod scanners;
+pub mod utils;
 
-use crate::detection::get_dns_client;
+use crate::detection::{get_dns_client, get_dns_server_config};
 use crate::modules::WorkerMessages;
+use crate::utils::get_dns_rr;
 
 #[derive(Debug, Clone)]
 pub struct IpToResolve {
@@ -143,32 +149,20 @@ impl Worker {
         self
     }
 
-    pub fn receive_request(&mut self, server_request: WorkerMessages) -> &Worker {
+    fn work_on_cidr(&mut self, cidr: IpCidr) {
-        match server_request {
-            WorkerMessages::DoWorkRequest { neworks } => {
-                info!("Should work on: {:?}", neworks);
-                for cidr in neworks {
-                    let cidr = cidr.0;
         info!("Picking up: {cidr}");
         info!("Range, from {} to {}", cidr.first(), cidr.last());
         let addresses = cidr.iter().addresses();
         let count = addresses.count();
         let mut current = 0;
+        let mut rr_dns_servers = get_dns_rr();
+
         for addr in addresses {
-                        let client = get_dns_client();
+            let client = get_dns_client(&get_dns_server_config(&rr_dns_servers.next().unwrap()));
             match get_ptr(addr, client) {
                 Ok(result) => match detect_scanner(&result) {
                     Ok(Some(scanner_name)) => {
-                                    info!("Detected {:?} for {addr}", scanner_name);
+                        self.report_detection(scanner_name, addr, result);
-                                    let request = WorkerMessages::ScannerFoundResponse {
-                                        name: result.result.unwrap().to_string(),
-                                        address: addr,
-                                    };
-                                    let msg_string: String = request.to_string();
-                                    match self.ws.send(Message::Text(msg_string)) {
-                                        Ok(_) => {}
-                                        Err(err) => error!("Unable to send scanner result: {err}"),
-                                    }
                     }
                     Ok(None) => {}
 
@@ -180,9 +174,34 @@ impl Worker {
             };
 
             current += 1;
+            if current % 10 == 0 {
+                info!("Progress: {count}/{current}");
             }
         }
     }
+
+    fn report_detection(&mut self, scanner_name: Scanners, addr: IpAddr, result: ResolvedResult) {
+        info!("Detected {:?} for {addr}", scanner_name);
+        let request = WorkerMessages::ScannerFoundResponse {
+            name: result.result.unwrap().to_string(),
+            address: addr,
+        };
+        let msg_string: String = request.to_string();
+        match self.ws.send(Message::Text(msg_string)) {
+            Ok(_) => {}
+            Err(err) => error!("Unable to send scanner result: {err}"),
+        }
+    }
+
+    pub fn receive_request(&mut self, server_request: WorkerMessages) -> &Worker {
+        match server_request {
+            WorkerMessages::DoWorkRequest { neworks } => {
+                info!("Work request received for neworks: {:?}", neworks);
+                for cidr in neworks {
+                    let cidr = cidr.0;
+                    self.work_on_cidr(cidr);
+                }
+            }
             WorkerMessages::AuthenticateRequest { .. }
             | WorkerMessages::ScannerFoundResponse { .. }
             | WorkerMessages::GetWorkRequest {}
@@ -194,6 +213,68 @@ impl Worker {
     }
 }
 
+/*fn resolve_file(addresses: InetAddressIterator<IpAddr>, dns_servers: Vec<&str>) {
+
+
+    let mut ips = vec![];
+    for address in addresses {
+        match IpAddr::from_str(address) {
+            Ok(addr) => ips.push(IpToResolve {
+                address: addr,
+                server: rr.next().unwrap(),
+            }),
+            Err(err) => {
+                eprintln!(
+                    "Something went wrong while parsing the IP ({}): {}",
+                    address, err
+                );
+                process::exit(1);
+            }
+        }
+    }
+
+    match rayon::ThreadPoolBuilder::new()
+        .num_threads(30)
+        .build_global()
+    {
+        Ok(r) => r,
+        Err(err) => {
+            eprintln!(
+                "Something went wrong while building the thread pool: {}",
+                err
+            );
+            process::exit(1);
+        }
+    }
+
+    ips.into_par_iter()
+        .enumerate()
+        .for_each(|(_i, to_resolve)| {
+            let server = NameServerConfigGroup::from_ips_clear(
+                &[to_resolve.server.ip()],
+                to_resolve.server.port(),
+                true,
+            );
+
+            let ptr_result = get_ptr(to_resolve.address, resolver);
+            match ptr_result {
+                Ok(ptr) => match ptr.result {
+                    Some(res) => println!("{} # {}", to_resolve.address, res),
+                    None => println!("{}", to_resolve.address),
+                },
+                Err(err) => {
+                    let two_hundred_millis = Duration::from_millis(400);
+                    thread::sleep(two_hundred_millis);
+
+                    eprintln!(
+                        "[{}] Error for {} -> {}",
+                        to_resolve.server, to_resolve.address, err.message
+                    )
+                }
+            }
+        });
+}*/
+
 fn main() -> () {
     let _log2 = log2::stdout()
         .module(true)