Document how to release

snow-scanner/.gitignore

@@ -5,7 +5,7 @@ target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
+#Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

snow-scanner/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snow-scanner"
-version = "1.0.0"
+version = "1.1.0"
 authors = ["William Desportes <williamdes@wdes.fr>"]
 edition = "2021"
 rust-version = "1.81.0"  # MSRV

snow-scanner/README.md

@@ -26,3 +26,18 @@ ROCKET_PROFILE="debug"
 # Setup TLS
 ROCKET_TLS='{certs="/etc/ssl/certs/cert.pem",key="/etc/ssl/private/key.pem", mutual={ca_certs="/etc/ssl/certs/cloudflare.crt",mandatory=true}}'
 ```
+
+## Make a new release
+
+Pre-requirements: `cargo install cargo-deb`
+
+- Bump `Cargo.toml`
+- Commit it
+- Run `cd snow-scanner`
+- Run `cargo update`
+- Commit the changes to the lockfile
+- Run tests `cargo test`
+- Run `cargo deb`
+- Run `./debian/upload.sh target/debian/snow-scanner_1.1.0-1_amd64.deb`
+- Run `git tag -a -s -m "snow-scanner/1.1.0" snow-scanner/1.1.0`
+- Check `debdiff ../../snow-scanner_1.0.0-1_amd64.deb target/debian/snow-scanner_1.1.0-1_amd64.deb`

snow-scanner/debian/postinst

@@ -0,0 +1,21 @@
+#!/bin/sh
+# postinst script for snow-scanner
+
+set -eu
+
+if [ "$1" = "configure" ]; then
+
+	if ! getent passwd | grep -q "^snow-scanner:"; then
+		useradd --shell /bin/sh snow-scanner
+	fi
+
+	if [ ! -d /etc/snow-scanner ]; then
+		mkdir /etc/snow-scanner
+		chown snow-scanner:snow-scanner /etc/snow-scanner
+		chmod 770 /etc/snow-scanner
+	fi
+fi
+
+#DEBHELPER#
+
+exit 0

snow-scanner/src/event_bus.rs

@@ -7,7 +7,7 @@ use rocket::futures::channel::mpsc as rocket_mpsc;
 use rocket::futures::StreamExt;
 use rocket::tokio;
 use snow_scanner_worker::detection::validate_ip;
-use snow_scanner_worker::scanners::Scanners;
+use snow_scanner_worker::scanners::ScannerNode;
 
 use crate::Scanner;
 
@@ -58,11 +58,11 @@ impl EventBus {
                     return;
                 }
                 let name = Name::from_str(name.as_str()).unwrap();
-                let scanner: Result<Scanners, String> = name.clone().try_into();
+                let scanner: Result<ScannerNode, String> = name.clone().try_into();
 
                 match scanner {
                     Ok(scanner_type) => {
-                        match Scanner::find_or_new(ip, scanner_type.to_owned(), Some(name), db)
+                        match Scanner::find_or_new(ip, scanner_type.info.to_owned(), Some(name), db)
                             .await
                         {
                             Ok(scanner) => {

snow-scanner/src/main.rs

@@ -33,11 +33,16 @@ use rocket_ws::WebSocket;
 use server::Server;
 use weighted_rs::Weight;
 
-use snow_scanner_worker::detection::{get_dns_client, get_dns_server_config, validate_ip};
-use snow_scanner_worker::modules::{Network, WorkerMessages};
 use snow_scanner_worker::scanners::ScannerMethods;
-use snow_scanner_worker::scanners::Scanners;
 use snow_scanner_worker::utils::get_dns_rr;
+use snow_scanner_worker::{
+    detection::{get_dns_client, get_dns_server_config, validate_ip},
+    scanners::STRETCHOID,
+};
+use snow_scanner_worker::{
+    modules::{Network, WorkerMessages},
+    scanners::{ScannerData, ScannerNode},
+};
 
 use std::net::SocketAddr;
 use std::{
@@ -123,7 +128,7 @@ impl FromFormField<'_> for SafeIpAddr {
 
 async fn handle_ip(
     query_address: IpAddr,
-) -> Result<(IpAddr, Option<Scanners>, ResolvedResult), String> {
+) -> Result<(IpAddr, Option<ScannerNode>, ResolvedResult), String> {
     let ptr_result: Result<ResolvedResult, String> = std::thread::spawn(move || {
         let mut rr_dns_servers = get_dns_rr();
         let client = get_dns_client(&get_dns_server_config(&rr_dns_servers.next().unwrap()));
@@ -139,7 +144,7 @@ async fn handle_ip(
 
     match ptr_result {
         Ok(result) => {
-            let scanner: Result<Scanners, String> = result.query.clone().try_into();
+            let scanner: Result<ScannerNode, String> = result.query.clone().try_into();
 
             match scanner {
                 Ok(scanner_type) => {
@@ -244,7 +249,7 @@ async fn handle_scan(
 
                 let msg = EventBusWriterEvent::BroadcastMessage(
                     WorkerMessages::DoWorkRequest {
-                        neworks: vec![Network(cidr)],
+                        networks: vec![Network(cidr)],
                     }
                     .into(),
                 );
@@ -263,17 +268,17 @@ pub struct ReportParams {
     ip: SafeIpAddr,
 }
 
-fn reply_contents_for_scanner_found(scanner: Scanner) -> HtmlContents {
+fn reply_contents_for_scanner_found(scanner: Scanner, scanner_type: ScannerData) -> HtmlContents {
     HtmlContents(match scanner.last_checked_at {
         Some(date) => format!(
             "Reported a {}! <b>{}</b> known as {} since {date}.",
-            scanner.scanner_name.funny_name(),
+            scanner_type.funny_name,
             scanner.ip,
             scanner.ip_ptr.unwrap_or("".to_string())
         ),
         None => format!(
             "Reported a {}! <b>{}</b> known as {}.",
-            scanner.scanner_name.funny_name(),
+            scanner_type.funny_name,
             scanner.ip,
             scanner.ip_ptr.unwrap_or("".to_string())
         ),
@@ -286,13 +291,16 @@ async fn handle_report(mut db: DbConn, form: Form<ReportParams>) -> MultiReply {
         Ok((query_address, scanner_type, result)) => match scanner_type {
             Some(scanner_type) => match Scanner::find_or_new(
                 query_address,
-                scanner_type,
+                scanner_type.info,
                 result.result.clone(),
                 &mut db,
             )
             .await
             {
-                Ok(scanner) => MultiReply::Content(reply_contents_for_scanner_found(scanner)),
+                Ok(scanner) => MultiReply::Content(reply_contents_for_scanner_found(
+                    scanner,
+                    scanner_type.info,
+                )),
                 Err(err) => MultiReply::Error(ServerError(format!(
                     "The IP {} resolved as {} could not be saved, server error: {err}.",
                     form.ip.addr,
@@ -365,7 +373,7 @@ async fn handle_get_collection(
 #[get("/scanners/<scanner_name>")]
 async fn handle_list_scanners(
     mut db: DbConn,
-    scanner_name: Scanners,
+    scanner_name: ScannerNode,
     app_configs: &State<AppConfigs>,
 ) -> MultiReply {
     let static_data_dir: String = app_configs.static_data_dir.clone();
@@ -385,7 +393,7 @@ async fn handle_list_scanners(
         };
     }
 
-    let scanners_list = match Scanner::list_names(scanner_name, &mut db).await {
+    let scanners_list = match Scanner::list_names(scanner_name.info, &mut db).await {
         Ok(data) => Ok(data),
         Err(err) => Err(err),
     };
@@ -511,7 +519,7 @@ async fn report_counts<'a>(rocket: Rocket<rocket::Build>) -> Rocket<rocket::Buil
             span_error!("failed to connect to MySQL database" => error!("{e}"));
             panic!("aborting launch");
         });
-    match Scanner::list_names(Scanners::Stretchoid, &mut DbConnection(conn)).await {
+    match Scanner::list_names(STRETCHOID, &mut DbConnection(conn)).await {
         Ok(d) => info!("Found {} Stretchoid scanners", d.len()),
         Err(err) => error!("Unable to fetch Stretchoid scanners: {err}"),
     }

snow-scanner/src/models.rs

@@ -1,9 +1,10 @@
 use std::net::IpAddr;
 
-use crate::{DbConn, Scanners};
+use crate::DbConn;
 use chrono::{NaiveDateTime, Utc};
 use hickory_resolver::Name;
 use rocket_db_pools::diesel::{dsl::insert_into, prelude::*, result::Error as DieselError};
+use snow_scanner_worker::scanners::ScannerData;
 
 use crate::schema::scan_tasks::dsl::scan_tasks;
 use crate::schema::scanners::dsl::scanners;
@@ -14,7 +15,7 @@ use crate::schema::scanners::dsl::scanners;
 pub struct Scanner {
     pub ip: String,
     pub ip_type: u8,
-    pub scanner_name: Scanners,
+    pub scanner_name: String,
     pub ip_ptr: Option<String>,
     pub created_at: NaiveDateTime,
     pub updated_at: Option<NaiveDateTime>,
@@ -25,7 +26,7 @@ pub struct Scanner {
 impl Scanner {
     pub async fn find_or_new(
         query_address: IpAddr,
-        scanner_name: Scanners,
+        scanner_data: ScannerData<'static>,
         ptr: Option<Name>,
         conn: &mut DbConn,
     ) -> Result<Scanner, DieselError> {
@@ -45,7 +46,7 @@ impl Scanner {
             Scanner {
                 ip: query_address.to_string(),
                 ip_type: ip_type,
-                scanner_name: scanner_name.clone(),
+                scanner_name: scanner_data.value.to_string(),
                 ip_ptr: match ptr {
                     Some(ptr) => Some(ptr.to_string()),
                     None => None,
@@ -79,15 +80,16 @@ impl Scanner {
     }
 
     pub async fn list_names(
-        scanner_name: Scanners,
+        scanner_data: ScannerData<'static>,
         conn: &mut DbConn,
     ) -> Result<Vec<String>, DieselError> {
         use crate::schema::scanners;
         use crate::schema::scanners::ip;
+        use crate::schema::scanners::scanner_name;
 
         scanners
             .select(ip)
-            .filter(scanners::scanner_name.eq(scanner_name.to_string()))
+            .filter(scanner_name.eq(scanner_data.value))
             .order((scanners::ip_type.desc(), scanners::created_at.desc()))
             .load::<String>(conn)
             .await

snow-scanner/src/server.rs

@@ -271,7 +271,7 @@ impl<'a> Worker<'a> {
                 Ok(())
             }
             WorkerMessages::GetWorkRequest {} => {
-                worker_reply = Some(WorkerMessages::DoWorkRequest { neworks: vec![] });
+                worker_reply = Some(WorkerMessages::DoWorkRequest { networks: vec![] });
                 Ok(())
             }
             WorkerMessages::DoWorkRequest { .. } | WorkerMessages::Invalid { .. } => {

snow-scanner/src/worker/modules.rs

@@ -15,7 +15,7 @@ pub enum WorkerMessages {
     #[serde(rename = "get_work")]
     GetWorkRequest {},
     #[serde(rename = "do_work")]
-    DoWorkRequest { neworks: Vec<Network> },
+    DoWorkRequest { networks: Vec<Network> },
     #[serde(rename = "scanner_found")]
     ScannerFoundResponse { name: String, address: IpAddr },
     #[serde(rename = "")]
@@ -95,25 +95,25 @@ mod tests {
 
     #[test]
     fn deserialize_do_work_empty() {
-        let data = "{\"type\":\"do_work\",\"request\":{\"neworks\":[]}}";
+        let data = "{\"type\":\"do_work\",\"request\":{\"networks\":[]}}";
         let result: WorkerMessages = data.to_string().into();
         assert_eq!(
             result,
             WorkerMessages::DoWorkRequest {
-                neworks: [].to_vec()
+                networks: [].to_vec()
             }
         );
     }
 
     #[test]
     fn deserialize_do_work() {
-        let data = "{\"type\":\"do_work\",\"request\":{\"neworks\":[\"127.0.0.0/31\"]}}";
+        let data = "{\"type\":\"do_work\",\"request\":{\"networks\":[\"127.0.0.0/31\"]}}";
         let result: WorkerMessages = data.to_string().into();
         let cidr: IpCidr = IpCidr::from_str("127.0.0.0/31").unwrap();
         assert_eq!(
             result,
             WorkerMessages::DoWorkRequest {
-                neworks: [Network(cidr)].to_vec()
+                networks: [Network(cidr)].to_vec()
             }
         );
     }

snow-scanner/src/worker/scanners.rs

@@ -12,13 +12,72 @@ use std::str::FromStr;
 use serde::{Deserialize, Deserializer};
 use std::io::Write;
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ScannerData<'a> {
+    pub static_file_name: Option<&'a str>,
+    pub funny_name: &'a str,
+    pub display_name: &'a str,
+    pub value: &'a str,
+    pub dns_prefix: Option<&'a str>,
+}
+
+pub const STRETCHOID: ScannerData = ScannerData {
+    static_file_name: None,
+    funny_name: "stretchoid agent",
+    display_name: "stretchoid",
+    value: "stretchoid",
+    dns_prefix: Some("stretchoid.com."),
+};
+pub const BINARYEDGE: ScannerData = ScannerData {
+    static_file_name: None,
+    funny_name: "binaryedge ninja",
+    display_name: "binaryedge",
+    value: "binaryedge",
+    dns_prefix: Some("binaryedge.ninja."),
+};
+
+pub const SHADOWSERVER: ScannerData = ScannerData {
+    static_file_name: None,
+    funny_name: "cloudy shadowserver",
+    display_name: "shadowserver",
+    value: "shadowserver",
+    dns_prefix: Some("shadowserver.org."),
+};
+
+pub fn get_scanners() -> Vec<ScannerData<'static>> {
+    vec![
+        STRETCHOID,
+        BINARYEDGE,
+        SHADOWSERVER,
+        ScannerData {
+            static_file_name: Some("censys.txt"),
+            funny_name: "Censys node",
+            display_name: "censys",
+            value: "censys",
+            dns_prefix: None,
+        },
+        ScannerData {
+            static_file_name: Some("internet-measurement.com.txt"),
+            funny_name: "internet measurement probe",
+            display_name: "internet-measurement.com",
+            value: "internet-measurement.com",
+            dns_prefix: None,
+        },
+        ScannerData {
+            static_file_name: Some("anssi.txt"),
+            funny_name: "French ANSSI probe",
+            display_name: "anssi",
+            value: "anssi",
+            dns_prefix: None,
+        },
+    ]
+}
+
+pub type ScannerNode = ScannersWrapper<ScannerData<'static>>;
+
 #[derive(Debug, Clone, Copy, FromSqlRow, PartialEq)]
-pub enum Scanners {
-    Stretchoid,
-    Binaryedge,
-    Shadowserver,
-    Censys,
-    InternetMeasurement,
+pub struct ScannersWrapper<ScannerData> {
+    pub info: ScannerData,
 }
 
 pub trait ScannerMethods {
@@ -27,35 +86,21 @@ pub trait ScannerMethods {
     fn funny_name(self: &Self) -> &str;
 }
 
-impl ScannerMethods for Scanners {
+impl ScannerMethods for ScannerNode {
     fn is_static(self: &Self) -> bool {
-        match self {
-            Self::Censys => true,
-            Self::InternetMeasurement => true,
-            _ => false,
-        }
+        self.static_file_name().is_some()
     }
 
     fn static_file_name(self: &Self) -> Option<&str> {
-        match self {
-            Self::Censys => Some("censys.txt"),
-            Self::InternetMeasurement => Some("internet-measurement.com.txt"),
-            _ => None,
-        }
+        self.info.static_file_name
     }
 
     fn funny_name(self: &Self) -> &str {
-        match self {
-            Self::Stretchoid => "stretchoid agent",
-            Self::Binaryedge => "binaryedge ninja",
-            Self::Censys => "Censys node",
-            Self::InternetMeasurement => "internet measurement probe",
-            Self::Shadowserver => "cloudy shadowserver",
-        }
+        self.info.funny_name
     }
 }
 
-impl FromParam<'_> for Scanners {
+impl FromParam<'_> for ScannerNode {
     type Error = String;
 
     fn from_param(param: &'_ str) -> Result<Self, Self::Error> {
@@ -63,7 +108,7 @@ impl FromParam<'_> for Scanners {
     }
 }
 
-impl<'de> Deserialize<'de> for Scanners {
+impl<'de> Deserialize<'de> for ScannerNode {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
@@ -77,32 +122,33 @@ impl<'de> Deserialize<'de> for Scanners {
     }
 }
 
-impl ToString for Scanners {
+impl ToString for ScannerNode {
     fn to_string(&self) -> String {
-        match self {
-            Self::Stretchoid => "stretchoid",
-            Self::Binaryedge => "binaryedge",
-            Self::Censys => "censys",
-            Self::InternetMeasurement => "internet-measurement.com",
-            Self::Shadowserver => "shadowserver",
-        }
-        .to_string()
+        let res: &str = (*self).into();
+        res.to_string()
     }
 }
 
-impl serialize::ToSql<Text, Mysql> for Scanners {
+impl Into<&str> for ScannerNode {
+    fn into(self) -> &'static str {
+        self.info.display_name
+    }
+}
+
+impl serialize::ToSql<Text, Mysql> for ScannerNode {
     fn to_sql(&self, out: &mut serialize::Output<Mysql>) -> serialize::Result {
-        out.write_all(self.to_string().as_bytes())?;
+        let res: &str = (*self).into();
+        out.write_all(res.as_bytes())?;
 
         Ok(IsNull::No)
     }
 }
 
-impl deserialize::FromSql<Text, Mysql> for Scanners {
+impl deserialize::FromSql<Text, Mysql> for ScannerNode {
     fn from_sql(bytes: MysqlValue) -> deserialize::Result<Self> {
         let value = <String as deserialize::FromSql<Text, Mysql>>::from_sql(bytes)?;
         let value = &value as &str;
-        let value: Result<Scanners, String> = value.try_into();
+        let value: Result<ScannerNode, String> = value.try_into();
         match value {
             Ok(d) => Ok(d),
             Err(err) => Err(err.into()),
@@ -111,49 +157,38 @@ impl deserialize::FromSql<Text, Mysql> for Scanners {
 }
 
 // Used for FromSql & FromParam & Deserialize
-impl TryInto<Scanners> for &str {
+impl TryInto<ScannerNode> for &str {
     type Error = String;
 
-    fn try_into(self) -> Result<Scanners, Self::Error> {
-        match self.replace(".txt", "").as_str() {
-            "stretchoid" => Ok(Scanners::Stretchoid),
-            "binaryedge" => Ok(Scanners::Binaryedge),
-            "internet-measurement.com" => Ok(Scanners::InternetMeasurement),
-            "shadowserver" => Ok(Scanners::Shadowserver),
-            "censys" => Ok(Scanners::Censys),
-            value => Err(format!("Invalid value: {value}")),
+    fn try_into(self) -> Result<ScannerNode, Self::Error> {
+        let value: String = self.replace(".txt", "").as_str().to_string();
+        match get_scanners()
+            .iter()
+            .find(|scanner| scanner.value.eq(&value))
+        {
+            Some(scanner) => Ok(ScannersWrapper { info: *scanner }),
+            None => Err(format!("Invalid value: {value}")),
         }
     }
 }
 
 // Used by the DNS logic
-impl TryInto<Scanners> for Name {
+impl TryInto<ScannerNode> for Name {
     type Error = String;
 
-    fn try_into(self) -> Result<Scanners, Self::Error> {
-        match self {
-            ref name
-                if name
-                    .trim_to(2)
-                    .eq_case(&Name::from_str("binaryedge.ninja.").expect("Should parse")) =>
-            {
-                Ok(Scanners::Binaryedge)
-            }
-            ref name
-                if name
-                    .trim_to(2)
-                    .eq_case(&Name::from_str("stretchoid.com.").expect("Should parse")) =>
-            {
-                Ok(Scanners::Stretchoid)
-            }
-            ref name
-                if name
-                    .trim_to(2)
-                    .eq_case(&Name::from_str("shadowserver.org.").expect("Should parse")) =>
-            {
-                Ok(Scanners::Shadowserver)
-            }
-            ref name => Err(format!("Invalid hostname: {name}")),
+    fn try_into(self) -> Result<ScannerNode, Self::Error> {
+        let short_name = self.trim_to(2);
+        match get_scanners()
+            .iter()
+            .filter(|scanner| scanner.dns_prefix.is_some())
+            .find(|scanner| {
+                short_name.eq_case(
+                    &Name::from_str(scanner.dns_prefix.expect("Should have a DNS prefix"))
+                        .expect("Should parse"),
+                )
+            }) {
+            Some(scanner) => Ok(ScannersWrapper { info: *scanner }),
+            None => Err(format!("Invalid hostname: {self}")),
         }
     }
 }
@@ -167,24 +202,8 @@ mod test {
     fn test_detect_scanner_from_name() {
         let ptr = Name::from_str("scan-47e.shadowserver.org.").unwrap();
 
-        let res: Result<Scanners, String> = ptr.try_into();
+        let res: Result<ScannerNode, String> = ptr.try_into();
 
-        assert_eq!(res.unwrap(), Scanners::Shadowserver);
-    }
-
-    #[test]
-    fn test_detect_scanner() {
-        let cname_ptr = Name::from_str("111.0-24.197.62.64.in-addr.arpa.").unwrap();
-        let ptr = Name::from_str("scan-47e.shadowserver.org.").unwrap();
-
-        assert_eq!(
-            detect_scanner(&ResolvedResult {
-                query: cname_ptr,
-                result: Some(ptr),
-                error: None
-            })
-            .unwrap(),
-            Some(Scanners::Shadowserver)
-        );
+        assert_eq!(res.unwrap().info, SHADOWSERVER);
     }
 }

snow-scanner/src/worker/worker.rs

@@ -2,10 +2,9 @@ use std::{env, net::IpAddr};
 
 use chrono::{Duration, NaiveDateTime, Utc};
 use cidr::IpCidr;
-use detection::detect_scanner;
 use dns_ptr_resolver::{get_ptr, ResolvedResult};
 use log2::*;
-use scanners::Scanners;
+use scanners::ScannerNode;
 use tungstenite::stream::MaybeTlsStream;
 use tungstenite::{connect, Error, Message, WebSocket};
 use weighted_rs::Weight;
@@ -160,14 +159,16 @@ impl Worker {
         for addr in addresses {
             let client = get_dns_client(&get_dns_server_config(&rr_dns_servers.next().unwrap()));
             match get_ptr(addr, client) {
-                Ok(result) => match detect_scanner(&result) {
-                    Ok(Some(scanner_name)) => {
-                        self.report_detection(scanner_name, addr, result);
-                    }
-                    Ok(None) => {}
+                Ok(result) => {
+                    let scanner: Result<ScannerNode, String> = result.query.clone().try_into();
 
-                    Err(err) => error!("Error detecting for {addr}: {:?}", err),
-                },
+                    match scanner {
+                        Ok(scanner_name) => {
+                            self.report_detection(scanner_name, addr, result);
+                        }
+                        Err(err) => error!("Error detecting for {addr}: {:?}", err),
+                    }
+                }
                 Err(_) => {
                     //debug!("Error processing {addr}: {err}")
                 }
@@ -180,7 +181,12 @@ impl Worker {
         }
     }
 
-    fn report_detection(&mut self, scanner_name: Scanners, addr: IpAddr, result: ResolvedResult) {
+    fn report_detection(
+        &mut self,
+        scanner_name: ScannerNode,
+        addr: IpAddr,
+        result: ResolvedResult,
+    ) {
         info!("Detected {:?} for {addr}", scanner_name);
         let request = WorkerMessages::ScannerFoundResponse {
             name: result.result.unwrap().to_string(),
@@ -195,9 +201,9 @@ impl Worker {
 
     pub fn receive_request(&mut self, server_request: WorkerMessages) -> &Worker {
         match server_request {
-            WorkerMessages::DoWorkRequest { neworks } => {
-                info!("Work request received for neworks: {:?}", neworks);
-                for cidr in neworks {
+            WorkerMessages::DoWorkRequest { networks } => {
+                info!("Work request received for networks: {:?}", networks);
+                for cidr in networks {
                     let cidr = cidr.0;
                     self.work_on_cidr(cidr);
                 }