Merge pull request #14 from bgpkit/feature/as-population

add apnic population data
bgpkit · Jun 24, 2024 · 43b194b · 43b194b
2 parents 4e6a6d1 + 2252d42
commit 43b194b
Show file tree

Hide file tree

Showing 4 changed files with 127 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -83,6 +83,7 @@ println!(
 Data source:
 - RIPE NCC asnames: <https://ftp.ripe.net/ripe/asnames/asn.txt>
 - CAIDA as-to-organization mapping: <https://www.caida.org/catalog/datasets/as-organizations/>
+- APNIC AS population data: <https://stats.labs.apnic.net/cgi-bin/aspop>
 
 #### Data structure
 
@@ -93,7 +94,7 @@ pub struct AsName {
     pub asn: u32,
     pub name: String,
     pub country: String,
-    pub as2org: Option<As2orgInfo>,
+    pub population: Option<AsnPopulationData>,
 }
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct As2orgInfo {
@@ -102,6 +103,13 @@ pub struct As2orgInfo {
     pub org_id: String,
     pub org_name: String,
 }
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AsnPopulationData {
+    pub user_count: i64,
+    pub percent_country: f64,
+    pub percent_global: f64,
+    pub sample_count: i64,
+}
 ```
 
 #### Usage example

diff --git a/src/asnames/mod.rs b/src/asnames/mod.rs
@@ -15,6 +15,7 @@
 //!     pub name: String,
 //!     pub country: String,
 //!     pub as2org: Option<As2orgInfo>,
+//!     pub population: Option<AsnPopulationData>,
 //! }
 //! #[derive(Debug, Clone, Serialize, Deserialize)]
 //! pub struct As2orgInfo {
@@ -23,6 +24,13 @@
 //!     pub org_id: String,
 //!     pub org_name: String,
 //! }
+//! #[derive(Debug, Clone, Serialize, Deserialize)]
+//! pub struct AsnPopulationData {
+//!     pub user_count: i64,
+//!     pub percent_country: f64,
+//!     pub percent_global: f64,
+//!     pub sample_count: i64,
+//! }
 //! ```
 //!
 //! # Example
@@ -37,16 +45,21 @@
 //! assert_eq!(asnames.get(&400644).unwrap().country, "US");
 //! ```
 
+mod population;
+
+use crate::asnames::population::AsnPopulationData;
 use anyhow::Result;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use tracing::info;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AsName {
     pub asn: u32,
     pub name: String,
     pub country: String,
     pub as2org: Option<As2orgInfo>,
+    pub population: Option<AsnPopulationData>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -60,8 +73,13 @@ pub struct As2orgInfo {
 const DATA_URL: &str = "https://ftp.ripe.net/ripe/asnames/asn.txt";
 
 pub fn get_asnames() -> Result<HashMap<u32, AsName>> {
+    info!("loading asnames from RIPE NCC...");
     let text = oneio::read_to_string(DATA_URL)?;
+    info!("loading as2org data from CAIDA...");
     let as2org = as2org_rs::As2org::new(None)?;
+    info!("loading ASN population data from APNIC...");
+    let population = population::AsnPopulation::new()?;
+
     let asnames = text
         .lines()
         .filter_map(|line| {
@@ -80,11 +98,13 @@ pub fn get_asnames() -> Result<HashMap<u32, AsName>> {
                 org_id: info.org_id.clone(),
                 org_name: info.org_name.clone(),
             });
+            let population = population.get(asn);
             Some(AsName {
                 asn,
                 name: name_str.to_string(),
                 country: country_str.to_string(),
                 as2org,
+                population,
             })
         })
         .collect::<Vec<AsName>>();

diff --git a/src/asnames/population.rs b/src/asnames/population.rs
@@ -0,0 +1,89 @@
+use anyhow::Result;
+use chrono::NaiveDate;
+use serde::{de, Deserialize, Deserializer, Serialize};
+use std::collections::HashMap;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ApnicAsnPopulationEntry {
+    pub rank: u32,
+    #[serde(rename = "AS")]
+    pub asn: u32,
+    #[serde(rename = "Description")]
+    pub description: String,
+    #[serde(rename = "CC")]
+    pub country_code: String,
+    #[serde(rename = "Users")]
+    pub user_count: i64,
+    #[serde(rename = "Percent of CC Pop")]
+    pub percent_country: f64,
+    #[serde(rename = "Percent of Internet")]
+    pub percent_global: f64,
+    #[serde(rename = "Samples")]
+    pub sample_count: i64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct ApnicAsnPopulation {
+    pub copyright: String,
+    pub description: String,
+    #[serde(rename = "Date", deserialize_with = "deserialize_date")]
+    pub date: NaiveDate,
+    #[serde(rename = "Window")]
+    pub window: String,
+    #[serde(rename = "Data")]
+    pub data: Vec<ApnicAsnPopulationEntry>,
+}
+
+fn deserialize_date<'de, D>(d: D) -> Result<NaiveDate, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let string = String::deserialize(d)?;
+    NaiveDate::parse_from_str(string.as_str(), "%d/%m/%Y").map_err(de::Error::custom)
+}
+
+pub struct AsnPopulation {
+    population_map: HashMap<u32, ApnicAsnPopulationEntry>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AsnPopulationData {
+    pub user_count: i64,
+    pub percent_country: f64,
+    pub percent_global: f64,
+    pub sample_count: i64,
+}
+
+impl AsnPopulation {
+    pub fn new() -> Result<Self> {
+        let population: ApnicAsnPopulation =
+            oneio::read_json_struct("https://stats.labs.apnic.net/cgi-bin/aspop?f=j")?;
+        let mut population_map = HashMap::new();
+        for entry in population.data {
+            population_map.insert(entry.asn, entry);
+        }
+        Ok(AsnPopulation { population_map })
+    }
+
+    pub fn get(&self, asn: u32) -> Option<AsnPopulationData> {
+        self.population_map
+            .get(&asn)
+            .map(|entry| AsnPopulationData {
+                user_count: entry.user_count,
+                percent_country: entry.percent_country,
+                percent_global: entry.percent_global,
+                sample_count: entry.sample_count,
+            })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_load_asn_population() {
+        let population = AsnPopulation::new().unwrap();
+        dbg!(population.get(15169).unwrap());
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
@@ -74,6 +74,7 @@
 //! Data source:
 //! - RIPE NCC asnames: <https://ftp.ripe.net/ripe/asnames/asn.txt>
 //! - CAIDA as-to-organization mapping: <https://www.caida.org/catalog/datasets/as-organizations/>
+//! - APNIC AS population data: <https://stats.labs.apnic.net/cgi-bin/aspop>
 //!
 //! ### Data structure
 //!
@@ -84,7 +85,7 @@
 //!     pub asn: u32,
 //!     pub name: String,
 //!     pub country: String,
-//!     pub as2org: Option<As2orgInfo>,
+//!     pub population: Option<AsnPopulationData>,
 //! }
 //! #[derive(Debug, Clone, Serialize, Deserialize)]
 //! pub struct As2orgInfo {
@@ -93,6 +94,13 @@
 //!     pub org_id: String,
 //!     pub org_name: String,
 //! }
+//! #[derive(Debug, Clone, Serialize, Deserialize)]
+//! pub struct AsnPopulationData {
+//!     pub user_count: i64,
+//!     pub percent_country: f64,
+//!     pub percent_global: f64,
+//!     pub sample_count: i64,
+//! }
 //! ```
 //!
 //! ### Usage example