implement mmdb file handling and worker

Cargo.toml

@@ -98,6 +98,7 @@ ctrlc = "3.5.0"
 arc-swap = "1.7.1"
 prometheus = "0.14.0"
 once_cell = "1.21.3"
+maxminddb = "0.23"
 axum-server = { version = "0.7.3", features = ["tls-openssl"] }
 axum = { version = "0.8.7" }
 tower-http = { version = "0.6.8", features = ["fs"] }

docs/ENVIRONMNET_VARS.md

@@ -14,6 +14,9 @@ export AX_NETWORK_DISABLE_XDP="false"
 # Gen0Sec configuration
 export AX_ARXIGNIS_API_KEY="your-api-key"
 export AX_ARXIGNIS_BASE_URL="https://api.gen0sec.com/v1"
+export AX_ARXIGNIS_THREAT_MMDB_URL="https://s3.amazonaws.com/your-bucket/indicators-latest.mmdb"
+export AX_ARXIGNIS_THREAT_MMDB_PATH="/var/cache/synapse/threat.mmdb"
+export AX_ARXIGNIS_THREAT_MMDB_REFRESH_SECS="0"
 
 # CAPTCHA configuration
 export AX_CAPTCHA_SITE_KEY="your-site-key"

src/cli.rs

@@ -136,6 +136,30 @@ pub struct Gen0SecConfig {
     pub api_key: String,
     #[serde(default = "default_base_url")]
     pub base_url: String,
+    /// URL to the threat-intel MMDB file 
+    #[serde(default)]
+    pub threat_mmdb_url: String,
+    /// Optional local path to store/read the threat MMDB file
+    #[serde(default)]
+    pub threat_mmdb_path: Option<PathBuf>,
+    /// Optional versions.txt URL to check for the latest threat MMDB version
+    #[serde(default)]
+    pub threat_mmdb_versions_url: String,
+    /// How often to refresh the threat MMDB from the remote URL (seconds). Default: 300 (5 minutes)
+    #[serde(default = "default_threat_mmdb_refresh_secs")]
+    pub threat_mmdb_refresh_secs: u64,
+    /// URL to the GeoIP MMDB file
+    #[serde(default)]
+    pub geoip_mmdb_url: String,
+    /// Optional local path to store/read the GeoIP MMDB file
+    #[serde(default)]
+    pub geoip_mmdb_path: Option<PathBuf>,
+    /// Optional versions.txt URL to check for the latest GeoIP MMDB version
+    #[serde(default)]
+    pub geoip_mmdb_versions_url: String,
+    /// How often to refresh the GeoIP MMDB from the remote URL (seconds). Default: 28800 (8 hours)
+    #[serde(default = "default_geoip_mmdb_refresh_secs")]
+    pub geoip_mmdb_refresh_secs: u64,
     #[serde(default = "default_log_sending_enabled")]
     pub log_sending_enabled: bool,
     #[serde(default = "default_include_response_body")]
@@ -150,6 +174,14 @@ fn default_base_url() -> String {
     "https://api.gen0sec.com/v1".to_string()
 }
 
+fn default_threat_mmdb_refresh_secs() -> u64 {
+    300 // 5 minutes
+}
+
+fn default_geoip_mmdb_refresh_secs() -> u64 {
+    28800 // 8 hours
+}
+
 fn default_log_sending_enabled() -> bool {
     true
 }
@@ -229,6 +261,14 @@ impl Config {
             arxignis: Gen0SecConfig {
                 api_key: "".to_string(),
                 base_url: "https://api.gen0sec.com/v1".to_string(),
+                threat_mmdb_url: "".to_string(),
+                threat_mmdb_path: None,
+                threat_mmdb_versions_url: "".to_string(),
+                threat_mmdb_refresh_secs: 300,
+                geoip_mmdb_url: "".to_string(),
+                geoip_mmdb_path: None,
+                geoip_mmdb_versions_url: "".to_string(),
+                geoip_mmdb_refresh_secs: 28800,
                 log_sending_enabled: true,
                 include_response_body: true,
                 max_body_size: 1024 * 1024, // 1MB

src/main.rs

@@ -63,7 +63,6 @@ fn main() -> Result<()> {
         .map_err(|e| anyhow::anyhow!("Failed to install rustls crypto provider: {:?}", e))?;
 
     let args = Args::parse();
-
     // Handle clear certificate command (runs before loading full config)
     if let Some(certificate_name) = &args.clear_certificate {
         // Initialize minimal runtime for async operations
@@ -85,7 +84,11 @@ fn main() -> Result<()> {
         }
 
         // Get certificate path from config
-        let certificate_path = config.pingora.proxy_certificates.unwrap_or_else(|| "/etc/synapse/certs".to_string());
+        let certificate_path = config
+            .pingora
+            .proxy_certificates
+            .clone()
+            .unwrap_or_else(|| "/etc/synapse/certs".to_string());
 
         // Clear the certificate
         rt.block_on(crate::worker::certificate::clear_certificate(
@@ -99,7 +102,9 @@ fn main() -> Result<()> {
     // Validate required arguments when no config file is provided
     if args.config.is_none() {
         if args.arxignis_api_key.is_none() {
-            return Err(anyhow::anyhow!("--arxignis-api-key is required when no config file is provided"));
+            return Err(anyhow::anyhow!(
+                "--arxignis-api-key is required when no config file is provided"
+            ));
         }
     }
 
@@ -415,7 +420,6 @@ async fn async_main(_args: Args, config: Config) -> Result<()> {
     } else {
         log::info!("Embedded ACME server disabled (acme.enabled: false)");
     }
-
     let (shutdown_tx, shutdown_rx) = watch::channel(false);
 
     // Initialize Redis manager if Redis URL is provided
@@ -556,48 +560,115 @@ async fn async_main(_args: Args, config: Config) -> Result<()> {
         if let Err(e) = init_config(
             config.arxignis.base_url.clone(),
             config.arxignis.api_key.clone(),
-        ).await {
+        )
+        .await
+        {
             log::error!("Failed to initialize HTTP filter with config: {}", e);
             log::error!("Aborting startup because WAF config could not be loaded");
             return Err(e);
         }
 
-        // Initialize threat intelligence client
-        if let Err(e) = threat::init_threat_client(
-            config.arxignis.base_url.clone(),
-            config.arxignis.api_key.clone(),
-        ).await {
-            log::warn!("Failed to initialize threat client: {}", e);
+        // Initialize threat intelligence client (Threat MMDB → GeoIP MMDB fallback)
+        if !config.arxignis.threat_mmdb_url.is_empty() || !config.arxignis.geoip_mmdb_url.is_empty() {
+            if let Err(e) = threat::init_threat_client(
+                config.arxignis.base_url.clone(),
+                config.arxignis.api_key.clone(),
+                config.arxignis.threat_mmdb_path.clone(),
+                config.arxignis.geoip_mmdb_path.clone(),
+            )
+            .await
+            {
+                log::warn!("Failed to initialize threat client: {}", e);
+            } else {
+                log::info!("Threat intelligence client initialized");
+
+                // Register Threat MMDB refresh worker if configured
+                // Allow empty versions_url for direct MMDB download
+                if !config.arxignis.threat_mmdb_url.is_empty()
+                    && config.arxignis.threat_mmdb_refresh_secs > 0
+                {
+                    let refresh_interval = config.arxignis.threat_mmdb_refresh_secs;
+                    let worker_config = worker::WorkerConfig {
+                        name: "threat_mmdb".to_string(),
+                        interval_secs: refresh_interval,
+                        enabled: true,
+                    };
+                    let worker = worker::threat_mmdb::ThreatMmdbWorker::new(
+                        refresh_interval,
+                        config.arxignis.threat_mmdb_url.clone(),
+                        config.arxignis.threat_mmdb_versions_url.clone(),
+                        config.arxignis.threat_mmdb_path.clone(),
+                    );
+                    if let Err(e) = worker_manager.register_worker(worker_config, worker) {
+                        log::error!("Failed to register threat MMDB worker: {}", e);
+                    } else {
+                        log::info!(
+                            "Registered threat MMDB worker (interval: {}s, checks for threat intel updates)",
+                            refresh_interval
+                        );
+                    }
+                }
+
+                // Register GeoIP MMDB refresh worker if configured
+                if !config.arxignis.geoip_mmdb_url.is_empty()
+                    && config.arxignis.geoip_mmdb_refresh_secs > 0
+                {
+                    let refresh_interval = config.arxignis.geoip_mmdb_refresh_secs;
+                    let worker_config = worker::WorkerConfig {
+                        name: "geoip_mmdb".to_string(),
+                        interval_secs: refresh_interval,
+                        enabled: true,
+                    };
+                    let worker = worker::geoip_mmdb::GeoipMmdbWorker::new(
+                        refresh_interval,
+                        config.arxignis.geoip_mmdb_url.clone(),
+                        config.arxignis.geoip_mmdb_versions_url.clone(),
+                        config.arxignis.geoip_mmdb_path.clone(),
+                    );
+                    if let Err(e) = worker_manager.register_worker(worker_config, worker) {
+                        log::error!("Failed to register GeoIP MMDB worker: {}", e);
+                    } else {
+                        log::info!(
+                            "Registered GeoIP MMDB worker (interval: {}s, checks for GeoIP database updates)",
+                            refresh_interval
+                        );
+                    }
+                }
+            }
         } else {
-            log::info!("Threat intelligence client initialized");
+            log::warn!("Threat and GeoIP MMDB URLs not configured; threat lookups will use API only");
         }
 
         // Initialize captcha client if configuration is provided
         if let (Some(site_key), Some(secret_key), Some(jwt_secret)) = (
             &config.arxignis.captcha.site_key,
             &config.arxignis.captcha.secret_key,
-            &config.arxignis.captcha.jwt_secret
+            &config.arxignis.captcha.jwt_secret,
         ) {
             let captcha_config = CaptchaConfig {
                 site_key: site_key.clone(),
                 secret_key: secret_key.clone(),
                 jwt_secret: jwt_secret.clone(),
-                provider: CaptchaProvider::from_str(&config.arxignis.captcha.provider).unwrap_or(CaptchaProvider::HCaptcha),
+                provider: CaptchaProvider::from_str(&config.arxignis.captcha.provider)
+                    .unwrap_or(CaptchaProvider::HCaptcha),
                 token_ttl_seconds: config.arxignis.captcha.token_ttl,
                 validation_cache_ttl_seconds: config.arxignis.captcha.cache_ttl,
             };
 
-            if let Err(e) = init_captcha_client(
-                captcha_config,
-            ).await {
+            if let Err(e) = init_captcha_client(captcha_config).await {
                 log::warn!("Failed to initialize captcha client: {}", e);
             } else {
-                log::info!("Captcha client initialized with provider: {}", config.arxignis.captcha.provider);
+                log::info!(
+                    "Captcha client initialized with provider: {}",
+                    config.arxignis.captcha.provider
+                );
                 // Start captcha cache cleanup task
                 start_cache_cleanup_task().await;
             }
         } else {
-            log::info!("Captcha client not initialized (missing site_key, secret_key, or jwt_secret)");
+            log::info!(
+                "Captcha client not initialized (missing site_key, secret_key, or jwt_secret)"
+            );
         }
     } else {
         log::warn!("No API credentials provided, HTTP filter will not be initialized");