Initial commit

src/downloader.rs

@@ -0,0 +1,82 @@
+use anyhow::{Context, Result};
+use futures::{StreamExt, stream};
+use std::fs;
+use std::path::{Path, PathBuf};
+
+/// A report summarizing the outcome of the download process.
+#[derive(Debug)]
+pub struct DownloadReport {
+    pub successful: usize,
+    pub failed: usize,
+}
+
+/// Downloads a list of URLs concurrently, with a specified limit.
+pub async fn download_all_rules(
+    urls: &[String],
+    output_dir: &Path,
+    concurrency: usize,
+) -> Result<DownloadReport> {
+    let client = reqwest::Client::new();
+    let mut successful = 0;
+    let mut failed = 0;
+
+    let fetches = stream::iter(urls)
+        .map(|url| {
+            let client = client.clone();
+            let output_path = output_dir.to_path_buf();
+            async move {
+                match download_rule(&client, url, &output_path).await {
+                    Ok(path) => {
+                        println!("[OK]       Saved {}", path.display());
+                        Ok(())
+                    }
+                    Err(e) => {
+                        eprintln!("[FAIL]     Download error for {}: {:?}", url, e);
+                        Err(())
+                    }
+                }
+            }
+        })
+        .buffer_unordered(concurrency)
+        .collect::<Vec<_>>()
+        .await;
+
+    for result in fetches {
+        if result.is_ok() {
+            successful += 1;
+        } else {
+            failed += 1;
+        }
+    }
+
+    Ok(DownloadReport { successful, failed })
+}
+
+/// Downloads a single file from a URL to a destination directory.
+/// Uses a temporary file to ensure atomic writes.
+async fn download_rule(client: &reqwest::Client, url: &str, output_dir: &Path) -> Result<PathBuf> {
+    let file_name = url
+        .split('/')
+        .last()
+        .filter(|s| !s.is_empty())
+        .with_context(|| format!("Could not determine filename for URL '{}'", url))?;
+
+    println!("[DOWNLOAD] from {}", url);
+
+    let final_path = output_dir.join(file_name);
+    let tmp_path = output_dir.join(format!("{}.tmp", file_name));
+
+    // Perform the download
+    let response = client.get(url).send().await?.error_for_status()?;
+    let content = response.bytes().await?;
+
+    // Write to a temporary file first
+    fs::write(&tmp_path, &content)
+        .with_context(|| format!("Failed to write temporary file {}", tmp_path.display()))?;
+
+    // Atomically rename the temporary file to the final destination
+    fs::rename(&tmp_path, &final_path)
+        .with_context(|| format!("Failed to rename temp file to {}", final_path.display()))?;
+
+    Ok(final_path)
+}