83 lines
2.5 KiB
Rust
83 lines
2.5 KiB
Rust
use anyhow::{Context, Result};
|
|
use futures::{StreamExt, stream};
|
|
use std::fs;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
/// A report summarizing the outcome of the download process.
|
|
#[derive(Debug)]
|
|
pub struct DownloadReport {
|
|
pub successful: usize,
|
|
pub failed: usize,
|
|
}
|
|
|
|
/// Downloads a list of URLs concurrently, with a specified limit.
|
|
pub async fn download_all_rules(
|
|
urls: &[String],
|
|
output_dir: &Path,
|
|
concurrency: usize,
|
|
) -> Result<DownloadReport> {
|
|
let client = reqwest::Client::new();
|
|
let mut successful = 0;
|
|
let mut failed = 0;
|
|
|
|
let fetches = stream::iter(urls)
|
|
.map(|url| {
|
|
let client = client.clone();
|
|
let output_path = output_dir.to_path_buf();
|
|
async move {
|
|
match download_rule(&client, url, &output_path).await {
|
|
Ok(path) => {
|
|
println!("[OK] Saved {}", path.display());
|
|
Ok(())
|
|
}
|
|
Err(e) => {
|
|
eprintln!("[FAIL] Download error for {}: {:?}", url, e);
|
|
Err(())
|
|
}
|
|
}
|
|
}
|
|
})
|
|
.buffer_unordered(concurrency)
|
|
.collect::<Vec<_>>()
|
|
.await;
|
|
|
|
for result in fetches {
|
|
if result.is_ok() {
|
|
successful += 1;
|
|
} else {
|
|
failed += 1;
|
|
}
|
|
}
|
|
|
|
Ok(DownloadReport { successful, failed })
|
|
}
|
|
|
|
/// Downloads a single file from a URL to a destination directory.
|
|
/// Uses a temporary file to ensure atomic writes.
|
|
async fn download_rule(client: &reqwest::Client, url: &str, output_dir: &Path) -> Result<PathBuf> {
|
|
let file_name = url
|
|
.split('/')
|
|
.last()
|
|
.filter(|s| !s.is_empty())
|
|
.with_context(|| format!("Could not determine filename for URL '{}'", url))?;
|
|
|
|
println!("[DOWNLOAD] from {}", url);
|
|
|
|
let final_path = output_dir.join(file_name);
|
|
let tmp_path = output_dir.join(format!("{}.tmp", file_name));
|
|
|
|
// Perform the download
|
|
let response = client.get(url).send().await?.error_for_status()?;
|
|
let content = response.bytes().await?;
|
|
|
|
// Write to a temporary file first
|
|
fs::write(&tmp_path, &content)
|
|
.with_context(|| format!("Failed to write temporary file {}", tmp_path.display()))?;
|
|
|
|
// Atomically rename the temporary file to the final destination
|
|
fs::rename(&tmp_path, &final_path)
|
|
.with_context(|| format!("Failed to rename temp file to {}", final_path.display()))?;
|
|
|
|
Ok(final_path)
|
|
}
|