feat: migrate to tracing
This commit is contained in:
174
Cargo.lock
generated
174
Cargo.lock
generated
@ -17,6 +17,15 @@ version = "2.0.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aho-corasick"
|
||||||
|
version = "1.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anstream"
|
name = "anstream"
|
||||||
version = "0.6.19"
|
version = "0.6.19"
|
||||||
@ -695,6 +704,12 @@ dependencies = [
|
|||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lazy_static"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libc"
|
name = "libc"
|
||||||
version = "0.2.174"
|
version = "0.2.174"
|
||||||
@ -729,6 +744,15 @@ version = "0.4.27"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "matchers"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
|
||||||
|
dependencies = [
|
||||||
|
"regex-automata 0.1.10",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "memchr"
|
name = "memchr"
|
||||||
version = "2.7.5"
|
version = "2.7.5"
|
||||||
@ -778,6 +802,16 @@ dependencies = [
|
|||||||
"tempfile",
|
"tempfile",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nu-ansi-term"
|
||||||
|
version = "0.46.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
|
||||||
|
dependencies = [
|
||||||
|
"overload",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "object"
|
name = "object"
|
||||||
version = "0.36.7"
|
version = "0.36.7"
|
||||||
@ -843,6 +877,12 @@ dependencies = [
|
|||||||
"vcpkg",
|
"vcpkg",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "overload"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.12.4"
|
version = "0.12.4"
|
||||||
@ -932,6 +972,50 @@ dependencies = [
|
|||||||
"bitflags",
|
"bitflags",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex"
|
||||||
|
version = "1.11.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-automata 0.4.9",
|
||||||
|
"regex-syntax 0.8.5",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-automata"
|
||||||
|
version = "0.1.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
|
||||||
|
dependencies = [
|
||||||
|
"regex-syntax 0.6.29",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-automata"
|
||||||
|
version = "0.4.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-syntax 0.8.5",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.6.29"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.8.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "reqwest"
|
name = "reqwest"
|
||||||
version = "0.12.20"
|
version = "0.12.20"
|
||||||
@ -1065,6 +1149,8 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tracing",
|
||||||
|
"tracing-subscriber",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -1149,6 +1235,15 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sharded-slab"
|
||||||
|
version = "0.1.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
|
||||||
|
dependencies = [
|
||||||
|
"lazy_static",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "shlex"
|
name = "shlex"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
@ -1269,6 +1364,15 @@ dependencies = [
|
|||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thread_local"
|
||||||
|
version = "1.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tinystr"
|
name = "tinystr"
|
||||||
version = "0.8.1"
|
version = "0.8.1"
|
||||||
@ -1393,9 +1497,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
|
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
"tracing-attributes",
|
||||||
"tracing-core",
|
"tracing-core",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-attributes"
|
||||||
|
version = "0.1.29"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tracing-core"
|
name = "tracing-core"
|
||||||
version = "0.1.34"
|
version = "0.1.34"
|
||||||
@ -1403,6 +1519,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
|
checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"once_cell",
|
"once_cell",
|
||||||
|
"valuable",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-log"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
|
||||||
|
dependencies = [
|
||||||
|
"log",
|
||||||
|
"once_cell",
|
||||||
|
"tracing-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-subscriber"
|
||||||
|
version = "0.3.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
|
||||||
|
dependencies = [
|
||||||
|
"matchers",
|
||||||
|
"nu-ansi-term",
|
||||||
|
"once_cell",
|
||||||
|
"regex",
|
||||||
|
"sharded-slab",
|
||||||
|
"smallvec",
|
||||||
|
"thread_local",
|
||||||
|
"tracing",
|
||||||
|
"tracing-core",
|
||||||
|
"tracing-log",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -1446,6 +1592,12 @@ version = "0.2.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "valuable"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vcpkg"
|
name = "vcpkg"
|
||||||
version = "0.2.15"
|
version = "0.2.15"
|
||||||
@ -1570,6 +1722,28 @@ dependencies = [
|
|||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi"
|
||||||
|
version = "0.3.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||||
|
dependencies = [
|
||||||
|
"winapi-i686-pc-windows-gnu",
|
||||||
|
"winapi-x86_64-pc-windows-gnu",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi-i686-pc-windows-gnu"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi-x86_64-pc-windows-gnu"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-link"
|
name = "windows-link"
|
||||||
version = "0.1.3"
|
version = "0.1.3"
|
||||||
|
@ -11,3 +11,5 @@ serde = { version = "1.0", features = ["derive"] }
|
|||||||
tokio = { version = "1.45", features = ["full"] }
|
tokio = { version = "1.45", features = ["full"] }
|
||||||
clap = { version = "4.5", features = ["derive"] }
|
clap = { version = "4.5", features = ["derive"] }
|
||||||
reqwest = { version = "0.12", features = ["json", "stream", "blocking"] }
|
reqwest = { version = "0.12", features = ["json", "stream", "blocking"] }
|
||||||
|
tracing = "0.1"
|
||||||
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
@ -15,4 +15,8 @@ pub struct Args {
|
|||||||
/// Number of concurrent download operations.
|
/// Number of concurrent download operations.
|
||||||
#[arg(long, default_value_t = 10)]
|
#[arg(long, default_value_t = 10)]
|
||||||
pub concurrency: usize,
|
pub concurrency: usize,
|
||||||
|
|
||||||
|
/// Increase logging verbosity. Can be used multiple times (e.g., -v, -vv).
|
||||||
|
#[arg(short, long, action = clap::ArgAction::Count)]
|
||||||
|
pub verbose: u8,
|
||||||
}
|
}
|
||||||
|
@ -3,6 +3,7 @@ use futures::{StreamExt, stream};
|
|||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use tracing::{debug, error, info, instrument, warn};
|
||||||
|
|
||||||
/// A report summarizing the outcome of the download process.
|
/// A report summarizing the outcome of the download process.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@ -21,8 +22,9 @@ pub fn url_to_filename(url: &str) -> Result<String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Scans the output directory and removes any files not in the expected set.
|
/// Scans the output directory and removes any files not in the expected set.
|
||||||
|
#[instrument(skip(output_dir, expected_files))]
|
||||||
pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>) -> Result<()> {
|
pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>) -> Result<()> {
|
||||||
println!("\n▶️ Cleaning up stale rule files...");
|
info!("Checking for stale rule files...");
|
||||||
let mut cleaned_count = 0;
|
let mut cleaned_count = 0;
|
||||||
|
|
||||||
for entry in fs::read_dir(output_dir)
|
for entry in fs::read_dir(output_dir)
|
||||||
@ -31,43 +33,40 @@ pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>)
|
|||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
|
|
||||||
// Ignore directories and files that are not rule files (e.g., .tmp files)
|
|
||||||
if !path.is_file() {
|
if !path.is_file() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// If the file is not in our set of expected files, remove it.
|
|
||||||
if !expected_files.contains(&path) {
|
if !expected_files.contains(&path) {
|
||||||
match fs::remove_file(&path) {
|
match fs::remove_file(&path) {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
println!("[CLEAN] Removed stale file {}", path.display());
|
info!(path = %path.display(), "Removed stale file");
|
||||||
cleaned_count += 1;
|
cleaned_count += 1;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!(
|
warn!(path = %path.display(), error = %e, "Failed to remove stale file");
|
||||||
"[WARN] Failed to remove stale file {}: {}",
|
|
||||||
path.display(),
|
|
||||||
e
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if cleaned_count == 0 {
|
|
||||||
println!("[CLEAN] No stale files found to clean up.");
|
if cleaned_count > 0 {
|
||||||
|
info!(count = cleaned_count, "Cleanup complete.");
|
||||||
} else {
|
} else {
|
||||||
println!("[CLEAN] Cleaned up {} stale files.", cleaned_count);
|
debug!("No stale files found.");
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Downloads a list of URLs concurrently, with a specified limit.
|
/// Downloads a list of URLs concurrently, with a specified limit.
|
||||||
|
#[instrument(skip_all, fields(count = urls.len(), concurrency))]
|
||||||
pub async fn download_all_rules(
|
pub async fn download_all_rules(
|
||||||
urls: &[String],
|
urls: &[String],
|
||||||
output_dir: &Path,
|
output_dir: &Path,
|
||||||
concurrency: usize,
|
concurrency: usize,
|
||||||
) -> Result<DownloadReport> {
|
) -> Result<DownloadReport> {
|
||||||
println!("\n▶️ Starting download process...");
|
info!("Starting download process...");
|
||||||
let client = reqwest::Client::new();
|
let client = reqwest::Client::new();
|
||||||
let mut successful = 0;
|
let mut successful = 0;
|
||||||
let mut failed = 0;
|
let mut failed = 0;
|
||||||
@ -79,11 +78,11 @@ pub async fn download_all_rules(
|
|||||||
async move {
|
async move {
|
||||||
match download_rule(&client, url, &output_path).await {
|
match download_rule(&client, url, &output_path).await {
|
||||||
Ok(path) => {
|
Ok(path) => {
|
||||||
println!("[OK] Saved {}", path.display());
|
debug!(path = %path.display(), "Rule saved successfully");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("[FAIL] Download error for {}: {:?}", url, e);
|
error!(url, error = ?e, "Download failed");
|
||||||
Err(())
|
Err(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -105,23 +104,20 @@ pub async fn download_all_rules(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Downloads a single file from a URL to a destination directory.
|
/// Downloads a single file from a URL to a destination directory.
|
||||||
/// Uses a temporary file to ensure atomic writes.
|
#[instrument(skip(client, output_dir), fields(url))]
|
||||||
async fn download_rule(client: &reqwest::Client, url: &str, output_dir: &Path) -> Result<PathBuf> {
|
async fn download_rule(client: &reqwest::Client, url: &str, output_dir: &Path) -> Result<PathBuf> {
|
||||||
let file_name = url_to_filename(url)?;
|
let file_name = url_to_filename(url)?;
|
||||||
println!("[DOWNLOAD] from {}", url);
|
debug!(file_name, "Downloading rule");
|
||||||
|
|
||||||
let final_path = output_dir.join(&file_name);
|
let final_path = output_dir.join(&file_name);
|
||||||
let tmp_path = output_dir.join(format!("{}.tmp", file_name));
|
let tmp_path = output_dir.join(format!("{}.tmp", file_name));
|
||||||
|
|
||||||
// Perform the download
|
|
||||||
let response = client.get(url).send().await?.error_for_status()?;
|
let response = client.get(url).send().await?.error_for_status()?;
|
||||||
let content = response.bytes().await?;
|
let content = response.bytes().await?;
|
||||||
|
|
||||||
// Write to a temporary file first
|
|
||||||
fs::write(&tmp_path, &content)
|
fs::write(&tmp_path, &content)
|
||||||
.with_context(|| format!("Failed to write temporary file {}", tmp_path.display()))?;
|
.with_context(|| format!("Failed to write temporary file {}", tmp_path.display()))?;
|
||||||
|
|
||||||
// Atomically rename the temporary file to the final destination
|
|
||||||
fs::rename(&tmp_path, &final_path)
|
fs::rename(&tmp_path, &final_path)
|
||||||
.with_context(|| format!("Failed to rename temp file to {}", final_path.display()))?;
|
.with_context(|| format!("Failed to rename temp file to {}", final_path.display()))?;
|
||||||
|
|
||||||
|
37
src/lib.rs
37
src/lib.rs
@ -1,5 +1,6 @@
|
|||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
use tracing::{info, instrument};
|
||||||
|
|
||||||
pub mod cli;
|
pub mod cli;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
@ -9,17 +10,18 @@ use cli::Args;
|
|||||||
use config::Config;
|
use config::Config;
|
||||||
|
|
||||||
/// The main application logic.
|
/// The main application logic.
|
||||||
|
#[instrument(skip_all, fields(
|
||||||
|
config_file = %args.input.display(),
|
||||||
|
output_dir = %args.output.display()
|
||||||
|
))]
|
||||||
pub async fn run(args: Args) -> Result<()> {
|
pub async fn run(args: Args) -> Result<()> {
|
||||||
println!("▶️ Starting ruleset processor...");
|
info!("Starting ruleset processor");
|
||||||
println!(" Config file: {}", args.input.display());
|
|
||||||
println!(" Output directory: {}", args.output.display());
|
|
||||||
println!(" Concurrency level: {}", args.concurrency);
|
|
||||||
|
|
||||||
// Load and parse the configuration file into strongly-typed structs.
|
// Load and parse the configuration file.
|
||||||
let config = Config::load(&args.input)
|
let config = Config::load(&args.input)
|
||||||
.with_context(|| format!("Failed to load config from {}", args.input.display()))?;
|
.with_context(|| format!("Failed to load config from {}", args.input.display()))?;
|
||||||
|
|
||||||
// Ensure the output directory exists before any operations.
|
// Ensure the output directory exists.
|
||||||
fs::create_dir_all(&args.output).with_context(|| {
|
fs::create_dir_all(&args.output).with_context(|| {
|
||||||
format!(
|
format!(
|
||||||
"Failed to create output directory '{}'",
|
"Failed to create output directory '{}'",
|
||||||
@ -27,33 +29,34 @@ pub async fn run(args: Args) -> Result<()> {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
// Determine the set of files that should exist based on the config.
|
// Determine the set of files that should exist.
|
||||||
let expected_files = config
|
let expected_files = config
|
||||||
.get_expected_files(&args.output)
|
.get_expected_files(&args.output)
|
||||||
.context("Failed to determine expected files from config")?;
|
.context("Failed to determine expected files from config")?;
|
||||||
|
|
||||||
// Clean up any files in the output directory that are not in our expected set.
|
// Clean up any stale files.
|
||||||
downloader::cleanup_stale_files(&args.output, &expected_files)?;
|
downloader::cleanup_stale_files(&args.output, &expected_files)?;
|
||||||
|
|
||||||
// Proceed to download files defined in the config.
|
// Proceed to download files defined in the config.
|
||||||
let urls_to_download = config.extract_urls();
|
let urls_to_download = config.extract_urls();
|
||||||
if urls_to_download.is_empty() {
|
if urls_to_download.is_empty() {
|
||||||
println!("\n✔️ No rule sets with URLs found. Process complete.");
|
info!("No rule sets with URLs found in config. Process complete.");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
println!(
|
|
||||||
"\n✔️ Found {} rule sets to download/update.",
|
info!(
|
||||||
urls_to_download.len()
|
count = urls_to_download.len(),
|
||||||
|
"Found rule sets to download/update."
|
||||||
);
|
);
|
||||||
|
|
||||||
// Download all files concurrently.
|
// Download all files concurrently.
|
||||||
let download_report =
|
let download_report =
|
||||||
downloader::download_all_rules(&urls_to_download, &args.output, args.concurrency).await?;
|
downloader::download_all_rules(&urls_to_download, &args.output, args.concurrency).await?;
|
||||||
|
|
||||||
println!("\n✅ Download process finished.");
|
info!(
|
||||||
println!(
|
successful = download_report.successful,
|
||||||
" {} successful, {} failed.",
|
failed = download_report.failed,
|
||||||
download_report.successful, download_report.failed
|
"Download process finished."
|
||||||
);
|
);
|
||||||
|
|
||||||
// If any downloads failed, abort with an error message.
|
// If any downloads failed, abort with an error message.
|
||||||
@ -64,6 +67,6 @@ pub async fn run(args: Args) -> Result<()> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
println!("\n✔️ Ruleset synchronization complete.");
|
info!("Ruleset synchronization completed successfully.");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
26
src/main.rs
26
src/main.rs
@ -1,15 +1,35 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use sb_ruleset_sync::cli::Args;
|
use sb_ruleset_sync::cli::Args;
|
||||||
|
use tracing::error;
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
// 1. Parse command-line arguments.
|
// 1. Parse command-line arguments first to get the verbosity level.
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
|
||||||
// 2. Execute the main application logic from the library.
|
// 2. Set up the tracing subscriber.
|
||||||
|
// This configures a logger that prints to the console.
|
||||||
|
// The verbosity is controlled by the `-v` flag.
|
||||||
|
let log_level = match args.verbose {
|
||||||
|
0 => "info", // Default level
|
||||||
|
1 => "debug", // -v
|
||||||
|
_ => "trace", // -vv or more
|
||||||
|
};
|
||||||
|
|
||||||
|
// We construct a filter that applies the log level to our crate,
|
||||||
|
// but keeps other crates at a higher level (e.g., info) to reduce noise.
|
||||||
|
let filter = format!("sb_ruleset_sync={}", log_level);
|
||||||
|
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(filter)
|
||||||
|
.with_writer(std::io::stderr) // Log to stderr, common for daemons/cron
|
||||||
|
.init();
|
||||||
|
|
||||||
|
// 3. Execute the main application logic.
|
||||||
if let Err(e) = sb_ruleset_sync::run(args).await {
|
if let Err(e) = sb_ruleset_sync::run(args).await {
|
||||||
eprintln!("\nError: {e:?}");
|
// Log the final error in a structured way before exiting.
|
||||||
|
error!(error = ?e, "Application failed");
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user