feat: migrate to tracing

This commit is contained in:
2025-06-29 11:07:06 +03:00
parent 672e2fff59
commit f067da9835
6 changed files with 239 additions and 40 deletions

174
Cargo.lock generated
View File

@ -17,6 +17,15 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.6.19" version = "0.6.19"
@ -695,6 +704,12 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "lazy_static"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.174" version = "0.2.174"
@ -729,6 +744,15 @@ version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "matchers"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
dependencies = [
"regex-automata 0.1.10",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.5" version = "2.7.5"
@ -778,6 +802,16 @@ dependencies = [
"tempfile", "tempfile",
] ]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
dependencies = [
"overload",
"winapi",
]
[[package]] [[package]]
name = "object" name = "object"
version = "0.36.7" version = "0.36.7"
@ -843,6 +877,12 @@ dependencies = [
"vcpkg", "vcpkg",
] ]
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.4" version = "0.12.4"
@ -932,6 +972,50 @@ dependencies = [
"bitflags", "bitflags",
] ]
[[package]]
name = "regex"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.4.9",
"regex-syntax 0.8.5",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
"regex-syntax 0.6.29",
]
[[package]]
name = "regex-automata"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.5",
]
[[package]]
name = "regex-syntax"
version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]] [[package]]
name = "reqwest" name = "reqwest"
version = "0.12.20" version = "0.12.20"
@ -1065,6 +1149,8 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"tokio", "tokio",
"tracing",
"tracing-subscriber",
] ]
[[package]] [[package]]
@ -1149,6 +1235,15 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]] [[package]]
name = "shlex" name = "shlex"
version = "1.3.0" version = "1.3.0"
@ -1269,6 +1364,15 @@ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.59.0",
] ]
[[package]]
name = "thread_local"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "tinystr" name = "tinystr"
version = "0.8.1" version = "0.8.1"
@ -1393,9 +1497,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
dependencies = [ dependencies = [
"pin-project-lite", "pin-project-lite",
"tracing-attributes",
"tracing-core", "tracing-core",
] ]
[[package]]
name = "tracing-attributes"
version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "tracing-core" name = "tracing-core"
version = "0.1.34" version = "0.1.34"
@ -1403,6 +1519,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"valuable",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
dependencies = [
"matchers",
"nu-ansi-term",
"once_cell",
"regex",
"sharded-slab",
"smallvec",
"thread_local",
"tracing",
"tracing-core",
"tracing-log",
] ]
[[package]] [[package]]
@ -1446,6 +1592,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "valuable"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]] [[package]]
name = "vcpkg" name = "vcpkg"
version = "0.2.15" version = "0.2.15"
@ -1570,6 +1722,28 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]] [[package]]
name = "windows-link" name = "windows-link"
version = "0.1.3" version = "0.1.3"

View File

@ -11,3 +11,5 @@ serde = { version = "1.0", features = ["derive"] }
tokio = { version = "1.45", features = ["full"] } tokio = { version = "1.45", features = ["full"] }
clap = { version = "4.5", features = ["derive"] } clap = { version = "4.5", features = ["derive"] }
reqwest = { version = "0.12", features = ["json", "stream", "blocking"] } reqwest = { version = "0.12", features = ["json", "stream", "blocking"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }

View File

@ -15,4 +15,8 @@ pub struct Args {
/// Number of concurrent download operations. /// Number of concurrent download operations.
#[arg(long, default_value_t = 10)] #[arg(long, default_value_t = 10)]
pub concurrency: usize, pub concurrency: usize,
/// Increase logging verbosity. Can be used multiple times (e.g., -v, -vv).
#[arg(short, long, action = clap::ArgAction::Count)]
pub verbose: u8,
} }

View File

@ -3,6 +3,7 @@ use futures::{StreamExt, stream};
use std::collections::HashSet; use std::collections::HashSet;
use std::fs; use std::fs;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use tracing::{debug, error, info, instrument, warn};
/// A report summarizing the outcome of the download process. /// A report summarizing the outcome of the download process.
#[derive(Debug)] #[derive(Debug)]
@ -21,8 +22,9 @@ pub fn url_to_filename(url: &str) -> Result<String> {
} }
/// Scans the output directory and removes any files not in the expected set. /// Scans the output directory and removes any files not in the expected set.
#[instrument(skip(output_dir, expected_files))]
pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>) -> Result<()> { pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>) -> Result<()> {
println!("\n▶️ Cleaning up stale rule files..."); info!("Checking for stale rule files...");
let mut cleaned_count = 0; let mut cleaned_count = 0;
for entry in fs::read_dir(output_dir) for entry in fs::read_dir(output_dir)
@ -31,43 +33,40 @@ pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>)
let entry = entry?; let entry = entry?;
let path = entry.path(); let path = entry.path();
// Ignore directories and files that are not rule files (e.g., .tmp files)
if !path.is_file() { if !path.is_file() {
continue; continue;
} }
// If the file is not in our set of expected files, remove it.
if !expected_files.contains(&path) { if !expected_files.contains(&path) {
match fs::remove_file(&path) { match fs::remove_file(&path) {
Ok(_) => { Ok(_) => {
println!("[CLEAN] Removed stale file {}", path.display()); info!(path = %path.display(), "Removed stale file");
cleaned_count += 1; cleaned_count += 1;
} }
Err(e) => { Err(e) => {
eprintln!( warn!(path = %path.display(), error = %e, "Failed to remove stale file");
"[WARN] Failed to remove stale file {}: {}",
path.display(),
e
);
} }
} }
} }
} }
if cleaned_count == 0 {
println!("[CLEAN] No stale files found to clean up."); if cleaned_count > 0 {
info!(count = cleaned_count, "Cleanup complete.");
} else { } else {
println!("[CLEAN] Cleaned up {} stale files.", cleaned_count); debug!("No stale files found.");
} }
Ok(()) Ok(())
} }
/// Downloads a list of URLs concurrently, with a specified limit. /// Downloads a list of URLs concurrently, with a specified limit.
#[instrument(skip_all, fields(count = urls.len(), concurrency))]
pub async fn download_all_rules( pub async fn download_all_rules(
urls: &[String], urls: &[String],
output_dir: &Path, output_dir: &Path,
concurrency: usize, concurrency: usize,
) -> Result<DownloadReport> { ) -> Result<DownloadReport> {
println!("\n▶️ Starting download process..."); info!("Starting download process...");
let client = reqwest::Client::new(); let client = reqwest::Client::new();
let mut successful = 0; let mut successful = 0;
let mut failed = 0; let mut failed = 0;
@ -79,11 +78,11 @@ pub async fn download_all_rules(
async move { async move {
match download_rule(&client, url, &output_path).await { match download_rule(&client, url, &output_path).await {
Ok(path) => { Ok(path) => {
println!("[OK] Saved {}", path.display()); debug!(path = %path.display(), "Rule saved successfully");
Ok(()) Ok(())
} }
Err(e) => { Err(e) => {
eprintln!("[FAIL] Download error for {}: {:?}", url, e); error!(url, error = ?e, "Download failed");
Err(()) Err(())
} }
} }
@ -105,23 +104,20 @@ pub async fn download_all_rules(
} }
/// Downloads a single file from a URL to a destination directory. /// Downloads a single file from a URL to a destination directory.
/// Uses a temporary file to ensure atomic writes. #[instrument(skip(client, output_dir), fields(url))]
async fn download_rule(client: &reqwest::Client, url: &str, output_dir: &Path) -> Result<PathBuf> { async fn download_rule(client: &reqwest::Client, url: &str, output_dir: &Path) -> Result<PathBuf> {
let file_name = url_to_filename(url)?; let file_name = url_to_filename(url)?;
println!("[DOWNLOAD] from {}", url); debug!(file_name, "Downloading rule");
let final_path = output_dir.join(&file_name); let final_path = output_dir.join(&file_name);
let tmp_path = output_dir.join(format!("{}.tmp", file_name)); let tmp_path = output_dir.join(format!("{}.tmp", file_name));
// Perform the download
let response = client.get(url).send().await?.error_for_status()?; let response = client.get(url).send().await?.error_for_status()?;
let content = response.bytes().await?; let content = response.bytes().await?;
// Write to a temporary file first
fs::write(&tmp_path, &content) fs::write(&tmp_path, &content)
.with_context(|| format!("Failed to write temporary file {}", tmp_path.display()))?; .with_context(|| format!("Failed to write temporary file {}", tmp_path.display()))?;
// Atomically rename the temporary file to the final destination
fs::rename(&tmp_path, &final_path) fs::rename(&tmp_path, &final_path)
.with_context(|| format!("Failed to rename temp file to {}", final_path.display()))?; .with_context(|| format!("Failed to rename temp file to {}", final_path.display()))?;

View File

@ -1,5 +1,6 @@
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use std::fs; use std::fs;
use tracing::{info, instrument};
pub mod cli; pub mod cli;
pub mod config; pub mod config;
@ -9,17 +10,18 @@ use cli::Args;
use config::Config; use config::Config;
/// The main application logic. /// The main application logic.
#[instrument(skip_all, fields(
config_file = %args.input.display(),
output_dir = %args.output.display()
))]
pub async fn run(args: Args) -> Result<()> { pub async fn run(args: Args) -> Result<()> {
println!("▶️ Starting ruleset processor..."); info!("Starting ruleset processor");
println!(" Config file: {}", args.input.display());
println!(" Output directory: {}", args.output.display());
println!(" Concurrency level: {}", args.concurrency);
// Load and parse the configuration file into strongly-typed structs. // Load and parse the configuration file.
let config = Config::load(&args.input) let config = Config::load(&args.input)
.with_context(|| format!("Failed to load config from {}", args.input.display()))?; .with_context(|| format!("Failed to load config from {}", args.input.display()))?;
// Ensure the output directory exists before any operations. // Ensure the output directory exists.
fs::create_dir_all(&args.output).with_context(|| { fs::create_dir_all(&args.output).with_context(|| {
format!( format!(
"Failed to create output directory '{}'", "Failed to create output directory '{}'",
@ -27,33 +29,34 @@ pub async fn run(args: Args) -> Result<()> {
) )
})?; })?;
// Determine the set of files that should exist based on the config. // Determine the set of files that should exist.
let expected_files = config let expected_files = config
.get_expected_files(&args.output) .get_expected_files(&args.output)
.context("Failed to determine expected files from config")?; .context("Failed to determine expected files from config")?;
// Clean up any files in the output directory that are not in our expected set. // Clean up any stale files.
downloader::cleanup_stale_files(&args.output, &expected_files)?; downloader::cleanup_stale_files(&args.output, &expected_files)?;
// Proceed to download files defined in the config. // Proceed to download files defined in the config.
let urls_to_download = config.extract_urls(); let urls_to_download = config.extract_urls();
if urls_to_download.is_empty() { if urls_to_download.is_empty() {
println!("\n✔️ No rule sets with URLs found. Process complete."); info!("No rule sets with URLs found in config. Process complete.");
return Ok(()); return Ok(());
} }
println!(
"\n✔️ Found {} rule sets to download/update.", info!(
urls_to_download.len() count = urls_to_download.len(),
"Found rule sets to download/update."
); );
// Download all files concurrently. // Download all files concurrently.
let download_report = let download_report =
downloader::download_all_rules(&urls_to_download, &args.output, args.concurrency).await?; downloader::download_all_rules(&urls_to_download, &args.output, args.concurrency).await?;
println!("\n✅ Download process finished."); info!(
println!( successful = download_report.successful,
" {} successful, {} failed.", failed = download_report.failed,
download_report.successful, download_report.failed "Download process finished."
); );
// If any downloads failed, abort with an error message. // If any downloads failed, abort with an error message.
@ -64,6 +67,6 @@ pub async fn run(args: Args) -> Result<()> {
); );
} }
println!("\n✔️ Ruleset synchronization complete."); info!("Ruleset synchronization completed successfully.");
Ok(()) Ok(())
} }

View File

@ -1,15 +1,35 @@
use anyhow::Result; use anyhow::Result;
use clap::Parser; use clap::Parser;
use sb_ruleset_sync::cli::Args; use sb_ruleset_sync::cli::Args;
use tracing::error;
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
// 1. Parse command-line arguments. // 1. Parse command-line arguments first to get the verbosity level.
let args = Args::parse(); let args = Args::parse();
// 2. Execute the main application logic from the library. // 2. Set up the tracing subscriber.
// This configures a logger that prints to the console.
// The verbosity is controlled by the `-v` flag.
let log_level = match args.verbose {
0 => "info", // Default level
1 => "debug", // -v
_ => "trace", // -vv or more
};
// We construct a filter that applies the log level to our crate,
// but keeps other crates at a higher level (e.g., info) to reduce noise.
let filter = format!("sb_ruleset_sync={}", log_level);
tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(std::io::stderr) // Log to stderr, common for daemons/cron
.init();
// 3. Execute the main application logic.
if let Err(e) = sb_ruleset_sync::run(args).await { if let Err(e) = sb_ruleset_sync::run(args).await {
eprintln!("\nError: {e:?}"); // Log the final error in a structured way before exiting.
error!(error = ?e, "Application failed");
std::process::exit(1); std::process::exit(1);
} }