feat: migrate to tracing
This commit is contained in:
174
Cargo.lock
generated
174
Cargo.lock
generated
@ -17,6 +17,15 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.19"
|
||||
@ -695,6 +704,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.174"
|
||||
@ -729,6 +744,15 @@ version = "0.4.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
|
||||
dependencies = [
|
||||
"regex-automata 0.1.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.5"
|
||||
@ -778,6 +802,16 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.46.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
|
||||
dependencies = [
|
||||
"overload",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.36.7"
|
||||
@ -843,6 +877,12 @@ dependencies = [
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "overload"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.4"
|
||||
@ -932,6 +972,50 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata 0.4.9",
|
||||
"regex-syntax 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
|
||||
dependencies = [
|
||||
"regex-syntax 0.6.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.12.20"
|
||||
@ -1065,6 +1149,8 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1149,6 +1235,15 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sharded-slab"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
@ -1269,6 +1364,15 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.1"
|
||||
@ -1393,9 +1497,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-attributes"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.34"
|
||||
@ -1403,6 +1519,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-subscriber"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
|
||||
dependencies = [
|
||||
"matchers",
|
||||
"nu-ansi-term",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"sharded-slab",
|
||||
"smallvec",
|
||||
"thread_local",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1446,6 +1592,12 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
@ -1570,6 +1722,28 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.3"
|
||||
|
||||
@ -11,3 +11,5 @@ serde = { version = "1.0", features = ["derive"] }
|
||||
tokio = { version = "1.45", features = ["full"] }
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
reqwest = { version = "0.12", features = ["json", "stream", "blocking"] }
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
@ -15,4 +15,8 @@ pub struct Args {
|
||||
/// Number of concurrent download operations.
|
||||
#[arg(long, default_value_t = 10)]
|
||||
pub concurrency: usize,
|
||||
|
||||
/// Increase logging verbosity. Can be used multiple times (e.g., -v, -vv).
|
||||
#[arg(short, long, action = clap::ArgAction::Count)]
|
||||
pub verbose: u8,
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ use futures::{StreamExt, stream};
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
/// A report summarizing the outcome of the download process.
|
||||
#[derive(Debug)]
|
||||
@ -21,8 +22,9 @@ pub fn url_to_filename(url: &str) -> Result<String> {
|
||||
}
|
||||
|
||||
/// Scans the output directory and removes any files not in the expected set.
|
||||
#[instrument(skip(output_dir, expected_files))]
|
||||
pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>) -> Result<()> {
|
||||
println!("\n▶️ Cleaning up stale rule files...");
|
||||
info!("Checking for stale rule files...");
|
||||
let mut cleaned_count = 0;
|
||||
|
||||
for entry in fs::read_dir(output_dir)
|
||||
@ -31,43 +33,40 @@ pub fn cleanup_stale_files(output_dir: &Path, expected_files: &HashSet<PathBuf>)
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
// Ignore directories and files that are not rule files (e.g., .tmp files)
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If the file is not in our set of expected files, remove it.
|
||||
if !expected_files.contains(&path) {
|
||||
match fs::remove_file(&path) {
|
||||
Ok(_) => {
|
||||
println!("[CLEAN] Removed stale file {}", path.display());
|
||||
info!(path = %path.display(), "Removed stale file");
|
||||
cleaned_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"[WARN] Failed to remove stale file {}: {}",
|
||||
path.display(),
|
||||
e
|
||||
);
|
||||
warn!(path = %path.display(), error = %e, "Failed to remove stale file");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if cleaned_count == 0 {
|
||||
println!("[CLEAN] No stale files found to clean up.");
|
||||
|
||||
if cleaned_count > 0 {
|
||||
info!(count = cleaned_count, "Cleanup complete.");
|
||||
} else {
|
||||
println!("[CLEAN] Cleaned up {} stale files.", cleaned_count);
|
||||
debug!("No stale files found.");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Downloads a list of URLs concurrently, with a specified limit.
|
||||
#[instrument(skip_all, fields(count = urls.len(), concurrency))]
|
||||
pub async fn download_all_rules(
|
||||
urls: &[String],
|
||||
output_dir: &Path,
|
||||
concurrency: usize,
|
||||
) -> Result<DownloadReport> {
|
||||
println!("\n▶️ Starting download process...");
|
||||
info!("Starting download process...");
|
||||
let client = reqwest::Client::new();
|
||||
let mut successful = 0;
|
||||
let mut failed = 0;
|
||||
@ -79,11 +78,11 @@ pub async fn download_all_rules(
|
||||
async move {
|
||||
match download_rule(&client, url, &output_path).await {
|
||||
Ok(path) => {
|
||||
println!("[OK] Saved {}", path.display());
|
||||
debug!(path = %path.display(), "Rule saved successfully");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("[FAIL] Download error for {}: {:?}", url, e);
|
||||
error!(url, error = ?e, "Download failed");
|
||||
Err(())
|
||||
}
|
||||
}
|
||||
@ -105,23 +104,20 @@ pub async fn download_all_rules(
|
||||
}
|
||||
|
||||
/// Downloads a single file from a URL to a destination directory.
|
||||
/// Uses a temporary file to ensure atomic writes.
|
||||
#[instrument(skip(client, output_dir), fields(url))]
|
||||
async fn download_rule(client: &reqwest::Client, url: &str, output_dir: &Path) -> Result<PathBuf> {
|
||||
let file_name = url_to_filename(url)?;
|
||||
println!("[DOWNLOAD] from {}", url);
|
||||
debug!(file_name, "Downloading rule");
|
||||
|
||||
let final_path = output_dir.join(&file_name);
|
||||
let tmp_path = output_dir.join(format!("{}.tmp", file_name));
|
||||
|
||||
// Perform the download
|
||||
let response = client.get(url).send().await?.error_for_status()?;
|
||||
let content = response.bytes().await?;
|
||||
|
||||
// Write to a temporary file first
|
||||
fs::write(&tmp_path, &content)
|
||||
.with_context(|| format!("Failed to write temporary file {}", tmp_path.display()))?;
|
||||
|
||||
// Atomically rename the temporary file to the final destination
|
||||
fs::rename(&tmp_path, &final_path)
|
||||
.with_context(|| format!("Failed to rename temp file to {}", final_path.display()))?;
|
||||
|
||||
|
||||
37
src/lib.rs
37
src/lib.rs
@ -1,5 +1,6 @@
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::fs;
|
||||
use tracing::{info, instrument};
|
||||
|
||||
pub mod cli;
|
||||
pub mod config;
|
||||
@ -9,17 +10,18 @@ use cli::Args;
|
||||
use config::Config;
|
||||
|
||||
/// The main application logic.
|
||||
#[instrument(skip_all, fields(
|
||||
config_file = %args.input.display(),
|
||||
output_dir = %args.output.display()
|
||||
))]
|
||||
pub async fn run(args: Args) -> Result<()> {
|
||||
println!("▶️ Starting ruleset processor...");
|
||||
println!(" Config file: {}", args.input.display());
|
||||
println!(" Output directory: {}", args.output.display());
|
||||
println!(" Concurrency level: {}", args.concurrency);
|
||||
info!("Starting ruleset processor");
|
||||
|
||||
// Load and parse the configuration file into strongly-typed structs.
|
||||
// Load and parse the configuration file.
|
||||
let config = Config::load(&args.input)
|
||||
.with_context(|| format!("Failed to load config from {}", args.input.display()))?;
|
||||
|
||||
// Ensure the output directory exists before any operations.
|
||||
// Ensure the output directory exists.
|
||||
fs::create_dir_all(&args.output).with_context(|| {
|
||||
format!(
|
||||
"Failed to create output directory '{}'",
|
||||
@ -27,33 +29,34 @@ pub async fn run(args: Args) -> Result<()> {
|
||||
)
|
||||
})?;
|
||||
|
||||
// Determine the set of files that should exist based on the config.
|
||||
// Determine the set of files that should exist.
|
||||
let expected_files = config
|
||||
.get_expected_files(&args.output)
|
||||
.context("Failed to determine expected files from config")?;
|
||||
|
||||
// Clean up any files in the output directory that are not in our expected set.
|
||||
// Clean up any stale files.
|
||||
downloader::cleanup_stale_files(&args.output, &expected_files)?;
|
||||
|
||||
// Proceed to download files defined in the config.
|
||||
let urls_to_download = config.extract_urls();
|
||||
if urls_to_download.is_empty() {
|
||||
println!("\n✔️ No rule sets with URLs found. Process complete.");
|
||||
info!("No rule sets with URLs found in config. Process complete.");
|
||||
return Ok(());
|
||||
}
|
||||
println!(
|
||||
"\n✔️ Found {} rule sets to download/update.",
|
||||
urls_to_download.len()
|
||||
|
||||
info!(
|
||||
count = urls_to_download.len(),
|
||||
"Found rule sets to download/update."
|
||||
);
|
||||
|
||||
// Download all files concurrently.
|
||||
let download_report =
|
||||
downloader::download_all_rules(&urls_to_download, &args.output, args.concurrency).await?;
|
||||
|
||||
println!("\n✅ Download process finished.");
|
||||
println!(
|
||||
" {} successful, {} failed.",
|
||||
download_report.successful, download_report.failed
|
||||
info!(
|
||||
successful = download_report.successful,
|
||||
failed = download_report.failed,
|
||||
"Download process finished."
|
||||
);
|
||||
|
||||
// If any downloads failed, abort with an error message.
|
||||
@ -64,6 +67,6 @@ pub async fn run(args: Args) -> Result<()> {
|
||||
);
|
||||
}
|
||||
|
||||
println!("\n✔️ Ruleset synchronization complete.");
|
||||
info!("Ruleset synchronization completed successfully.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
26
src/main.rs
26
src/main.rs
@ -1,15 +1,35 @@
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
use sb_ruleset_sync::cli::Args;
|
||||
use tracing::error;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// 1. Parse command-line arguments.
|
||||
// 1. Parse command-line arguments first to get the verbosity level.
|
||||
let args = Args::parse();
|
||||
|
||||
// 2. Execute the main application logic from the library.
|
||||
// 2. Set up the tracing subscriber.
|
||||
// This configures a logger that prints to the console.
|
||||
// The verbosity is controlled by the `-v` flag.
|
||||
let log_level = match args.verbose {
|
||||
0 => "info", // Default level
|
||||
1 => "debug", // -v
|
||||
_ => "trace", // -vv or more
|
||||
};
|
||||
|
||||
// We construct a filter that applies the log level to our crate,
|
||||
// but keeps other crates at a higher level (e.g., info) to reduce noise.
|
||||
let filter = format!("sb_ruleset_sync={}", log_level);
|
||||
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(filter)
|
||||
.with_writer(std::io::stderr) // Log to stderr, common for daemons/cron
|
||||
.init();
|
||||
|
||||
// 3. Execute the main application logic.
|
||||
if let Err(e) = sb_ruleset_sync::run(args).await {
|
||||
eprintln!("\nError: {e:?}");
|
||||
// Log the final error in a structured way before exiting.
|
||||
error!(error = ?e, "Application failed");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user