Fix tracing logging to use only tracing and get instrument/spans going
This commit is contained in:
43
Cargo.lock
generated
43
Cargo.lock
generated
@@ -654,9 +654,7 @@ dependencies = [
|
|||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"dirs",
|
"dirs",
|
||||||
"env_logger",
|
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"log",
|
|
||||||
"regex",
|
"regex",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"scraper",
|
"scraper",
|
||||||
@@ -696,7 +694,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
|
checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"log",
|
"log",
|
||||||
"regex",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -708,7 +705,6 @@ dependencies = [
|
|||||||
"anstream",
|
"anstream",
|
||||||
"anstyle",
|
"anstyle",
|
||||||
"env_filter",
|
"env_filter",
|
||||||
"jiff",
|
|
||||||
"log",
|
"log",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1095,30 +1091,6 @@ version = "1.0.15"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "jiff"
|
|
||||||
version = "0.2.15"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
|
|
||||||
dependencies = [
|
|
||||||
"jiff-static",
|
|
||||||
"log",
|
|
||||||
"portable-atomic",
|
|
||||||
"portable-atomic-util",
|
|
||||||
"serde",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "jiff-static"
|
|
||||||
version = "0.2.15"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "jobserver"
|
name = "jobserver"
|
||||||
version = "0.1.33"
|
version = "0.1.33"
|
||||||
@@ -1453,21 +1425,6 @@ version = "0.3.32"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "portable-atomic"
|
|
||||||
version = "1.11.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "portable-atomic-util"
|
|
||||||
version = "0.2.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
|
|
||||||
dependencies = [
|
|
||||||
"portable-atomic",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "potential_utf"
|
name = "potential_utf"
|
||||||
version = "0.1.2"
|
version = "0.1.2"
|
||||||
|
@@ -8,17 +8,15 @@ actix-web = "4.11.0"
|
|||||||
chrono = { version = "0.4.41", features = ["serde"] }
|
chrono = { version = "0.4.41", features = ["serde"] }
|
||||||
clap = { version = "4.5.40", features = ["derive"] }
|
clap = { version = "4.5.40", features = ["derive"] }
|
||||||
dirs = "6.0.0"
|
dirs = "6.0.0"
|
||||||
env_logger = "0.11.8"
|
|
||||||
lazy_static = "1.5.0"
|
lazy_static = "1.5.0"
|
||||||
log = "0.4.27"
|
|
||||||
regex = "1.11.1"
|
regex = "1.11.1"
|
||||||
rusqlite = { version = "0.36.0", features = ["bundled", "chrono"] }
|
rusqlite = { version = "0.36.0", features = ["bundled", "chrono"] }
|
||||||
scraper = "0.23.1"
|
scraper = "0.23.1"
|
||||||
serde = { version = "1.0.219", features = ["derive"] }
|
serde = { version = "1.0.219", features = ["derive"] }
|
||||||
serde_json = "1.0.140"
|
serde_json = "1.0.140"
|
||||||
test-log = { version = "0.2.17", features = ["trace"] }
|
test-log = { version = "0.2.17", features = ["trace"] }
|
||||||
tracing = "0.1.41"
|
tracing = { version = "0.1.41", features = ["attributes"] }
|
||||||
tracing-subscriber = "0.3.19"
|
tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
similar-asserts = "1.7.0"
|
similar-asserts = "1.7.0"
|
||||||
|
30
src/main.rs
30
src/main.rs
@@ -7,7 +7,13 @@ use ebay_scraper_rust::db::{
|
|||||||
use ebay_scraper_rust::{parser_ebay, parser_storage};
|
use ebay_scraper_rust::{parser_ebay, parser_storage};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
use tracing::info;
|
use std::time::Instant;
|
||||||
|
use tracing::{error, info, instrument};
|
||||||
|
|
||||||
|
use tracing_subscriber::filter::EnvFilter;
|
||||||
|
use tracing_subscriber::fmt;
|
||||||
|
use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
|
||||||
|
use tracing_subscriber::util::SubscriberInitExt;
|
||||||
|
|
||||||
mod xdg_dirs;
|
mod xdg_dirs;
|
||||||
|
|
||||||
@@ -94,7 +100,6 @@ pub fn timestamps_from_dir(path: &Path) -> Vec<i64> {
|
|||||||
|
|
||||||
std::fs::read_dir(path)
|
std::fs::read_dir(path)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.inspect(|fpath| info!("Found {:?}", fpath))
|
|
||||||
.map(|fpath| fpath.unwrap().path())
|
.map(|fpath| fpath.unwrap().path())
|
||||||
.filter_map(|fstem| {
|
.filter_map(|fstem| {
|
||||||
fstem
|
fstem
|
||||||
@@ -108,6 +113,7 @@ pub fn timestamps_from_dir(path: &Path) -> Vec<i64> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[post("page/parse/{category}")]
|
#[post("page/parse/{category}")]
|
||||||
|
#[instrument(skip_all)]
|
||||||
async fn parse_post(
|
async fn parse_post(
|
||||||
db: Data<Mutex<rusqlite::Connection>>,
|
db: Data<Mutex<rusqlite::Connection>>,
|
||||||
downloaddir: Data<PathBuf>,
|
downloaddir: Data<PathBuf>,
|
||||||
@@ -135,21 +141,33 @@ async fn parse_post(
|
|||||||
|
|
||||||
// Timestamp never seen before, lets pass it on.
|
// Timestamp never seen before, lets pass it on.
|
||||||
if p.is_none() {
|
if p.is_none() {
|
||||||
|
info!(
|
||||||
|
"Page of timestamp:{} and catagory:{category} never seen before, processing ...",
|
||||||
|
ts.timestamp()
|
||||||
|
);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Timestamp was seen before *and* from the same catagory, don't pass
|
// Timestamp was seen before *and* from the same catagory, don't pass
|
||||||
// it on.
|
// it on.
|
||||||
if p.unwrap().category == *category {
|
if p.unwrap().category == *category {
|
||||||
|
info!(
|
||||||
|
"Page of timestamp:{} and catagory:{category} seen before, skipping ...",
|
||||||
|
ts.timestamp()
|
||||||
|
);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"Page of timestamp:{} seen before, but not of catagory:{category}, processing ...",
|
||||||
|
ts.timestamp()
|
||||||
|
);
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut added_count = 0;
|
let mut added_count = 0;
|
||||||
for p in to_parse {
|
for p in to_parse {
|
||||||
let ts = chrono::DateTime::from_timestamp(*p, 0).unwrap();
|
let ts = chrono::DateTime::from_timestamp(*p, 0).unwrap();
|
||||||
info!("Adding page with a timestamp of {ts} and catagory of {category} to db");
|
|
||||||
ParsedPage {
|
ParsedPage {
|
||||||
timestamp: ts,
|
timestamp: ts,
|
||||||
category: category.to_string(),
|
category: category.to_string(),
|
||||||
@@ -171,12 +189,16 @@ async fn parse_post(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
info!("Added {added_count} listings");
|
||||||
Ok(added_count.to_string())
|
Ok(added_count.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[actix_web::main]
|
#[actix_web::main]
|
||||||
async fn main() -> std::io::Result<()> {
|
async fn main() -> std::io::Result<()> {
|
||||||
env_logger::init_from_env(env_logger::Env::new().default_filter_or("info"));
|
tracing_subscriber::registry()
|
||||||
|
.with(fmt::layer())
|
||||||
|
.with(EnvFilter::from_default_env())
|
||||||
|
.init();
|
||||||
let _ = Args::parse();
|
let _ = Args::parse();
|
||||||
|
|
||||||
let scrapedatadir = xdg_dirs::ensure_scrapedata_dir_exists("ebay_scraper", None);
|
let scrapedatadir = xdg_dirs::ensure_scrapedata_dir_exists("ebay_scraper", None);
|
||||||
|
Reference in New Issue
Block a user