hak8or/ebay_scraper_rust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ItemAppearances to track price and page history
All checks were successful
Cargo Build & Test / Rust project - latest (1.88) (push) Successful in 3m30s
Details
Cargo Build & Test / Rust project - latest (1.87) (push) Successful in 3m57s
Details
Cargo Build & Test / Rust project - latest (1.85.1) (push) Successful in 4m9s
Details
Cargo Build & Test / Rust project - latest (1.86) (push) Successful in 9m50s
Details

Browse Source
This commit is contained in:
hak8or
2025-06-28 01:00:28 -04:00
parent b9cc62e3dd
commit 817b1d6275
3 changed files with 178 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
src/db.rs
View File

@@ -192,13 +192,81 @@ impl ParsedStorage {
}
}
#[derive(Serialize, Debug, PartialEq, Clone)]
pub struct ItemAppearances {
pub item: i64,
pub timestamp: DateTime<Utc>,
pub category: String,
pub current_bid_usd_cents: Option<i64>,
}
impl DBTable for ItemAppearances {
const TABLE_NAME: &'static str = "Item_Appearances";
const TABLE_SCHEMA: &'static str = "
id INTEGER PRIMARY KEY,
item INTEGER NOT NULL,
category TEXT NOT NULL,
timestamp INTEGER NOT NULL,
current_bid_usd_cents INTEGER,
UNIQUE(item, timestamp),
FOREIGN KEY(item) REFERENCES Ebay_Items(item_id),
FOREIGN KEY(category, timestamp) REFERENCES Pages_Parsed(category, timestamp)
";
}
impl ItemAppearances {
pub fn add_or_update(&self, conn: &Connection) {
let count = conn
.execute(
&format!(
"
INSERT OR REPLACE INTO {}
(item, timestamp, category, current_bid_usd_cents)
VALUES
(?1, ?2, ?3, ?4)",
Self::TABLE_NAME
),
(
self.item,
&self.timestamp,
&self.category,
self.current_bid_usd_cents,
),
)
.unwrap();
if count != 1 {
panic!("Expected count to be 1 but got {}", count);
}
}
pub fn lookup(conn: &Connection, listing_id: i64) -> Vec<ItemAppearances> {
let mut stmt = conn
.prepare(&format!(
"
SELECT * FROM {}
WHERE item IS ?1",
Self::TABLE_NAME,
))
.ok()
.unwrap();
stmt.query_map([listing_id], |row| {
Ok(ItemAppearances {
item: row.get(1)?,
category: row.get(2)?,
timestamp: row.get(3)?,
current_bid_usd_cents: row.get(4)?,
})
})
.ok()
.unwrap()
.map(|e| e.unwrap())
.collect()
}
}
#[derive(Serialize, Debug, PartialEq, Clone)]
pub struct Listing {
pub id: i64,
pub item_id: i64,
pub title: String,
pub added_time: DateTime<Utc>,
pub current_bid_price: Option<f64>,
pub buy_it_now_price: Option<f64>,
pub has_best_offer: bool,
pub image_url: String,
@@ -209,8 +277,6 @@ impl DBTable for Listing {
id INTEGER PRIMARY KEY,
item_id INTEGER NOT NULL UNIQUE,
title TEXT NOT NULL,
added_time INTEGER NOT NULL,
current_bid_usd_cents INTEGER,
buy_it_now_usd_cents INTEGER,
has_best_offer INTEGER NOT NULL,
image_url TEXT NOT NULL
@@ -229,39 +295,43 @@ impl Listing {
id: row.get(0)?,
item_id: row.get(1)?,
title: row.get(2)?,
added_time: row.get(3)?,
current_bid_price: row.get(4)?,
buy_it_now_price: row.get(5)?,
has_best_offer: row.get(6)?,
image_url: row.get(7)?,
buy_it_now_price: row.get(3)?,
has_best_offer: row.get(4)?,
image_url: row.get(5)?,
})
})
.ok()
}
pub fn lookup_since(conn: &Connection, since: i64, limit: i64) -> Vec<Self> {
pub fn lookup_since(conn: &Connection, since: DateTime<Utc>, limit: i64) -> Vec<Self> {
let mut stmt = conn
.prepare(&format!(
"SELECT * FROM {}
WHERE added_time >= ?1
ORDER BY added_time
"
SELECT *
FROM {0}
WHERE EXISTS (
SELECT 1
FROM {1}
WHERE
{1}.item = {0}.item_id AND
{1}.timestamp >= ?1
)
LIMIT ?2
",
Self::TABLE_NAME
Self::TABLE_NAME,
ItemAppearances::TABLE_NAME
))
.ok()
.unwrap();
stmt.query_map([since, limit], |row| {
stmt.query_map([since.timestamp(), limit], |row| {
Ok(Listing {
id: row.get(0)?,
item_id: row.get(1)?,
title: row.get(2)?,
added_time: row.get(3)?,
current_bid_price: row.get(4)?,
buy_it_now_price: row.get(5)?,
has_best_offer: row.get(6)?,
image_url: row.get(7)?,
buy_it_now_price: row.get(3)?,
has_best_offer: row.get(4)?,
image_url: row.get(5)?,
})
})
.ok()
@@ -297,20 +367,16 @@ impl Listing {
(
item_id,
title,
added_time,
current_bid_usd_cents,
buy_it_now_usd_cents,
has_best_offer,
image_url
)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
VALUES (?1, ?2, ?3, ?4, ?5)",
Self::TABLE_NAME
),
(
self.item_id,
&self.title,
self.added_time,
self.current_bid_price,
self.buy_it_now_price,
self.has_best_offer,
self.image_url.clone(),
@@ -334,6 +400,7 @@ pub fn get_initialized(path: Option<&Path>) -> Connection {
Listing::initialize(&conn);
ParsedStorage::initialize(&conn);
ParsedPage::initialize(&conn);
ItemAppearances::initialize(&conn);
conn
}
@@ -357,8 +424,6 @@ mod tests {
id: 1,
item_id: 1234,
title: "Some Title".to_string(),
added_time: std::time::SystemTime::now().into(),
current_bid_price: Some(0.12),
buy_it_now_price: Some(1.23),
has_best_offer: false,
image_url: "google.com".to_string(),
@@ -383,6 +448,23 @@ mod tests {
timestamp: std::time::SystemTime::now().into(),
};
page.add_or_update_db(&db);
assert_eq!(ParsedPage::lookup_db(&db, page.timestamp), Some(page));
assert_eq!(
ParsedPage::lookup_db(&db, page.timestamp),
Some(page.clone())
);
let apperance = ItemAppearances {
item: listing.item_id,
timestamp: page.timestamp,
category: page.category,
current_bid_usd_cents: Some(1233),
};
apperance.add_or_update(&db);
assert_eq!(
ItemAppearances::lookup(&db, listing.item_id),
vec![apperance]
);
assert_eq!(Listing::lookup_since(&db, page.timestamp, 3), vec![listing]);
}
}