use chrono::{DateTime, Utc}; use rusqlite::Connection; use serde::Serialize; use std::path::Path; use tracing::info; trait DBTable { const TABLE_NAME: &'static str; const TABLE_SCHEMA: &'static str; fn initialize(conn: &Connection) { let create_table = &format!( "CREATE TABLE IF NOT EXISTS {} ( {} )", Self::TABLE_NAME, Self::TABLE_SCHEMA ); info!("Creating table with following schema;"); info!("{} ({})", Self::TABLE_NAME, Self::TABLE_SCHEMA); conn.execute(create_table, ()).unwrap(); } } #[derive(Serialize, Debug, PartialEq, Clone)] pub struct SearchURL { pub full_url: String, pub name: String, } impl DBTable for SearchURL { const TABLE_NAME: &'static str = "SearchURLs"; const TABLE_SCHEMA: &'static str = " id INTEGER PRIMARY KEY, url TEXT NOT NULL UNIQUE, name TEXT NOT NULL UNIQUE"; } impl SearchURL { pub fn lookup(conn: &Connection, name: &str) -> Option { let mut stmt = conn .prepare(&format!( "SELECT * FROM {} WHERE name = ?", Self::TABLE_NAME )) .ok()?; stmt.query_one([name], |row| { Ok(SearchURL { // id: row.get(0)?, full_url: row.get(1)?, name: row.get(2)?, }) }) .ok() } pub fn add_or_update(&self, conn: &Connection) { let _ = conn .execute( &format!( "INSERT OR REPLACE INTO {} (name, url) VALUES (?1, ?2)", Self::TABLE_NAME ), (&self.name, &self.full_url), ) .unwrap(); } pub fn names(conn: &Connection) -> Vec { let mut stmt = conn .prepare(&format!("SELECT name FROM {}", Self::TABLE_NAME)) .ok() .unwrap(); stmt.query_map([], |row| Ok(row.get(0))) .ok() .unwrap() .map(|e| e.unwrap()) .flatten() .collect() } } #[derive(Serialize, Debug, PartialEq, Clone)] pub struct ParsedPage { pub timestamp: DateTime, pub category: String, } impl DBTable for ParsedPage { const TABLE_NAME: &'static str = "Pages_Parsed"; const TABLE_SCHEMA: &'static str = " id INTEGER PRIMARY KEY, category TEXT NOT NULL, timestamp INTEGER NOT NULL, UNIQUE(category, timestamp) FOREIGN KEY(category) REFERENCES SearchURLs(name) "; } impl ParsedPage { pub fn lookup_db(conn: &Connection, timestamp: DateTime) -> Option { let mut stmt = conn .prepare(&format!( "SELECT * FROM {} WHERE timestamp = ?", Self::TABLE_NAME )) .ok()?; stmt.query_one([timestamp], |row| { Ok(ParsedPage { // id: row.get(0)?, category: row.get(1)?, timestamp: row.get(2)?, }) }) .ok() } pub fn add_or_update_db(&self, conn: &Connection) { let _ = conn .execute( &format!( "INSERT OR REPLACE INTO {} (category, timestamp) VALUES (?1, ?2)", Self::TABLE_NAME ), (&self.category, self.timestamp), ) .unwrap(); } } #[derive(Serialize, Debug, PartialEq, Copy, Clone)] pub struct ParsedStorage { pub id: i64, pub item: i64, pub total_gigabytes: i64, pub quantity: i64, pub individual_size_gigabytes: i64, pub parse_engine: i64, pub needed_description_check: bool, } impl DBTable for ParsedStorage { const TABLE_NAME: &'static str = "Storage_Parsed"; const TABLE_SCHEMA: &'static str = " id INTEGER PRIMARY KEY, item INTEGER, total_gigabytes INTEGER, quantity INTEGER, sizes_gigabytes TEXT, parse_engine INTEGER, need_description_check INTEGER, UNIQUE(item, parse_engine) FOREIGN KEY(item) REFERENCES Ebay_Items(item_id) "; } impl ParsedStorage { pub fn lookup_db(conn: &Connection, item: i64) -> Vec { let mut stmt = conn .prepare(&format!("SELECT * FROM {} WHERE id = ?", Self::TABLE_NAME)) .ok() .unwrap(); stmt.query_map([item], |row| { Ok(ParsedStorage { id: row.get(0)?, item: row.get(1)?, total_gigabytes: row.get(2)?, quantity: row.get(3)?, individual_size_gigabytes: { let r: String = row.get(4)?; r.parse().unwrap() }, parse_engine: row.get(5)?, needed_description_check: row.get(6)?, }) }) .ok() .unwrap() .map(|e| e.unwrap()) .collect() } pub fn add_or_update_db(&self, conn: &Connection) { let _ = conn.execute(&format!(" INSERT OR REPLACE INTO {} (item, total_gigabytes, quantity, sizes_gigabytes, parse_engine, need_description_check) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", Self::TABLE_NAME), ( &self.item, self.total_gigabytes, self.quantity, self.individual_size_gigabytes.to_string(), self.parse_engine, self.needed_description_check ) ).unwrap(); } } #[derive(Serialize, Debug, PartialEq, Clone)] pub struct Listing { pub id: i64, pub item_id: i64, pub title: String, pub added_time: DateTime, pub current_bid_price: Option, pub buy_it_now_price: Option, pub has_best_offer: bool, pub image_url: String, } impl DBTable for Listing { const TABLE_NAME: &'static str = "Ebay_Items"; const TABLE_SCHEMA: &'static str = " id INTEGER PRIMARY KEY, item_id INTEGER NOT NULL UNIQUE, title TEXT NOT NULL, added_time INTEGER NOT NULL, current_bid_usd_cents INTEGER, buy_it_now_usd_cents INTEGER, has_best_offer INTEGER NOT NULL, image_url TEXT NOT NULL "; } impl Listing { pub fn lookup(conn: &Connection, item_id: i64) -> Option { let mut stmt = conn .prepare(&format!( "SELECT * FROM {} WHERE item_id = ?", Self::TABLE_NAME )) .ok()?; stmt.query_one([item_id], |row| { Ok(Listing { id: row.get(0)?, item_id: row.get(1)?, title: row.get(2)?, added_time: row.get(3)?, current_bid_price: row.get(4)?, buy_it_now_price: row.get(5)?, has_best_offer: row.get(6)?, image_url: row.get(7)?, }) }) .ok() } pub fn lookup_since(conn: &Connection, since: i64, limit: i64) -> Vec { let mut stmt = conn .prepare(&format!( "SELECT * FROM {} WHERE added_time >= ?1 ORDER BY added_time LIMIT ?2 ", Self::TABLE_NAME )) .ok() .unwrap(); stmt.query_map([since, limit], |row| { Ok(Listing { id: row.get(0)?, item_id: row.get(1)?, title: row.get(2)?, added_time: row.get(3)?, current_bid_price: row.get(4)?, buy_it_now_price: row.get(5)?, has_best_offer: row.get(6)?, image_url: row.get(7)?, }) }) .ok() .unwrap() .map(|e| e.unwrap()) .collect() } pub fn lookup_non_parsed(conn: &Connection) -> Vec<(i64, String)> { let mut stmt = conn .prepare(&format!( " SELECT ei.item_id, ei.title FROM {} AS ei LEFT JOIN {} AS sp ON ei.item_id = sp.item WHERE sp.item IS NULL", Self::TABLE_NAME, ParsedStorage::TABLE_NAME )) .ok() .unwrap(); stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .ok() .unwrap() .map(|e| e.unwrap()) .collect() } pub fn add_or_update(&self, conn: &Connection) { let count = conn .execute( &format!( "INSERT OR REPLACE INTO {} ( item_id, title, added_time, current_bid_usd_cents, buy_it_now_usd_cents, has_best_offer, image_url ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", Self::TABLE_NAME ), ( self.item_id, &self.title, self.added_time, self.current_bid_price, self.buy_it_now_price, self.has_best_offer, self.image_url.clone(), ), ) .unwrap(); if count != 1 { panic!("Expected count to be 1 but got {}", count); } } } pub fn get_initialized(path: Option<&Path>) -> Connection { let conn = match path { Some(p) => Connection::open(&p), None => Connection::open_in_memory(), } .unwrap(); SearchURL::initialize(&conn); Listing::initialize(&conn); ParsedStorage::initialize(&conn); ParsedPage::initialize(&conn); conn } #[cfg(test)] mod tests { use super::*; #[test] fn sanity_check() { let db = get_initialized(None); let searchurl = SearchURL { full_url: "google".to_owned(), name: "ssd".to_owned(), }; searchurl.add_or_update(&db); assert_eq!(SearchURL::lookup(&db, &searchurl.name), Some(searchurl)); let listing = Listing { id: 1, item_id: 1234, title: "Some Title".to_string(), added_time: std::time::SystemTime::now().into(), current_bid_price: Some(0.12), buy_it_now_price: Some(1.23), has_best_offer: false, image_url: "google.com".to_string(), }; listing.add_or_update(&db); assert_eq!(Listing::lookup(&db, listing.item_id), Some(listing.clone())); let parsed = ParsedStorage { id: 1, item: 1234, total_gigabytes: 13, quantity: 3, individual_size_gigabytes: 13, parse_engine: 9, needed_description_check: true, }; parsed.add_or_update_db(&db); assert_eq!(ParsedStorage::lookup_db(&db, listing.id), vec![parsed]); let page = ParsedPage { category: "ssd".to_owned(), timestamp: std::time::SystemTime::now().into(), }; page.add_or_update_db(&db); assert_eq!(ParsedPage::lookup_db(&db, page.timestamp), Some(page)); } }