Ehhh, lets rethink this ... (parallel status)

src/db.rs

@@ -3,6 +3,8 @@ use rusqlite::Connection;
 use serde::Deserialize;
 use serde::Serialize;
 use std::path::Path;
+use strum::{EnumIter, IntoEnumIterator};
+// use strum_macros::EnumIter;
 use tracing::{error, info};
 
 pub trait DBTable {
@@ -140,30 +142,54 @@ impl SearchURL {
     }
 }
 
+#[derive(Serialize, Debug, PartialEq, Clone, EnumIter)]
+pub enum ParsedPageStatus {
+    PendingParse,
+    Ready,
+}
+impl TryFrom<i64> for ParsedPageStatus {
+    type Error = rusqlite::Error;
+
+    fn try_from(value: i64) -> Result<Self, Self::Error> {
+        match value {
+            0 => Ok(ParsedPageStatus::PendingParse),
+            1 => Ok(ParsedPageStatus::Ready),
+            _ => Err(rusqlite::Error::InvalidColumnType(
+                2,
+                "Invalid integer of {} for ParsedPageStatus".to_string(),
+                rusqlite::types::Type::Integer,
+            )),
+        }
+    }
+}
+
 #[derive(Serialize, Debug, PartialEq, Clone)]
-pub struct ParsedPage {
+pub struct Page {
     pub timestamp: DateTime<Utc>,
     pub category: String,
+    pub status: ParsedPageStatus,
 }
-impl DBTable for ParsedPage {
-    const TABLE_NAME: &'static str = "Pages_Parsed";
+impl DBTable for Page {
+    const TABLE_NAME: &'static str = "Pages";
     const TABLE_SCHEMA: &'static str = "
         id                INTEGER PRIMARY KEY,
         category          TEXT NOT NULL,
         timestamp         INTEGER NOT NULL,
+        status            INTEGER NOT NULL,
         UNIQUE(category, timestamp)
         FOREIGN KEY(category) REFERENCES SearchURLs(name)
     ";
 
     fn get_all(conn: &Connection) -> rusqlite::Result<Vec<Self>> {
         let mut stmt = conn.prepare(&format!(
-            "SELECT category, timestamp FROM {}",
+            "SELECT category, timestamp, status FROM {}",
             Self::TABLE_NAME
         ))?;
         let iter = stmt.query_map([], |row| {
-            Ok(ParsedPage {
+            Ok(Page {
                 category: row.get(0)?,
                 timestamp: row.get(1)?,
+                status: row.get::<_, i64>(2)?.try_into().unwrap(),
             })
         })?;
 
@@ -174,7 +200,7 @@ impl DBTable for ParsedPage {
         Ok(result)
     }
 }
-impl ParsedPage {
+impl Page {
     pub fn lookup(conn: &Connection, timestamp: DateTime<Utc>) -> Option<Self> {
         let mut stmt = conn
             .prepare(&format!(
@@ -183,10 +209,11 @@ impl ParsedPage {
             ))
             .ok()?;
         stmt.query_one([timestamp], |row| {
-            Ok(ParsedPage {
+            Ok(Page {
                 // id: row.get(0)?,
                 category: row.get(1)?,
                 timestamp: row.get(2)?,
+                status: row.get::<_, i64>(3)?.try_into().unwrap(),
             })
         })
         .ok()
@@ -196,13 +223,70 @@ impl ParsedPage {
         let _ = conn
             .execute(
                 &format!(
-                    "INSERT OR REPLACE INTO {} (category, timestamp) VALUES (?1, ?2)",
+                    "INSERT OR REPLACE INTO {} (category, timestamp, status) VALUES (?1, ?2, ?3)",
                     Self::TABLE_NAME
                 ),
-                (&self.category, self.timestamp),
+                (&self.category, self.timestamp, self.status.clone() as i64),
             )
             .unwrap();
     }
+
+    pub fn lookup_status(
+        conn: &Connection,
+        status: ParsedPageStatus,
+        category: &str,
+        max: usize,
+    ) -> Vec<Self> {
+        let mut stmt = conn
+            .prepare(&format!(
+                "SELECT category, timestamp, status FROM {} WHERE status = {} AND category = ?1 LIMIT {}",
+                Self::TABLE_NAME,
+                status.clone() as i64,
+                max
+            ))
+            .unwrap();
+        stmt.query_map([category], |row| {
+            Ok(Self {
+                category: row.get(0)?,
+                timestamp: row.get(1)?,
+                status: row.get::<_, i64>(2)?.try_into().unwrap(),
+            })
+        })
+        .unwrap()
+        .inspect(|e| info!("debugging saw {:?}", e))
+        .filter_map(|e| e.ok())
+        .collect()
+    }
+
+    pub fn category_stats(conn: &Connection, category: &str) -> Vec<(ParsedPageStatus, i64, i64)> {
+        let mut res: Vec<(ParsedPageStatus, i64, i64)> = vec![];
+
+        for status in ParsedPageStatus::iter() {
+            let cnt_category_status = conn
+                .prepare(&format!(
+                    "SELECT COUNT(*) FROM {} WHERE category = ?1 AND status = {}",
+                    Self::TABLE_NAME,
+                    status.clone() as i64
+                ))
+                .ok()
+                .unwrap()
+                .query_one([category], |r| r.get(0))
+                .inspect_err(|e| error!("Failed to get count due to error\"{:?}\", returning 0", e))
+                .unwrap_or(0);
+            let cnt_category_total = conn
+                .prepare(&format!(
+                    "SELECT COUNT(*) FROM {} WHERE category = ?1",
+                    Self::TABLE_NAME
+                ))
+                .ok()
+                .unwrap()
+                .query_one([category], |r| r.get(0))
+                .inspect_err(|e| error!("Failed to get count due to error\"{:?}\", returning 0", e))
+                .unwrap_or(0);
+            res.push((status, cnt_category_status, cnt_category_total));
+        }
+        res
+    }
 }
 
 #[derive(Serialize, Debug, PartialEq, Copy, Clone)]
@@ -318,7 +402,7 @@ impl DBTable for ItemAppearances {
         current_bid_usd_cents   INTEGER,
         UNIQUE(item, timestamp),
         FOREIGN KEY(item)       REFERENCES Listings(item_id),
-        FOREIGN KEY(category, timestamp) REFERENCES Pages_Parsed(category, timestamp)
+        FOREIGN KEY(category, timestamp) REFERENCES Pages(category, timestamp)
     ";
 
     fn get_all(conn: &Connection) -> rusqlite::Result<Vec<Self>> {
@@ -624,7 +708,7 @@ pub fn get_initialized(path: Option<&Path>) -> Connection {
     SearchURL::initialize(&conn);
     Listing::initialize(&conn);
     ParsedStorage::initialize(&conn);
-    ParsedPage::initialize(&conn);
+    Page::initialize(&conn);
     ItemAppearances::initialize(&conn);
 
     conn
@@ -644,7 +728,7 @@ pub fn get_stats(conn: &Connection) -> Stats {
         rows_search_url: SearchURL::get_count(conn),
         rows_listing: Listing::get_count(conn),
         rows_parsed_storage: ParsedStorage::get_count(conn),
-        rows_parsed_page: ParsedPage::get_count(conn),
+        rows_parsed_page: Page::get_count(conn),
         rows_item_appearances: ItemAppearances::get_count(conn),
     }
 }
@@ -687,12 +771,28 @@ mod tests {
         parsed.add_or_update(&db);
         assert_eq!(ParsedStorage::lookup(&db, listing.item_id), vec![parsed]);
 
-        let page = ParsedPage {
+        let page = Page {
             category: "ssd".to_owned(),
             timestamp: std::time::SystemTime::now().into(),
+            status: ParsedPageStatus::PendingParse,
         };
         page.add_or_update(&db);
-        assert_eq!(ParsedPage::lookup(&db, page.timestamp), Some(page.clone()));
+        assert_eq!(Page::lookup(&db, page.timestamp), Some(page.clone()));
+        assert_eq!(
+            Page::lookup_status(&db, ParsedPageStatus::PendingParse, "ssd", 10),
+            vec![page.clone()]
+        );
+        assert_eq!(
+            Page::lookup_status(&db, ParsedPageStatus::Ready, "ssd", 10),
+            vec![]
+        );
+        assert_eq!(
+            Page::category_stats(&db, "ssd"),
+            vec![
+                (ParsedPageStatus::PendingParse, 1, 1),
+                (ParsedPageStatus::Ready, 0, 1)
+            ]
+        );
 
         let apperance = ItemAppearances {
             item: listing.item_id,