hak8or
9331b55e08
All checks were successful
Cargo Build & Test / Rust project - latest (1.85.1) (push) Successful in 4m31s
Cargo Build & Test / Rust project - latest (1.86) (push) Successful in 4m40s
Cargo Build & Test / Rust project - latest (1.87) (push) Successful in 4m55s
Cargo Build & Test / Rust project - latest (1.88) (push) Successful in 10m2s
9.5 KiB
9.5 KiB
Scraper Web API
This is a dumb little tool which ingests raw HTML files, does some parsing on them, and serves the results over a web API.
export URL_BASE="localhost:9876"; \
echo run0 && http POST "$URL_BASE/category/ssd/discover" && \
echo run1 && http POST "$URL_BASE/category/ssd/parse" && \
echo run2 && http GET "$URL_BASE/category/ssd/parse" && \
echo run3 && http POST "$URL_BASE/listing/parse" && \
echo run4 && http GET "$URL_BASE/listings" since:=10099 limit:=10 cents_per_tbytes_max:=900 && \
echo run5 && http GET "$URL_BASE/listing/267267322597" && \
echo run6 && http GET "$URL_BASE/listing/267267322597/history" &&
echo run7 && http GET "$URL_BASE/listing/267267322597/parsed"
run0
HTTP/1.1 200 OK
content-length: 0
content-type: text/plain; charset=utf-8
date: Thu, 10 Jul 2025 04:26:49 GMT
run1
HTTP/1.1 200 OK
content-length: 0
content-type: text/plain; charset=utf-8
date: Thu, 10 Jul 2025 04:26:49 GMT
run2
HTTP/1.1 200 OK
content-length: 36
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT
[
[
"PendingParse",
1,
1
],
[
"Ready",
0,
1
]
]
run3
HTTP/1.1 200 OK
content-length: 2
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT
62
run4
HTTP/1.1 200 OK
content-length: 4232
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT
[
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 1260,
"item": 286605201240,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": true,
"id": 5,
"image_url": "https://i.ebayimg.com/images/g/3NoAAeSwPrtoDb1O/s-l500.webp",
"item_id": 286605201240,
"title": "Fanxiang M.2 SSD 1TB NVMe PCIe Gen 3x 4 M2 Internal Solid State Drive 3500MB/s"
},
"parsed": [
{
"id": 5,
"individual_size_gigabytes": 1024,
"item": 286605201240,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 1,
"total_gigabytes": 1024
}
]
},
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 2400,
"item": 177133381123,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 22,
"image_url": "https://i.ebayimg.com/images/g/-VMAAOSwaX1oNyx4/s-l500.webp",
"item_id": 177133381123,
"title": "SanDisk professional G-DRIVE SSD 2TB, A+ condition"
},
"parsed": [
{
"id": 22,
"individual_size_gigabytes": 2048,
"item": 177133381123,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 1,
"total_gigabytes": 2048
}
]
},
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 3108,
"item": 187263467837,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 35,
"image_url": "https://i.ebayimg.com/images/g/hn8AAOSw1hJoNrJm/s-l500.webp",
"item_id": 187263467837,
"title": "Used Fanxiang S880 4TB SSD NVME M.2 SSD PCIe 4x4 7300MBS Solid State Drive"
},
"parsed": [
{
"id": 35,
"individual_size_gigabytes": 4096,
"item": 187263467837,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 1,
"total_gigabytes": 4096
}
]
},
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 1000,
"item": 267267367821,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 37,
"image_url": "https://i.ebayimg.com/images/g/Cr8AAOSwXY1oN6m8/s-l500.webp",
"item_id": 267267367821,
"title": "(Lot of 6) Samsung MZ-VLB2560 256GB M.2 NVMe Internal SSD (MZVLB256HBHQ-000H1)"
},
"parsed": [
{
"id": 37,
"individual_size_gigabytes": 256,
"item": 267267367821,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 6,
"total_gigabytes": 1536
}
]
},
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 4600,
"item": 187263491149,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 44,
"image_url": "https://i.ebayimg.com/images/g/v2EAAOSwg9poNrTr/s-l500.webp",
"item_id": 187263491149,
"title": "Used Silicon Power 4TB US75 Nvme PCIe Gen4x4 M.2 2280 SSD R/W Up to 7000/6500 MB"
},
"parsed": [
{
"id": 44,
"individual_size_gigabytes": 4096,
"item": 187263491149,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 1,
"total_gigabytes": 4096
}
]
},
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 1000,
"item": 267267351339,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 46,
"image_url": "https://i.ebayimg.com/images/g/z8EAAOSwyKZoN6TW/s-l500.webp",
"item_id": 267267351339,
"title": "(Lot of 6) Used -Micron MTFDDAV256TBN 256GB, M.2 2280 Solid State Drive"
},
"parsed": [
{
"id": 46,
"individual_size_gigabytes": 256,
"item": 267267351339,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 6,
"total_gigabytes": 1536
}
]
},
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 99,
"item": 306325087069,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 59,
"image_url": "https://i.ebayimg.com/images/g/zuUAAOSwIoJoN5yC/s-l500.webp",
"item_id": 306325087069,
"title": "T298 ~ HP OEM Desktop Z240 Workstation Heatsink w NVMe M.2 256GB SSD 826414-001"
},
"parsed": [
{
"id": 59,
"individual_size_gigabytes": 256,
"item": 306325087069,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 1,
"total_gigabytes": 256
}
]
},
{
"history": [
{
"category": "ssd",
"current_bid_usd_cents": 1000,
"item": 267267322597,
"timestamp": "2025-06-19T21:44:23Z"
}
],
"listing": {
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 60,
"image_url": "https://i.ebayimg.com/images/g/r8YAAOSwlkdoN5uW/s-l500.webp",
"item_id": 267267322597,
"title": "(Lot of 5) Used - Micro 1100 256GB SATA III 2.5\" SSD MTFDDAK256TBN"
},
"parsed": [
{
"id": 60,
"individual_size_gigabytes": 256,
"item": 267267322597,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 5,
"total_gigabytes": 1280
}
]
}
]
run5
HTTP/1.1 200 OK
content-length: 237
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT
{
"buy_it_now_price_cents": null,
"has_best_offer": false,
"id": 60,
"image_url": "https://i.ebayimg.com/images/g/r8YAAOSwlkdoN5uW/s-l500.webp",
"item_id": 267267322597,
"title": "(Lot of 5) Used - Micro 1100 256GB SATA III 2.5\" SSD MTFDDAK256TBN"
}
run6
HTTP/1.1 200 OK
content-length: 62
content-type: application/json
date: Thu, 10 Jul 2025 04:26:50 GMT
[
{
"current_bid_usd_cents": 1000,
"when": "2025-06-19T21:44:23Z"
}
]
run7
HTTP/1.1 200 OK
content-length: 149
content-type: application/json
date: Thu, 10 Jul 2025 04:26:50 GMT
[
{
"id": 60,
"individual_size_gigabytes": 256,
"item": 267267322597,
"needed_description_check": false,
"parse_engine": 0,
"quantity": 5,
"total_gigabytes": 1280
}
]