# Scraper Web API

This is a dumb little tool which ingests raw HTML files, does some parsing on them, and serves the results over a web API.

```bash
export URL_BASE="localhost:9876"; \
echo run0 && http POST "$URL_BASE/category/ssd/discover" && \
echo run1 && http POST "$URL_BASE/category/ssd/parse" && \
echo run2 && http GET "$URL_BASE/category/ssd/parse" && \
echo run3 && http POST "$URL_BASE/listing/parse" && \
echo run4 && http GET "$URL_BASE/listings" since:=10099 limit:=10 cents_per_tbytes_max:=900 && \
echo run5 && http GET "$URL_BASE/listing/267267322597" && \
echo run6 && http GET "$URL_BASE/listing/267267322597/history" &&
echo run7 && http GET "$URL_BASE/listing/267267322597/parsed"
```

```
run0
HTTP/1.1 200 OK
content-length: 0
content-type: text/plain; charset=utf-8
date: Thu, 10 Jul 2025 04:26:49 GMT




run1
HTTP/1.1 200 OK
content-length: 0
content-type: text/plain; charset=utf-8
date: Thu, 10 Jul 2025 04:26:49 GMT




run2
HTTP/1.1 200 OK
content-length: 36
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT

[
    [
        "PendingParse",
        1,
        1
    ],
    [
        "Ready",
        0,
        1
    ]
]


run3
HTTP/1.1 200 OK
content-length: 2
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT

62


run4
HTTP/1.1 200 OK
content-length: 4232
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT

[
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 1260,
                "item": 286605201240,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": true,
            "id": 5,
            "image_url": "https://i.ebayimg.com/images/g/3NoAAeSwPrtoDb1O/s-l500.webp",
            "item_id": 286605201240,
            "title": "Fanxiang M.2 SSD 1TB NVMe PCIe Gen 3x 4 M2 Internal Solid State Drive 3500MB/s"
        },
        "parsed": [
            {
                "id": 5,
                "individual_size_gigabytes": 1024,
                "item": 286605201240,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 1,
                "total_gigabytes": 1024
            }
        ]
    },
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 2400,
                "item": 177133381123,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": false,
            "id": 22,
            "image_url": "https://i.ebayimg.com/images/g/-VMAAOSwaX1oNyx4/s-l500.webp",
            "item_id": 177133381123,
            "title": "SanDisk professional G-DRIVE SSD 2TB, A+ condition"
        },
        "parsed": [
            {
                "id": 22,
                "individual_size_gigabytes": 2048,
                "item": 177133381123,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 1,
                "total_gigabytes": 2048
            }
        ]
    },
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 3108,
                "item": 187263467837,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": false,
            "id": 35,
            "image_url": "https://i.ebayimg.com/images/g/hn8AAOSw1hJoNrJm/s-l500.webp",
            "item_id": 187263467837,
            "title": "Used Fanxiang S880 4TB SSD NVME M.2 SSD PCIe 4x4 7300MBS Solid State Drive"
        },
        "parsed": [
            {
                "id": 35,
                "individual_size_gigabytes": 4096,
                "item": 187263467837,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 1,
                "total_gigabytes": 4096
            }
        ]
    },
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 1000,
                "item": 267267367821,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": false,
            "id": 37,
            "image_url": "https://i.ebayimg.com/images/g/Cr8AAOSwXY1oN6m8/s-l500.webp",
            "item_id": 267267367821,
            "title": "(Lot of 6) Samsung MZ-VLB2560 256GB M.2 NVMe Internal SSD (MZVLB256HBHQ-000H1)"
        },
        "parsed": [
            {
                "id": 37,
                "individual_size_gigabytes": 256,
                "item": 267267367821,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 6,
                "total_gigabytes": 1536
            }
        ]
    },
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 4600,
                "item": 187263491149,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": false,
            "id": 44,
            "image_url": "https://i.ebayimg.com/images/g/v2EAAOSwg9poNrTr/s-l500.webp",
            "item_id": 187263491149,
            "title": "Used Silicon Power 4TB US75 Nvme PCIe Gen4x4 M.2 2280 SSD R/W Up to 7000/6500 MB"
        },
        "parsed": [
            {
                "id": 44,
                "individual_size_gigabytes": 4096,
                "item": 187263491149,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 1,
                "total_gigabytes": 4096
            }
        ]
    },
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 1000,
                "item": 267267351339,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": false,
            "id": 46,
            "image_url": "https://i.ebayimg.com/images/g/z8EAAOSwyKZoN6TW/s-l500.webp",
            "item_id": 267267351339,
            "title": "(Lot of 6) Used -Micron MTFDDAV256TBN 256GB, M.2 2280 Solid State Drive"
        },
        "parsed": [
            {
                "id": 46,
                "individual_size_gigabytes": 256,
                "item": 267267351339,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 6,
                "total_gigabytes": 1536
            }
        ]
    },
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 99,
                "item": 306325087069,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": false,
            "id": 59,
            "image_url": "https://i.ebayimg.com/images/g/zuUAAOSwIoJoN5yC/s-l500.webp",
            "item_id": 306325087069,
            "title": "T298 ~ HP OEM Desktop Z240 Workstation Heatsink w NVMe M.2 256GB SSD 826414-001"
        },
        "parsed": [
            {
                "id": 59,
                "individual_size_gigabytes": 256,
                "item": 306325087069,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 1,
                "total_gigabytes": 256
            }
        ]
    },
    {
        "history": [
            {
                "category": "ssd",
                "current_bid_usd_cents": 1000,
                "item": 267267322597,
                "timestamp": "2025-06-19T21:44:23Z"
            }
        ],
        "listing": {
            "buy_it_now_price_cents": null,
            "has_best_offer": false,
            "id": 60,
            "image_url": "https://i.ebayimg.com/images/g/r8YAAOSwlkdoN5uW/s-l500.webp",
            "item_id": 267267322597,
            "title": "(Lot of 5) Used - Micro 1100 256GB SATA III 2.5\" SSD MTFDDAK256TBN"
        },
        "parsed": [
            {
                "id": 60,
                "individual_size_gigabytes": 256,
                "item": 267267322597,
                "needed_description_check": false,
                "parse_engine": 0,
                "quantity": 5,
                "total_gigabytes": 1280
            }
        ]
    }
]


run5
HTTP/1.1 200 OK
content-length: 237
content-type: application/json
date: Thu, 10 Jul 2025 04:26:49 GMT

{
    "buy_it_now_price_cents": null,
    "has_best_offer": false,
    "id": 60,
    "image_url": "https://i.ebayimg.com/images/g/r8YAAOSwlkdoN5uW/s-l500.webp",
    "item_id": 267267322597,
    "title": "(Lot of 5) Used - Micro 1100 256GB SATA III 2.5\" SSD MTFDDAK256TBN"
}


run6
HTTP/1.1 200 OK
content-length: 62
content-type: application/json
date: Thu, 10 Jul 2025 04:26:50 GMT

[
    {
        "current_bid_usd_cents": 1000,
        "when": "2025-06-19T21:44:23Z"
    }
]


run7
HTTP/1.1 200 OK
content-length: 149
content-type: application/json
date: Thu, 10 Jul 2025 04:26:50 GMT

[
    {
        "id": 60,
        "individual_size_gigabytes": 256,
        "item": 267267322597,
        "needed_description_check": false,
        "parse_engine": 0,
        "quantity": 5,
        "total_gigabytes": 1280
    }
]
```