Moved ebay scraping into separate directory and added rust version
```
➜ time yarn --silent scrape --only_json --load foo4.html > /dev/null
________________________________________________________
Executed in 742.13 millis fish external
usr time 605.32 millis 408.00 micros 604.91 millis
sys time 229.09 millis 214.00 micros 228.88 millis
```
```
➜ time cargo run --release -- --only-json --load foo4.html > /dev/null
Finished `release` profile [optimized] target(s) in 0.06s
Running `target/release/ebay_scraper_rust --only-json --load foo4.html`
________________________________________________________
Executed in 122.54 millis fish external
usr time 87.85 millis 597.00 micros 87.26 millis
sys time 40.10 millis 152.00 micros 39.95 millis
```
This commit is contained in:
1
ebay_storage/javascript/.gitignore
vendored
Normal file
1
ebay_storage/javascript/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
/node_modules
|
||||
279
ebay_storage/javascript/ebay_command_line_tool.js
Normal file
279
ebay_storage/javascript/ebay_command_line_tool.js
Normal file
@ -0,0 +1,279 @@
|
||||
// ebay_command_line_tool.js V4.1
|
||||
// Node.js script with commands to scrape eBay and output JSON.
|
||||
// Images are now saved preserving their URL path structure within the save directory.
|
||||
|
||||
const puppeteer = require('puppeteer');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Command } = require('commander');
|
||||
const https = require('https'); // For downloading images
|
||||
const http = require('http'); // For downloading images (fallback)
|
||||
const { URL } = require('url'); // For parsing image URLs
|
||||
|
||||
// --- Load Core Script ---
|
||||
const coreScriptPath = path.join(__dirname, 'ebay_core.js');
|
||||
let ebayCoreScriptContent;
|
||||
try {
|
||||
ebayCoreScriptContent = fs.readFileSync(coreScriptPath, 'utf8');
|
||||
if (!ebayCoreScriptContent) throw new Error("ebay_core.js is empty.");
|
||||
} catch (e) {
|
||||
console.error(`Critical Error: Could not read ebay_core.js: ${e.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let quietMode = false;
|
||||
function logMessage(message) { if (!quietMode) console.log(message); }
|
||||
function logError(message) { if (!quietMode) console.error(message); }
|
||||
|
||||
// --- Image Downloading Function (Updated) ---
|
||||
async function downloadImage(imageUrl, baseSaveDirectory) {
|
||||
if (!imageUrl) return;
|
||||
try {
|
||||
const parsedUrl = new URL(imageUrl);
|
||||
|
||||
// Get the full path from the URL (e.g., /images/g/5okAAeSwIGdoN8Ed/s-l500.webp)
|
||||
// Ensure leading slash is removed for path.join to work as expected relative to baseSaveDirectory
|
||||
const imagePathFromUrl = parsedUrl.pathname.startsWith('/') ? parsedUrl.pathname.substring(1) : parsedUrl.pathname;
|
||||
|
||||
// Separate the directory part and the filename part from the URL path
|
||||
const imageName = path.basename(imagePathFromUrl);
|
||||
const imageSubdirectory = path.dirname(imagePathFromUrl);
|
||||
|
||||
// Construct the full local directory path
|
||||
const fullLocalDirectory = path.join(baseSaveDirectory, imageSubdirectory);
|
||||
const fullLocalImagePath = path.join(fullLocalDirectory, imageName);
|
||||
|
||||
// Ensure directory exists
|
||||
if (!fs.existsSync(fullLocalDirectory)) {
|
||||
fs.mkdirSync(fullLocalDirectory, { recursive: true });
|
||||
logMessage(`Created image directory: ${fullLocalDirectory}`);
|
||||
}
|
||||
|
||||
// Check if file already exists to avoid re-downloading (optional, can be useful)
|
||||
// if (fs.existsSync(fullLocalImagePath)) {
|
||||
// logMessage(`Image already exists, skipping: ${fullLocalImagePath}`);
|
||||
// return Promise.resolve();
|
||||
// }
|
||||
|
||||
const fileStream = fs.createWriteStream(fullLocalImagePath);
|
||||
const protocol = parsedUrl.protocol === 'https:' ? https : http;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = protocol.get(imageUrl, (response) => {
|
||||
if (response.statusCode !== 200) {
|
||||
logError(`Failed to download image ${imageUrl}. Status: ${response.statusCode}`);
|
||||
response.resume(); // Consume response data to free up resources
|
||||
reject(new Error(`Status code ${response.statusCode} for ${imageUrl}`));
|
||||
return;
|
||||
}
|
||||
response.pipe(fileStream);
|
||||
fileStream.on('finish', () => {
|
||||
fileStream.close(); // close() is async, call resolve after it's done
|
||||
logMessage(`Downloaded image: ${fullLocalImagePath}`);
|
||||
resolve();
|
||||
});
|
||||
fileStream.on('error', (err) => { // Handle stream errors
|
||||
logError(`Error writing image file ${fullLocalImagePath}: ${err.message}`);
|
||||
fs.unlink(fullLocalImagePath, () => {}); // Attempt to delete partial file
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
request.on('error', (err) => { // Handle request errors
|
||||
logError(`Error downloading image ${imageUrl}: ${err.message}`);
|
||||
// No partial file to unlink here as the request itself failed
|
||||
reject(err);
|
||||
});
|
||||
// Set a timeout for the request
|
||||
request.setTimeout(30000, () => { // 30 seconds timeout
|
||||
request.destroy(); // Destroy the request object on timeout
|
||||
logError(`Timeout downloading image ${imageUrl}`);
|
||||
reject(new Error(`Timeout downloading image ${imageUrl}`));
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
logError(`Error processing image URL ${imageUrl}: ${error.message}`);
|
||||
return Promise.reject(error); // Propagate the error
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// --- Main Scraping Function ---
|
||||
async function scrapeEbay({ url = null, htmlFile = null, saveFile = null }) {
|
||||
logMessage("Starting scraping process...");
|
||||
let browser;
|
||||
try {
|
||||
browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] });
|
||||
const page = await browser.newPage();
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36');
|
||||
|
||||
let htmlContentToParse;
|
||||
if (htmlFile) {
|
||||
logMessage(`Loading HTML from ${htmlFile}...`);
|
||||
htmlContentToParse = fs.readFileSync(htmlFile, 'utf8');
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => { request.abort(); });
|
||||
await page.setContent(htmlContentToParse, { waitUntil: 'domcontentloaded' });
|
||||
logMessage("HTML loaded. Network requests blocked.");
|
||||
} else if (url) {
|
||||
logMessage(`Navigating to ${url}...`);
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 90000 });
|
||||
logMessage("Navigation successful.");
|
||||
htmlContentToParse = await page.content();
|
||||
logMessage("Page content retrieved.");
|
||||
if (saveFile && htmlContentToParse) {
|
||||
logMessage(`Saving HTML to ${saveFile}...`);
|
||||
fs.writeFileSync(saveFile, htmlContentToParse, 'utf8');
|
||||
logMessage("HTML saved.");
|
||||
}
|
||||
} else {
|
||||
throw new Error("Internal Error: Neither URL nor HTML file was provided.");
|
||||
}
|
||||
|
||||
logMessage("Injecting core parser script...");
|
||||
await page.evaluate(ebayCoreScriptContent);
|
||||
logMessage("Core script injected. Extracting data...");
|
||||
const extractedResults = await page.evaluate(() => {
|
||||
if (typeof window.EbayParser === 'undefined' || typeof window.EbayParser.extractDataFromPage !== 'function') {
|
||||
throw new Error("EbayParser not found!");
|
||||
}
|
||||
return window.EbayParser.extractDataFromPage();
|
||||
});
|
||||
logMessage(`Data extraction complete. Found ${extractedResults.length} items.`);
|
||||
|
||||
// If HTML was fetched and --save was used, now download images
|
||||
if (url && saveFile && extractedResults.length > 0) {
|
||||
const baseSaveName = path.parse(saveFile).name; // e.g., "foo2"
|
||||
// The main directory for this save operation (e.g., "foo2/")
|
||||
const mainImageSaveDirectory = path.join(path.dirname(saveFile), baseSaveName);
|
||||
logMessage(`Downloading images for ${baseSaveName} into subdirectories of ${mainImageSaveDirectory}...`);
|
||||
|
||||
const downloadPromises = [];
|
||||
for (const item of extractedResults) {
|
||||
if (item.image_url) {
|
||||
// Pass the mainImageSaveDirectory as the base for creating nested structure
|
||||
downloadPromises.push(
|
||||
downloadImage(item.image_url, mainImageSaveDirectory).catch(e => {
|
||||
logError(`Skipping image download for item ID ${item.itemId || 'unknown'} (URL: ${item.image_url}) due to error: ${e.message}`);
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
await Promise.all(downloadPromises); // Wait for all image downloads to attempt completion
|
||||
logMessage("Image download process finished.");
|
||||
}
|
||||
return extractedResults;
|
||||
} catch (e) {
|
||||
logError(`Scraping process error: ${e.message}`);
|
||||
if (!quietMode && e.stack) console.error(e.stack);
|
||||
return [];
|
||||
} finally {
|
||||
if (browser) {
|
||||
await browser.close();
|
||||
logMessage("Browser closed.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const program = new Command();
|
||||
program
|
||||
.name('ebay-scraper')
|
||||
.description('Scrapes eBay search results.')
|
||||
.version('4.1.0') // Version bump
|
||||
.option('--save <filename>', 'Save scraped HTML to a file (and download images if fetching from URL).')
|
||||
.option('--load <filename>', 'Load HTML from a file (disables network). Image download will not occur with --load.')
|
||||
.option('--only_json', 'Suppress informational logs, output only final JSON.', false)
|
||||
.on('option:only_json', () => { quietMode = true; });
|
||||
|
||||
program
|
||||
.command('latest')
|
||||
.description('Scrapes latest listings. Use "ebay-scraper latest --help" for options.')
|
||||
.option('--per_page <number>', 'Items per page (60, 120, or 240)', '60')
|
||||
.option('--minimum_cost <number>', 'Minimum cost (e.g., 50.00)', '0.00')
|
||||
.action(async (cmdOptions) => {
|
||||
const globalOptions = program.opts();
|
||||
if (globalOptions.only_json) quietMode = true;
|
||||
if (globalOptions.load) {
|
||||
logMessage("Using --load for 'latest'. URL generation options ignored. Images will not be downloaded.");
|
||||
await runScraping({ htmlFile: globalOptions.load, saveFile: globalOptions.save });
|
||||
} else {
|
||||
const validPages = ['60', '120', '240'];
|
||||
if (!validPages.includes(cmdOptions.per_page)) {
|
||||
logError(`Error: --per_page must be one of ${validPages.join(', ')}.`);
|
||||
if (!quietMode) process.exit(1); else throw new Error("Invalid per_page");
|
||||
}
|
||||
const minCost = parseFloat(cmdOptions.minimum_cost);
|
||||
if (isNaN(minCost)) {
|
||||
logError("Error: --minimum_cost must be a number.");
|
||||
if (!quietMode) process.exit(1); else throw new Error("Invalid minimum_cost");
|
||||
}
|
||||
const baseUrl = 'https://www.ebay.com/sch/i.html?_nkw=&_sacat=175669&_from=R40&_fsrp=1&LH_PrefLoc=3&imm=1&_sop=10';
|
||||
const url = `${baseUrl}&_ipg=${cmdOptions.per_page}&_udlo=${minCost.toFixed(2)}`;
|
||||
logMessage(`Constructed URL for 'latest': ${url}`);
|
||||
await runScraping({ url: url, saveFile: globalOptions.save });
|
||||
}
|
||||
});
|
||||
|
||||
program
|
||||
.argument('[url]', 'The full eBay search URL to scrape.')
|
||||
.action(async (url) => {
|
||||
const globalOptions = program.opts();
|
||||
if (globalOptions.only_json) quietMode = true;
|
||||
if (globalOptions.load) {
|
||||
logMessage("Using --load. Provided URL argument ignored. Images will not be downloaded.");
|
||||
await runScraping({ htmlFile: globalOptions.load, saveFile: globalOptions.save });
|
||||
} else if (url) {
|
||||
await runScraping({ url: url, saveFile: globalOptions.save });
|
||||
} else {
|
||||
// If no URL, no --load, and not the 'latest' command, show help.
|
||||
// Check if 'latest' was an argument. If so, commander handles its action.
|
||||
// If not, and no URL, then show help.
|
||||
const isLatestCommand = process.argv.includes('latest');
|
||||
if (!isLatestCommand) {
|
||||
program.help();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
program.addHelpText('after', `
|
||||
Example calls:
|
||||
$ ebay-scraper latest --per_page 120
|
||||
$ ebay-scraper "https://www.ebay.com/sch/i.html?_nkw=ssd"
|
||||
$ ebay-scraper --load page.html --only_json | jq .
|
||||
$ ebay-scraper --save page.html "https://www.ebay.com/sch/i.html?_nkw=hdd"`);
|
||||
|
||||
async function runScraping(options) {
|
||||
try {
|
||||
const data = await scrapeEbay(options);
|
||||
if (quietMode) {
|
||||
process.stdout.write(JSON.stringify(data, null, 2));
|
||||
} else {
|
||||
if (data && data.length > 0) console.log(JSON.stringify(data, null, 2));
|
||||
else logMessage("No data extracted or a critical error occurred.");
|
||||
}
|
||||
} catch (e) {
|
||||
logError(`Critical error in runScraping: ${e.message}`);
|
||||
if (!quietMode && e.stack) console.error(e.stack);
|
||||
if (quietMode) process.stdout.write(JSON.stringify({error: e.message, data: []}));
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
await program.parseAsync(process.argv);
|
||||
// If no command was specified and no URL, Commander's default help might not trigger if only options are present.
|
||||
// This ensures help is shown if no actionable arguments are given.
|
||||
const args = process.argv.slice(2);
|
||||
const hasActionableArg = args.some(arg => !arg.startsWith('-') || program.commands.some(cmd => cmd.name() === arg));
|
||||
if (args.length > 0 && !hasActionableArg && !program.opts().load) { // If only options like --only_json but no command/url/load
|
||||
program.help();
|
||||
} else if (args.length === 0) { // No arguments at all
|
||||
program.help();
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logError(`Command parsing error: ${error.message}`);
|
||||
if (!quietMode && error.stack) console.error(error.stack);
|
||||
if (quietMode) process.stdout.write(JSON.stringify({error: error.message, data: []}));
|
||||
else process.exit(1);
|
||||
}
|
||||
})();
|
||||
250
ebay_storage/javascript/ebay_core.js
Normal file
250
ebay_storage/javascript/ebay_core.js
Normal file
@ -0,0 +1,250 @@
|
||||
// ebay_core.js V1.4 - Shared Parsing & Extraction Logic
|
||||
// - Restructured JSON output with a "parsed" sub-object.
|
||||
// - Added parser_engine version.
|
||||
// - Removed itemUrl, added image_url.
|
||||
(function (root, factory) {
|
||||
if (typeof module === 'object' && module.exports) {
|
||||
module.exports = factory();
|
||||
} else {
|
||||
root.EbayParser = factory();
|
||||
}
|
||||
}(typeof self !== 'undefined' ? self : this, function () {
|
||||
'use strict';
|
||||
|
||||
const EbayParser = {};
|
||||
const PARSER_ENGINE_VERSION = 1;
|
||||
|
||||
EbayParser.parseSizeAndQuantity = function(title) {
|
||||
title = title ? title.toUpperCase() : "";
|
||||
let totalTB = 0;
|
||||
let quantity = 1;
|
||||
let needed_description_check = false;
|
||||
let individualSizeTB = 0;
|
||||
|
||||
const explicitQtyPatterns = [
|
||||
/\b(?:LOT\s+OF|LOT)\s*\(?\s*(\d+)\s*\)?/i,
|
||||
/\b(?:LOT\s+OF|LOT)\s*\*\s*(\d+)/i,
|
||||
/\b(?:PACK\s+OF|PACK|BULK)\s*\(?\s*(\d+)\s*\)?/i,
|
||||
/\b(\d+)\s*-\s*PACK\b/i,
|
||||
/\b(\d+)\s*COUNT\b/i
|
||||
];
|
||||
|
||||
for (const pattern of explicitQtyPatterns) {
|
||||
const qtyMatch = title.match(pattern);
|
||||
if (qtyMatch && qtyMatch[1]) {
|
||||
const parsedQty = parseInt(qtyMatch[1], 10);
|
||||
if (parsedQty > 0 && parsedQty < 500) {
|
||||
quantity = parsedQty;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sizeMatches = [];
|
||||
const sizeRegex = /(\d+(?:\.\d+)?)\s*(TB|GB)(?:\b|(?=\s|-|,|\(|\)|$))/g;
|
||||
let match;
|
||||
while ((match = sizeRegex.exec(title)) !== null) {
|
||||
sizeMatches.push({ value: parseFloat(match[1]), unit: match[2].toUpperCase() });
|
||||
}
|
||||
|
||||
if (sizeMatches.length > 0) {
|
||||
const uniqueSizesTB = [...new Set(
|
||||
sizeMatches.map(sm => sm.unit === 'GB' ? sm.value / 1000 : sm.value)
|
||||
)].sort((a, b) => a - b);
|
||||
if (uniqueSizesTB.length > 0) {
|
||||
individualSizeTB = uniqueSizesTB[0];
|
||||
if (uniqueSizesTB.length > 1) needed_description_check = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (title.match(/\d+(?:\.\d+)?\s*(?:GB|TB)\s*(?:-|&|OR|TO)\s*\d+(?:\.\d+)?\s*(?:GB|TB)/i)) {
|
||||
needed_description_check = true;
|
||||
}
|
||||
if (quantity > 1 && title.includes("MIXED")) {
|
||||
needed_description_check = true;
|
||||
}
|
||||
if (title.includes("CHECK THE DESCRIPTION") || title.includes("CHECK DESCRIPTION") || title.includes("SEE DESCRIPTION")) {
|
||||
if (quantity > 1 || sizeMatches.length === 0 || sizeMatches.length > 1) {
|
||||
needed_description_check = true;
|
||||
}
|
||||
}
|
||||
if (individualSizeTB > 0) {
|
||||
totalTB = individualSizeTB * quantity;
|
||||
}
|
||||
if (quantity > 1 && totalTB === 0) {
|
||||
needed_description_check = true;
|
||||
}
|
||||
if (quantity === 1 && sizeMatches.length === 1 && !needed_description_check) {
|
||||
needed_description_check = false;
|
||||
}
|
||||
|
||||
return {
|
||||
totalTB: parseFloat(totalTB.toFixed(4)),
|
||||
quantity: quantity,
|
||||
needed_description_check: needed_description_check,
|
||||
individualSizeTB: parseFloat(individualSizeTB.toFixed(4))
|
||||
};
|
||||
};
|
||||
|
||||
EbayParser.parsePrice = function(priceText) {
|
||||
priceText = priceText || "";
|
||||
if (priceText.toLowerCase().includes(' to ')) {
|
||||
const rangeParts = priceText.split(/to/i);
|
||||
const firstPriceMatch = rangeParts[0] ? rangeParts[0].match(/\$?([\d,]+\.?\d*)/) : null;
|
||||
if (firstPriceMatch) {
|
||||
return parseFloat(firstPriceMatch[1].replace(/,/g, ''));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const priceMatch = priceText.match(/\$?([\d,]+\.?\d*)/);
|
||||
if (priceMatch) {
|
||||
return parseFloat(priceMatch[1].replace(/,/g, ''));
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
EbayParser.runUnitTests = function() {
|
||||
const log = typeof console !== 'undefined' ? console.log : function() {};
|
||||
const error = typeof console !== 'undefined' ? console.error : function() {};
|
||||
log("Ebay Cost/TB: --- Running Unit Tests ---");
|
||||
const testCases = [
|
||||
{ title: "LOT OF (9) MAJOR BRAND 2.5\" 7MM SSD * Kingston, Samsung, SanDisk& PNY*120-250GB", expected: { totalTB: 1.080, quantity: 9, individualSizeTB: 0.120, needed_description_check: true } },
|
||||
{ title: "Lot of 10 Intel 256 GB 2.5\" SATA SSD different Model check the Description", expected: { totalTB: 2.560, quantity: 10, individualSizeTB: 0.256, needed_description_check: true } },
|
||||
{ title: "Bulk 5 Lot Samsung 870 EVO 500GB SSD SATA - Used - Tested Passed Smart Test", expected: { totalTB: 2.500, quantity: 5, individualSizeTB: 0.500, needed_description_check: false } },
|
||||
{ title: "Samsung 1.6TB NVME PCIe 3.0 x8 2.75\" SSD MZPLK1T6HCHP PM1725 Series TLC", expected: { totalTB: 1.6, quantity: 1, individualSizeTB: 1.6, needed_description_check: false } },
|
||||
{ title: "Micron 5100 MAX 1.84TB SATA 6Gb/s 2.5\" SSD MTFDDAK1T9TCC-1AR1ZABYY", expected: { totalTB: 1.84, quantity: 1, individualSizeTB: 1.84, needed_description_check: false } },
|
||||
{ title: "10-PACK 1TB SSD", expected: { totalTB: 10.0, quantity: 10, individualSizeTB: 1.0, needed_description_check: false } },
|
||||
];
|
||||
let testsPassed = 0;
|
||||
let testsFailed = 0;
|
||||
testCases.forEach((test, index) => {
|
||||
const result = EbayParser.parseSizeAndQuantity(test.title);
|
||||
const tbCheck = Math.abs(result.totalTB - test.expected.totalTB) < 0.0001;
|
||||
const qCheck = result.quantity === test.expected.quantity;
|
||||
const sizeCheck = Math.abs(result.individualSizeTB - test.expected.individualSizeTB) < 0.0001;
|
||||
const needCheck = result.needed_description_check === test.expected.needed_description_check;
|
||||
if (tbCheck && qCheck && sizeCheck && needCheck) testsPassed++;
|
||||
else {
|
||||
error(`Test ${index + 1}: FAILED - "${test.title}"`);
|
||||
error(` Expected: TTB=${test.expected.totalTB.toFixed(4)}, Q=${test.expected.quantity}, STB=${test.expected.individualSizeTB.toFixed(4)}, Check=${test.expected.needed_description_check}`);
|
||||
error(` Actual: TTB=${result.totalTB.toFixed(4)}, Q=${result.quantity}, STB=${result.individualSizeTB.toFixed(4)}, Check=${result.needed_description_check}`);
|
||||
testsFailed++;
|
||||
}
|
||||
});
|
||||
log(`--- Unit Test Summary: ${testsPassed} Passed, ${testsFailed} Failed ---`);
|
||||
return testsFailed === 0;
|
||||
};
|
||||
|
||||
EbayParser.extractDataFromPage = function() {
|
||||
const itemSelector = 'li.s-item, li.srp-results__item, div.s-item[role="listitem"]';
|
||||
const itemElements = document.querySelectorAll(itemSelector);
|
||||
const items = [];
|
||||
const today = new Date().toISOString();
|
||||
|
||||
itemElements.forEach(item => {
|
||||
const titleElement = item.querySelector('.s-item__title, .srp-results__title');
|
||||
const priceElement = item.querySelector('.s-item__price');
|
||||
// const linkElement = item.querySelector('.s-item__link, a[href*="/itm/"]'); // Not used for itemUrl anymore
|
||||
const imageElement = item.querySelector('.s-item__image-wrapper img.s-item__image-img, .s-item__image img'); // Common image selectors
|
||||
|
||||
let rawTitle = titleElement ? titleElement.innerText.trim() : null;
|
||||
const priceText = priceElement ? priceElement.innerText.trim() : null;
|
||||
// const itemUrl = linkElement ? linkElement.href : null; // Removed
|
||||
|
||||
// Try to get image URL, prefer data-src for lazy-loaded images, fallback to src
|
||||
let imageUrl = null;
|
||||
if (imageElement) {
|
||||
imageUrl = imageElement.dataset.src || imageElement.getAttribute('src');
|
||||
}
|
||||
|
||||
|
||||
if (!rawTitle || !priceText) return; // Item ID is now critical, URL was for item ID
|
||||
|
||||
let cleanedTitle = rawTitle;
|
||||
const newListingRegex = /^\s*NEW LISTING\s*[:\-\s]*/i;
|
||||
if (newListingRegex.test(cleanedTitle)) {
|
||||
cleanedTitle = rawTitle.replace(newListingRegex, "").trim();
|
||||
} else if (newListingRegex.test(rawTitle)) {
|
||||
cleanedTitle = rawTitle.replace(newListingRegex, "").trim();
|
||||
}
|
||||
|
||||
const primaryDisplayPrice = EbayParser.parsePrice(priceText);
|
||||
|
||||
let currentBidPrice = null;
|
||||
let finalBuyItNowPrice = null;
|
||||
let hasBestOffer = false;
|
||||
let itemIsAuction = false;
|
||||
|
||||
const bidCountElement = item.querySelector('.s-item__bid-count');
|
||||
if (bidCountElement && bidCountElement.innerText.toLowerCase().includes('bid')) {
|
||||
itemIsAuction = true;
|
||||
}
|
||||
|
||||
const bestOfferElement = item.querySelector('.s-item__purchase-options--bo, .s-item__best-offer');
|
||||
if (bestOfferElement) {
|
||||
hasBestOffer = true;
|
||||
} else {
|
||||
const secondaryInfoElements = item.querySelectorAll('.s-item__subtitle, .s-item__secondary-text, .s-item__detail--secondary');
|
||||
secondaryInfoElements.forEach(el => {
|
||||
if (el.innerText.toLowerCase().includes('or best offer')) {
|
||||
hasBestOffer = true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (itemIsAuction) {
|
||||
currentBidPrice = primaryDisplayPrice;
|
||||
const auctionBinPriceElement = item.querySelector('.s-item__buy-it-now-price');
|
||||
if (auctionBinPriceElement) {
|
||||
finalBuyItNowPrice = EbayParser.parsePrice(auctionBinPriceElement.innerText);
|
||||
}
|
||||
} else {
|
||||
finalBuyItNowPrice = primaryDisplayPrice;
|
||||
}
|
||||
|
||||
const parsedInfo = EbayParser.parseSizeAndQuantity(cleanedTitle);
|
||||
const totalTB = parsedInfo.totalTB;
|
||||
const quantity = parsedInfo.quantity;
|
||||
const individualSizeTB = parsedInfo.individualSizeTB;
|
||||
const needed_description_check = parsedInfo.needed_description_check;
|
||||
|
||||
let costPerTB = null;
|
||||
if (primaryDisplayPrice !== null && totalTB > 0) {
|
||||
costPerTB = primaryDisplayPrice / totalTB;
|
||||
}
|
||||
|
||||
// Extract Item ID from the item's link (still need a link element for this)
|
||||
let itemId = null;
|
||||
const linkForIdElement = item.querySelector('a.s-item__link[href*="/itm/"], .s-item__info > a[href*="/itm/"]');
|
||||
if (linkForIdElement && linkForIdElement.href) {
|
||||
const itemMatch = linkForIdElement.href.match(/\/itm\/(\d+)/);
|
||||
if (itemMatch && itemMatch[1]) {
|
||||
itemId = itemMatch[1];
|
||||
}
|
||||
}
|
||||
if(!itemId) return; // Skip if no item ID can be found, as it's crucial
|
||||
|
||||
items.push({
|
||||
title: cleanedTitle,
|
||||
itemId: itemId, // Crucial
|
||||
dateFound: today,
|
||||
currentBidPrice: currentBidPrice,
|
||||
buyItNowPrice: finalBuyItNowPrice,
|
||||
hasBestOffer: hasBestOffer,
|
||||
image_url: imageUrl, // <-- Added
|
||||
parsed: { // <-- Nested object
|
||||
itemCount: quantity,
|
||||
sizePerItemTB: individualSizeTB > 0 ? parseFloat(individualSizeTB.toFixed(3)) : null,
|
||||
totalTB: totalTB > 0 ? parseFloat(totalTB.toFixed(3)) : null,
|
||||
costPerTB: costPerTB !== null ? parseFloat(costPerTB.toFixed(2)) : null,
|
||||
needed_description_check: needed_description_check,
|
||||
parser_engine: PARSER_ENGINE_VERSION // <-- Added
|
||||
}
|
||||
// itemUrl: itemUrl, // <-- Removed
|
||||
});
|
||||
});
|
||||
return items;
|
||||
};
|
||||
|
||||
return EbayParser;
|
||||
}));
|
||||
253
ebay_storage/javascript/ebay_hdd.js
Normal file
253
ebay_storage/javascript/ebay_hdd.js
Normal file
@ -0,0 +1,253 @@
|
||||
(function() {
|
||||
'use strict';
|
||||
|
||||
// Ensure EbayParser is loaded
|
||||
if (typeof EbayParser === 'undefined') {
|
||||
console.error("Ebay Cost/TB: CRITICAL - ebay_core.js was not loaded. @require path might be incorrect.");
|
||||
return;
|
||||
}
|
||||
|
||||
const DEBUG_MODE = true; // Set to true to run unit tests on load
|
||||
|
||||
// --- Global Variables (UI/State specific) ---
|
||||
let originalOrderMap = new Map();
|
||||
let isSorted = false;
|
||||
let mainListParentElement = null;
|
||||
let observer = null;
|
||||
let observerTargetNode = null;
|
||||
const observerConfig = { childList: true, subtree: true };
|
||||
let processTimer = null;
|
||||
|
||||
// --- UI & DOM Functions ---
|
||||
function addStyles() { /* ... (Keep existing addStyles function) ... */
|
||||
const css = `
|
||||
li.s-item, li.srp-results__item {
|
||||
position: relative !important; overflow: visible !important; padding-bottom: 5px !important;
|
||||
}
|
||||
.cost-per-tb-info {
|
||||
position: absolute; right: 10px; top: 40px;
|
||||
background-color: rgba(0, 100, 0, 0.85); color: white;
|
||||
padding: 4px 8px; border-radius: 5px; z-index: 1000;
|
||||
font-size: 0.9em; font-weight: bold; text-align: center;
|
||||
box-shadow: 1px 1px 3px rgba(0,0,0,0.5); transition: transform 0.2s ease;
|
||||
}
|
||||
.cost-per-tb-info:hover { transform: scale(1.05); }
|
||||
.cost-per-tb-info small.check-desc-note { color: #FFD700; font-weight: bold; }
|
||||
#costPerTbSortControl {
|
||||
display: inline-block; margin-left: 20px; padding: 8px 12px;
|
||||
border: 1px solid #ccc; border-radius: 4px; background-color: #f0f0f0;
|
||||
vertical-align: middle; color: #333; cursor: pointer; font-size: 14px;
|
||||
}
|
||||
#costPerTbSortControl:hover { background-color: #e9e9e9; }
|
||||
#costPerTbSortControl input[type="checkbox"] { margin-right: 8px; vertical-align: middle; }
|
||||
#costPerTbSortControl label { cursor: pointer; vertical-align: middle; color: #333; }
|
||||
`;
|
||||
try {
|
||||
const styleSheet = document.createElement("style");
|
||||
styleSheet.type = "text/css"; styleSheet.innerText = css;
|
||||
document.head.appendChild(styleSheet);
|
||||
} catch (e) { console.error("Ebay Cost/TB: Failed to add styles:", e); }
|
||||
}
|
||||
function getItemSelector() { /* ... (Keep existing getItemSelector function) ... */
|
||||
if (document.querySelector('li.s-item')) return 'li.s-item';
|
||||
if (document.querySelector('li.srp-results__item')) return 'li.srp-results__item';
|
||||
if (document.querySelector('div.s-item')) return 'div.s-item';
|
||||
return 'li[class*="s-item"], div[class*="s-item"]';
|
||||
}
|
||||
function getMainListParent() { /* ... (Keep existing getMainListParent function) ... */
|
||||
if (mainListParentElement && document.body.contains(mainListParentElement)) {
|
||||
return mainListParentElement;
|
||||
}
|
||||
mainListParentElement = document.querySelector('ul.srp-results, div#srp-river-results ul.srp-results');
|
||||
if (!mainListParentElement) {
|
||||
mainListParentElement = document.querySelector('div#srp-river-results ul');
|
||||
}
|
||||
if (!mainListParentElement) {
|
||||
const itemSelector = getItemSelector();
|
||||
const containers = document.querySelectorAll('ul, div');
|
||||
for (const container of containers) {
|
||||
if (container.querySelector(itemSelector)) {
|
||||
const directChildren = Array.from(container.children).filter(child => child.matches(itemSelector));
|
||||
if (directChildren.length > 1) {
|
||||
mainListParentElement = container;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return mainListParentElement;
|
||||
}
|
||||
function storeOriginalOrder(itemsNodeList) { /* ... (Keep existing storeOriginalOrder function) ... */
|
||||
if (originalOrderMap.size > 0 && itemsNodeList.length === originalOrderMap.size) {
|
||||
return;
|
||||
}
|
||||
originalOrderMap.clear();
|
||||
itemsNodeList.forEach((itemElement, index) => {
|
||||
let itemId = itemElement.id;
|
||||
if (!itemId) {
|
||||
const linkWithItemId = itemElement.querySelector('a[href*="/itm/"]');
|
||||
if (linkWithItemId && linkWithItemId.href) {
|
||||
const match = linkWithItemId.href.match(/\/itm\/(\d+)/);
|
||||
if (match && match[1]) {
|
||||
itemId = "ebayitem_" + match[1]; itemElement.id = itemId;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (itemId && !originalOrderMap.has(itemId)) originalOrderMap.set(itemId, index);
|
||||
});
|
||||
}
|
||||
|
||||
function processResults() {
|
||||
const parent = getMainListParent();
|
||||
if (!parent) return;
|
||||
|
||||
const itemSelector = getItemSelector();
|
||||
const itemsOnPage = parent.querySelectorAll(itemSelector + ':not(.cost-per-tb-processed-flag)');
|
||||
|
||||
itemsOnPage.forEach(item => {
|
||||
const titleElement = item.querySelector('.s-item__title, .srp-results__title');
|
||||
const priceElement = item.querySelector('.s-item__price, .srp-results__price');
|
||||
|
||||
if (titleElement && priceElement) {
|
||||
const title = titleElement.innerText;
|
||||
const priceText = priceElement.innerText;
|
||||
// --- Use Core Parser ---
|
||||
const price = EbayParser.parsePrice(priceText);
|
||||
const parsedInfo = EbayParser.parseSizeAndQuantity(title);
|
||||
// -----------------------
|
||||
const totalTB = parsedInfo.totalTB;
|
||||
const needed_description_check = parsedInfo.needed_description_check;
|
||||
|
||||
item.dataset.neededDescriptionCheck = String(needed_description_check);
|
||||
|
||||
let displayElement = item.querySelector('.cost-per-tb-info');
|
||||
if (!displayElement) {
|
||||
displayElement = document.createElement('div');
|
||||
displayElement.className = 'cost-per-tb-info';
|
||||
const imageContainer = item.querySelector('.s-item__image-section, .s-item__image');
|
||||
if (imageContainer && imageContainer.parentNode) {
|
||||
imageContainer.parentNode.style.position = 'relative';
|
||||
imageContainer.parentNode.appendChild(displayElement);
|
||||
} else { item.appendChild(displayElement); }
|
||||
}
|
||||
|
||||
if (price !== null && totalTB > 0) {
|
||||
const costPerTB = price / totalTB;
|
||||
if (needed_description_check) {
|
||||
item.dataset.costPerTb = '9999999';
|
||||
displayElement.innerHTML = `$${costPerTB.toFixed(2)}*<span> / TB</span><br><small class="check-desc-note">(${totalTB.toFixed(2)} TB, Check Desc.)</small>`;
|
||||
} else {
|
||||
item.dataset.costPerTb = costPerTB;
|
||||
displayElement.innerHTML = `$${costPerTB.toFixed(2)}<span> / TB</span><br><small>(${totalTB.toFixed(2)} TB)</small>`;
|
||||
}
|
||||
} else {
|
||||
item.dataset.costPerTb = '9999999';
|
||||
let ambiguousNote = needed_description_check || (parsedInfo.quantity > 1 && totalTB === 0) ? "(Check Desc.)" : "(Details N/A)";
|
||||
let tbDisplay = totalTB > 0 ? `(${totalTB.toFixed(2)} TB, Price N/A)` : ambiguousNote;
|
||||
if (totalTB === 0) tbDisplay = ambiguousNote;
|
||||
displayElement.innerHTML = `<i>Details unclear</i><br><small class="check-desc-note">${tbDisplay}</small>`;
|
||||
}
|
||||
|
||||
if(displayElement.querySelector('span')) displayElement.querySelector('span').style.fontSize = '0.8em';
|
||||
if(displayElement.querySelector('small')) displayElement.querySelector('small').style.fontSize = '0.7em';
|
||||
|
||||
} else { item.dataset.costPerTb = '9999999'; }
|
||||
item.classList.add('cost-per-tb-processed-flag');
|
||||
item.dataset.costPerTbProcessed = 'true';
|
||||
});
|
||||
|
||||
const allItems = parent.querySelectorAll(itemSelector);
|
||||
if (originalOrderMap.size === 0 || Math.abs(originalOrderMap.size - allItems.length) > 5) {
|
||||
if (allItems.length > 0) storeOriginalOrder(allItems);
|
||||
}
|
||||
if (isSorted) sortResults();
|
||||
}
|
||||
|
||||
function sortResults() { /* ... (Keep existing sortResults function) ... */
|
||||
const parent = getMainListParent(); if (!parent) return;
|
||||
const itemSelector = getItemSelector(); let itemsArray = Array.from(parent.querySelectorAll(itemSelector));
|
||||
if (originalOrderMap.size === 0 && itemsArray.length > 0) storeOriginalOrder(itemsArray);
|
||||
itemsArray.sort((a, b) => parseFloat(a.dataset.costPerTb || '9999999') - parseFloat(b.dataset.costPerTb || '9999999'));
|
||||
if (observer) observer.disconnect(); itemsArray.forEach(item => parent.appendChild(item));
|
||||
if (observer && observerTargetNode) observer.observe(observerTargetNode, observerConfig);
|
||||
}
|
||||
function restoreOriginalOrder() { /* ... (Keep existing restoreOriginalOrder function) ... */
|
||||
const parent = getMainListParent(); if (!parent) return; if (originalOrderMap.size === 0) return;
|
||||
let itemsToReorder = [];
|
||||
originalOrderMap.forEach((originalIndex, itemId) => {
|
||||
const itemElement = document.getElementById(itemId) || parent.querySelector(`[id="${itemId}"]`);
|
||||
if (itemElement && parent.contains(itemElement)) itemsToReorder.push({ element: itemElement, originalIndex: originalIndex });
|
||||
});
|
||||
const itemSelector = getItemSelector(); const currentItemsInDOM = parent.querySelectorAll(itemSelector); let newItemsCount = 0;
|
||||
currentItemsInDOM.forEach(domItem => {
|
||||
if (domItem.id && !originalOrderMap.has(domItem.id)) itemsToReorder.push({ element: domItem, originalIndex: 900000 + newItemsCount++ });
|
||||
});
|
||||
itemsToReorder.sort((a, b) => a.originalIndex - b.originalIndex);
|
||||
if (observer) observer.disconnect(); itemsToReorder.forEach(entry => parent.appendChild(entry.element));
|
||||
if (observer && observerTargetNode) observer.observe(observerTargetNode, observerConfig);
|
||||
}
|
||||
function handleSortToggle(event) { /* ... (Keep existing handleSortToggle function) ... */
|
||||
isSorted = event.currentTarget.checked;
|
||||
if (isSorted) sortResults(); else restoreOriginalOrder();
|
||||
}
|
||||
function addSortControl() { /* ... (Keep existing addSortControl function) ... */
|
||||
let controlsContainer = document.querySelector('.srp-controls__sort- σήμερα') || document.querySelector('.srp-controls__sort') || document.querySelector('div[class*="srp-controls__sort"]');
|
||||
if (!controlsContainer) controlsContainer = document.querySelector('.srp-sort, .srp-controls');
|
||||
if (!controlsContainer || document.getElementById('costPerTbSortControlWrapper')) return;
|
||||
const controlWrapper = document.createElement('div'); controlWrapper.id = 'costPerTbSortControlWrapper'; controlWrapper.style.display = 'inline-block'; controlWrapper.style.marginLeft = '20px'; controlWrapper.style.verticalAlign = 'middle';
|
||||
const controlDiv = document.createElement('div'); controlDiv.id = 'costPerTbSortControl';
|
||||
const checkbox = document.createElement('input'); checkbox.type = 'checkbox'; checkbox.id = 'costPerTbSortCheckbox'; checkbox.checked = isSorted; checkbox.style.verticalAlign = 'middle';
|
||||
const label = document.createElement('label'); label.htmlFor = 'costPerTbSortCheckbox'; label.innerText = 'Sort by $/TB'; label.style.verticalAlign = 'middle'; label.style.marginLeft = '5px';
|
||||
controlDiv.appendChild(checkbox); controlDiv.appendChild(label); controlWrapper.appendChild(controlDiv);
|
||||
controlDiv.addEventListener('click', (e) => { if (e.target !== checkbox) { checkbox.checked = !checkbox.checked; checkbox.dispatchEvent(new Event('change', { bubbles: true })); } });
|
||||
checkbox.addEventListener('change', handleSortToggle);
|
||||
if (controlsContainer.nextSibling) controlsContainer.parentNode.insertBefore(controlWrapper, controlsContainer.nextSibling);
|
||||
else controlsContainer.parentNode.appendChild(controlWrapper);
|
||||
}
|
||||
function setupObserver() { /* ... (Keep existing setupObserver function but without needsProcessing/callback logic, just call process/addControl) ... */
|
||||
observerTargetNode = getMainListParent() || document.body; if (!observerTargetNode) return;
|
||||
const itemSelector = getItemSelector();
|
||||
const callback = function(mutationsList, obs) {
|
||||
let needsProcessing = false;
|
||||
for(const mutation of mutationsList) {
|
||||
if (mutation.type === 'childList') {
|
||||
if (mutation.addedNodes.length > 0) {
|
||||
mutation.addedNodes.forEach(node => {
|
||||
if (node.nodeType === 1 && !node.classList.contains('cost-per-tb-info') && (node.matches && node.matches(itemSelector) || (node.querySelector && node.querySelector(itemSelector)))) {
|
||||
needsProcessing = true;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!document.getElementById('costPerTbSortControlWrapper') && document.querySelector('.srp-controls__sort')) {
|
||||
needsProcessing = true;
|
||||
}
|
||||
}
|
||||
if (needsProcessing) {
|
||||
clearTimeout(processTimer);
|
||||
processTimer = setTimeout(() => { processResults(); addSortControl(); }, 750);
|
||||
}
|
||||
};
|
||||
if (observer) observer.disconnect(); observer = new MutationObserver(callback);
|
||||
observer.observe(observerTargetNode, observerConfig);
|
||||
}
|
||||
|
||||
// --- Main Execution ---
|
||||
function init() {
|
||||
console.log("Ebay Cost/TB V2.7 (Refactored) starting...");
|
||||
if (DEBUG_MODE) {
|
||||
// --- Use Core Unit Tests ---
|
||||
EbayParser.runUnitTests();
|
||||
// --------------------------
|
||||
}
|
||||
addStyles();
|
||||
setTimeout(() => {
|
||||
processResults();
|
||||
addSortControl();
|
||||
setupObserver();
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
init(); // Run-at document-idle handles load state
|
||||
|
||||
})();
|
||||
13
ebay_storage/javascript/package.json
Normal file
13
ebay_storage/javascript/package.json
Normal file
@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "greasemonkey",
|
||||
"version": "1.0.0",
|
||||
"main": "ebay_command_line_tool.js",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"puppeteer": "^24.9.0",
|
||||
"commander": "^14.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"scrape": "node ebay_command_line_tool.js"
|
||||
}
|
||||
}
|
||||
50
ebay_storage/javascript/readme.md
Normal file
50
ebay_storage/javascript/readme.md
Normal file
@ -0,0 +1,50 @@
|
||||
# Greasemonkey scripts
|
||||
|
||||
## Ebay Scraper (Storage)
|
||||
|
||||
A truly awful *very* LLM generated scraping tool used to help me find good deals on ebay for Storage. Again, this is LLM generated, basically vibe coded, so the code quality has virtually zero oversight. The generation of this was done using Gemini 2.5 Pro, see the [first](https://g.co/gemini/share/bf17780ad083) and [second](https://g.co/gemini/share/3d80b96e42e9) conversations used to generate this.
|
||||
|
||||
|
||||
|
||||
```bash
|
||||
greasemonkey on master is 📦 v1.0.0 via ⬢ v23.11.1 at ☸ default took 7s 593ms
|
||||
➜ yarn --silent scrape --help
|
||||
Usage: ebay-scraper [options] [command] [url]
|
||||
|
||||
Scrapes eBay search results for SSD/HDD cost per TB.
|
||||
|
||||
Arguments:
|
||||
url The full eBay search URL to scrape.
|
||||
|
||||
Options:
|
||||
-V, --version output the version number
|
||||
--save <filename> Save the scraped HTML to a file.
|
||||
--load <filename> Load HTML from a file instead of fetching from eBay (disables network).
|
||||
--only_json Suppress all informational logs and output only the final JSON. (default: false)
|
||||
-h, --help display help for command
|
||||
|
||||
Commands:
|
||||
latest [options] Scrapes the latest listings using a predefined search. Use "ebay-scraper latest --help" to see specific options for this
|
||||
command.
|
||||
|
||||
Example calls:
|
||||
$ ebay-scraper latest --per_page 120 --minimum_cost 50
|
||||
$ ebay-scraper latest --help
|
||||
$ ebay-scraper "https://www.ebay.com/sch/i.html?_nkw=ssd"
|
||||
$ ebay-scraper --load saved_page.html --only_json | jq .
|
||||
$ ebay-scraper --save current_page.html "https://www.ebay.com/sch/i.html?_nkw=hdd"
|
||||
```
|
||||
|
||||
|
||||
```bash
|
||||
greasemonkey on master is 📦 v1.0.0 via ⬢ v23.11.1 at ☸ default
|
||||
➜ yarn --silent scrape latest --help
|
||||
Usage: ebay-scraper latest [options]
|
||||
|
||||
Scrapes the latest listings using a predefined search. Use "ebay-scraper latest --help" to see specific options for this command.
|
||||
|
||||
Options:
|
||||
--per_page <number> Items per page (60, 120, or 240) (default: "60")
|
||||
--minimum_cost <number> Minimum cost for listings (e.g., 50.00) (default: "0.00")
|
||||
-h, --help display help for command
|
||||
```
|
||||
BIN
ebay_storage/javascript/scraper_chrome.png
Normal file
BIN
ebay_storage/javascript/scraper_chrome.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 442 KiB |
671
ebay_storage/javascript/yarn.lock
Normal file
671
ebay_storage/javascript/yarn.lock
Normal file
@ -0,0 +1,671 @@
|
||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||
# yarn lockfile v1
|
||||
|
||||
|
||||
"@babel/code-frame@^7.0.0":
|
||||
version "7.27.1"
|
||||
resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.27.1.tgz#200f715e66d52a23b221a9435534a91cc13ad5be"
|
||||
integrity sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==
|
||||
dependencies:
|
||||
"@babel/helper-validator-identifier" "^7.27.1"
|
||||
js-tokens "^4.0.0"
|
||||
picocolors "^1.1.1"
|
||||
|
||||
"@babel/helper-validator-identifier@^7.27.1":
|
||||
version "7.27.1"
|
||||
resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz#a7054dcc145a967dd4dc8fee845a57c1316c9df8"
|
||||
integrity sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==
|
||||
|
||||
"@puppeteer/browsers@2.10.5":
|
||||
version "2.10.5"
|
||||
resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.10.5.tgz#dddb8f8716ae6364f6f2d31125e76f311dd4a49d"
|
||||
integrity sha512-eifa0o+i8dERnngJwKrfp3dEq7ia5XFyoqB17S4gK8GhsQE4/P8nxOfQSE0zQHxzzLo/cmF+7+ywEQ7wK7Fb+w==
|
||||
dependencies:
|
||||
debug "^4.4.1"
|
||||
extract-zip "^2.0.1"
|
||||
progress "^2.0.3"
|
||||
proxy-agent "^6.5.0"
|
||||
semver "^7.7.2"
|
||||
tar-fs "^3.0.8"
|
||||
yargs "^17.7.2"
|
||||
|
||||
"@tootallnate/quickjs-emscripten@^0.23.0":
|
||||
version "0.23.0"
|
||||
resolved "https://registry.yarnpkg.com/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz#db4ecfd499a9765ab24002c3b696d02e6d32a12c"
|
||||
integrity sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==
|
||||
|
||||
"@types/node@*":
|
||||
version "22.15.23"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-22.15.23.tgz#a0b7c03f951f1ffe381a6a345c68d80e48043dd0"
|
||||
integrity sha512-7Ec1zaFPF4RJ0eXu1YT/xgiebqwqoJz8rYPDi/O2BcZ++Wpt0Kq9cl0eg6NN6bYbPnR67ZLo7St5Q3UK0SnARw==
|
||||
dependencies:
|
||||
undici-types "~6.21.0"
|
||||
|
||||
"@types/yauzl@^2.9.1":
|
||||
version "2.10.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999"
|
||||
integrity sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
agent-base@^7.1.0, agent-base@^7.1.2:
|
||||
version "7.1.3"
|
||||
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.3.tgz#29435eb821bc4194633a5b89e5bc4703bafc25a1"
|
||||
integrity sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==
|
||||
|
||||
ansi-regex@^5.0.1:
|
||||
version "5.0.1"
|
||||
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
|
||||
integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==
|
||||
|
||||
ansi-styles@^4.0.0:
|
||||
version "4.3.0"
|
||||
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937"
|
||||
integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==
|
||||
dependencies:
|
||||
color-convert "^2.0.1"
|
||||
|
||||
argparse@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
|
||||
integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
|
||||
|
||||
ast-types@^0.13.4:
|
||||
version "0.13.4"
|
||||
resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.13.4.tgz#ee0d77b343263965ecc3fb62da16e7222b2b6782"
|
||||
integrity sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==
|
||||
dependencies:
|
||||
tslib "^2.0.1"
|
||||
|
||||
b4a@^1.6.4:
|
||||
version "1.6.7"
|
||||
resolved "https://registry.yarnpkg.com/b4a/-/b4a-1.6.7.tgz#a99587d4ebbfbd5a6e3b21bdb5d5fa385767abe4"
|
||||
integrity sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==
|
||||
|
||||
bare-events@^2.2.0, bare-events@^2.5.4:
|
||||
version "2.5.4"
|
||||
resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.5.4.tgz#16143d435e1ed9eafd1ab85f12b89b3357a41745"
|
||||
integrity sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==
|
||||
|
||||
bare-fs@^4.0.1:
|
||||
version "4.1.5"
|
||||
resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-4.1.5.tgz#1d06c076e68cc8bf97010d29af9e3ac3808cdcf7"
|
||||
integrity sha512-1zccWBMypln0jEE05LzZt+V/8y8AQsQQqxtklqaIyg5nu6OAYFhZxPXinJTSG+kU5qyNmeLgcn9AW7eHiCHVLA==
|
||||
dependencies:
|
||||
bare-events "^2.5.4"
|
||||
bare-path "^3.0.0"
|
||||
bare-stream "^2.6.4"
|
||||
|
||||
bare-os@^3.0.1:
|
||||
version "3.6.1"
|
||||
resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-3.6.1.tgz#9921f6f59edbe81afa9f56910658422c0f4858d4"
|
||||
integrity sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==
|
||||
|
||||
bare-path@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-3.0.0.tgz#b59d18130ba52a6af9276db3e96a2e3d3ea52178"
|
||||
integrity sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==
|
||||
dependencies:
|
||||
bare-os "^3.0.1"
|
||||
|
||||
bare-stream@^2.6.4:
|
||||
version "2.6.5"
|
||||
resolved "https://registry.yarnpkg.com/bare-stream/-/bare-stream-2.6.5.tgz#bba8e879674c4c27f7e27805df005c15d7a2ca07"
|
||||
integrity sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==
|
||||
dependencies:
|
||||
streamx "^2.21.0"
|
||||
|
||||
basic-ftp@^5.0.2:
|
||||
version "5.0.5"
|
||||
resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.0.5.tgz#14a474f5fffecca1f4f406f1c26b18f800225ac0"
|
||||
integrity sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==
|
||||
|
||||
buffer-crc32@~0.2.3:
|
||||
version "0.2.13"
|
||||
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
|
||||
integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==
|
||||
|
||||
callsites@^3.0.0:
|
||||
version "3.1.0"
|
||||
resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
|
||||
integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
|
||||
|
||||
chromium-bidi@5.1.0:
|
||||
version "5.1.0"
|
||||
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-5.1.0.tgz#8d0e47f7ac9270262df29792318dd5378e983e62"
|
||||
integrity sha512-9MSRhWRVoRPDG0TgzkHrshFSJJNZzfY5UFqUMuksg7zL1yoZIZ3jLB0YAgHclbiAxPI86pBnwDX1tbzoiV8aFw==
|
||||
dependencies:
|
||||
mitt "^3.0.1"
|
||||
zod "^3.24.1"
|
||||
|
||||
cliui@^8.0.1:
|
||||
version "8.0.1"
|
||||
resolved "https://registry.yarnpkg.com/cliui/-/cliui-8.0.1.tgz#0c04b075db02cbfe60dc8e6cf2f5486b1a3608aa"
|
||||
integrity sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==
|
||||
dependencies:
|
||||
string-width "^4.2.0"
|
||||
strip-ansi "^6.0.1"
|
||||
wrap-ansi "^7.0.0"
|
||||
|
||||
color-convert@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
|
||||
integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==
|
||||
dependencies:
|
||||
color-name "~1.1.4"
|
||||
|
||||
color-name@~1.1.4:
|
||||
version "1.1.4"
|
||||
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
|
||||
integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
|
||||
|
||||
commander@^14.0.0:
|
||||
version "14.0.0"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.0.tgz#f244fc74a92343514e56229f16ef5c5e22ced5e9"
|
||||
integrity sha512-2uM9rYjPvyq39NwLRqaiLtWHyDC1FvryJDa2ATTVims5YAS4PupsEQsDvP14FqhFr0P49CYDugi59xaxJlTXRA==
|
||||
|
||||
cosmiconfig@^9.0.0:
|
||||
version "9.0.0"
|
||||
resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-9.0.0.tgz#34c3fc58287b915f3ae905ab6dc3de258b55ad9d"
|
||||
integrity sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==
|
||||
dependencies:
|
||||
env-paths "^2.2.1"
|
||||
import-fresh "^3.3.0"
|
||||
js-yaml "^4.1.0"
|
||||
parse-json "^5.2.0"
|
||||
|
||||
data-uri-to-buffer@^6.0.2:
|
||||
version "6.0.2"
|
||||
resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz#8a58bb67384b261a38ef18bea1810cb01badd28b"
|
||||
integrity sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==
|
||||
|
||||
debug@4, debug@^4.1.1, debug@^4.3.4, debug@^4.4.1:
|
||||
version "4.4.1"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.1.tgz#e5a8bc6cbc4c6cd3e64308b0693a3d4fa550189b"
|
||||
integrity sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==
|
||||
dependencies:
|
||||
ms "^2.1.3"
|
||||
|
||||
degenerator@^5.0.0:
|
||||
version "5.0.1"
|
||||
resolved "https://registry.yarnpkg.com/degenerator/-/degenerator-5.0.1.tgz#9403bf297c6dad9a1ece409b37db27954f91f2f5"
|
||||
integrity sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==
|
||||
dependencies:
|
||||
ast-types "^0.13.4"
|
||||
escodegen "^2.1.0"
|
||||
esprima "^4.0.1"
|
||||
|
||||
devtools-protocol@0.0.1439962:
|
||||
version "0.0.1439962"
|
||||
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1439962.tgz#395c5ca1cd83aa451c667056a025f9873c4598c1"
|
||||
integrity sha512-jJF48UdryzKiWhJ1bLKr7BFWUQCEIT5uCNbDLqkQJBtkFxYzILJH44WN0PDKMIlGDN7Utb8vyUY85C3w4R/t2g==
|
||||
|
||||
emoji-regex@^8.0.0:
|
||||
version "8.0.0"
|
||||
resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
|
||||
integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
|
||||
|
||||
end-of-stream@^1.1.0:
|
||||
version "1.4.4"
|
||||
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
|
||||
integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
|
||||
dependencies:
|
||||
once "^1.4.0"
|
||||
|
||||
env-paths@^2.2.1:
|
||||
version "2.2.1"
|
||||
resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2"
|
||||
integrity sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==
|
||||
|
||||
error-ex@^1.3.1:
|
||||
version "1.3.2"
|
||||
resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf"
|
||||
integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==
|
||||
dependencies:
|
||||
is-arrayish "^0.2.1"
|
||||
|
||||
escalade@^3.1.1:
|
||||
version "3.2.0"
|
||||
resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5"
|
||||
integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==
|
||||
|
||||
escodegen@^2.1.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.1.0.tgz#ba93bbb7a43986d29d6041f99f5262da773e2e17"
|
||||
integrity sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==
|
||||
dependencies:
|
||||
esprima "^4.0.1"
|
||||
estraverse "^5.2.0"
|
||||
esutils "^2.0.2"
|
||||
optionalDependencies:
|
||||
source-map "~0.6.1"
|
||||
|
||||
esprima@^4.0.1:
|
||||
version "4.0.1"
|
||||
resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
|
||||
integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
|
||||
|
||||
estraverse@^5.2.0:
|
||||
version "5.3.0"
|
||||
resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123"
|
||||
integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==
|
||||
|
||||
esutils@^2.0.2:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
|
||||
integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==
|
||||
|
||||
extract-zip@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a"
|
||||
integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==
|
||||
dependencies:
|
||||
debug "^4.1.1"
|
||||
get-stream "^5.1.0"
|
||||
yauzl "^2.10.0"
|
||||
optionalDependencies:
|
||||
"@types/yauzl" "^2.9.1"
|
||||
|
||||
fast-fifo@^1.2.0, fast-fifo@^1.3.2:
|
||||
version "1.3.2"
|
||||
resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.2.tgz#286e31de96eb96d38a97899815740ba2a4f3640c"
|
||||
integrity sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==
|
||||
|
||||
fd-slicer@~1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e"
|
||||
integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==
|
||||
dependencies:
|
||||
pend "~1.2.0"
|
||||
|
||||
get-caller-file@^2.0.5:
|
||||
version "2.0.5"
|
||||
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
|
||||
integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==
|
||||
|
||||
get-stream@^5.1.0:
|
||||
version "5.2.0"
|
||||
resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3"
|
||||
integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==
|
||||
dependencies:
|
||||
pump "^3.0.0"
|
||||
|
||||
get-uri@^6.0.1:
|
||||
version "6.0.4"
|
||||
resolved "https://registry.yarnpkg.com/get-uri/-/get-uri-6.0.4.tgz#6daaee9e12f9759e19e55ba313956883ef50e0a7"
|
||||
integrity sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==
|
||||
dependencies:
|
||||
basic-ftp "^5.0.2"
|
||||
data-uri-to-buffer "^6.0.2"
|
||||
debug "^4.3.4"
|
||||
|
||||
http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1:
|
||||
version "7.0.2"
|
||||
resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e"
|
||||
integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==
|
||||
dependencies:
|
||||
agent-base "^7.1.0"
|
||||
debug "^4.3.4"
|
||||
|
||||
https-proxy-agent@^7.0.6:
|
||||
version "7.0.6"
|
||||
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9"
|
||||
integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==
|
||||
dependencies:
|
||||
agent-base "^7.1.2"
|
||||
debug "4"
|
||||
|
||||
import-fresh@^3.3.0:
|
||||
version "3.3.1"
|
||||
resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.1.tgz#9cecb56503c0ada1f2741dbbd6546e4b13b57ccf"
|
||||
integrity sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==
|
||||
dependencies:
|
||||
parent-module "^1.0.0"
|
||||
resolve-from "^4.0.0"
|
||||
|
||||
ip-address@^9.0.5:
|
||||
version "9.0.5"
|
||||
resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a"
|
||||
integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==
|
||||
dependencies:
|
||||
jsbn "1.1.0"
|
||||
sprintf-js "^1.1.3"
|
||||
|
||||
is-arrayish@^0.2.1:
|
||||
version "0.2.1"
|
||||
resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
|
||||
integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==
|
||||
|
||||
is-fullwidth-code-point@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
|
||||
integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
|
||||
|
||||
js-tokens@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
|
||||
integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==
|
||||
|
||||
js-yaml@^4.1.0:
|
||||
version "4.1.0"
|
||||
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602"
|
||||
integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==
|
||||
dependencies:
|
||||
argparse "^2.0.1"
|
||||
|
||||
jsbn@1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040"
|
||||
integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==
|
||||
|
||||
json-parse-even-better-errors@^2.3.0:
|
||||
version "2.3.1"
|
||||
resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d"
|
||||
integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==
|
||||
|
||||
lines-and-columns@^1.1.6:
|
||||
version "1.2.4"
|
||||
resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
|
||||
integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
|
||||
|
||||
lru-cache@^7.14.1:
|
||||
version "7.18.3"
|
||||
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89"
|
||||
integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==
|
||||
|
||||
mitt@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/mitt/-/mitt-3.0.1.tgz#ea36cf0cc30403601ae074c8f77b7092cdab36d1"
|
||||
integrity sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==
|
||||
|
||||
ms@^2.1.3:
|
||||
version "2.1.3"
|
||||
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
|
||||
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
|
||||
|
||||
netmask@^2.0.2:
|
||||
version "2.0.2"
|
||||
resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.0.2.tgz#8b01a07644065d536383835823bc52004ebac5e7"
|
||||
integrity sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==
|
||||
|
||||
once@^1.3.1, once@^1.4.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
|
||||
integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==
|
||||
dependencies:
|
||||
wrappy "1"
|
||||
|
||||
pac-proxy-agent@^7.1.0:
|
||||
version "7.2.0"
|
||||
resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz#9cfaf33ff25da36f6147a20844230ec92c06e5df"
|
||||
integrity sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==
|
||||
dependencies:
|
||||
"@tootallnate/quickjs-emscripten" "^0.23.0"
|
||||
agent-base "^7.1.2"
|
||||
debug "^4.3.4"
|
||||
get-uri "^6.0.1"
|
||||
http-proxy-agent "^7.0.0"
|
||||
https-proxy-agent "^7.0.6"
|
||||
pac-resolver "^7.0.1"
|
||||
socks-proxy-agent "^8.0.5"
|
||||
|
||||
pac-resolver@^7.0.1:
|
||||
version "7.0.1"
|
||||
resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-7.0.1.tgz#54675558ea368b64d210fd9c92a640b5f3b8abb6"
|
||||
integrity sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==
|
||||
dependencies:
|
||||
degenerator "^5.0.0"
|
||||
netmask "^2.0.2"
|
||||
|
||||
parent-module@^1.0.0:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2"
|
||||
integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==
|
||||
dependencies:
|
||||
callsites "^3.0.0"
|
||||
|
||||
parse-json@^5.2.0:
|
||||
version "5.2.0"
|
||||
resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd"
|
||||
integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==
|
||||
dependencies:
|
||||
"@babel/code-frame" "^7.0.0"
|
||||
error-ex "^1.3.1"
|
||||
json-parse-even-better-errors "^2.3.0"
|
||||
lines-and-columns "^1.1.6"
|
||||
|
||||
pend@~1.2.0:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
|
||||
integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==
|
||||
|
||||
picocolors@^1.1.1:
|
||||
version "1.1.1"
|
||||
resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b"
|
||||
integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==
|
||||
|
||||
progress@^2.0.3:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
|
||||
integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
|
||||
|
||||
proxy-agent@^6.5.0:
|
||||
version "6.5.0"
|
||||
resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.5.0.tgz#9e49acba8e4ee234aacb539f89ed9c23d02f232d"
|
||||
integrity sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==
|
||||
dependencies:
|
||||
agent-base "^7.1.2"
|
||||
debug "^4.3.4"
|
||||
http-proxy-agent "^7.0.1"
|
||||
https-proxy-agent "^7.0.6"
|
||||
lru-cache "^7.14.1"
|
||||
pac-proxy-agent "^7.1.0"
|
||||
proxy-from-env "^1.1.0"
|
||||
socks-proxy-agent "^8.0.5"
|
||||
|
||||
proxy-from-env@^1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
|
||||
integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
|
||||
|
||||
pump@^3.0.0:
|
||||
version "3.0.2"
|
||||
resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.2.tgz#836f3edd6bc2ee599256c924ffe0d88573ddcbf8"
|
||||
integrity sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==
|
||||
dependencies:
|
||||
end-of-stream "^1.1.0"
|
||||
once "^1.3.1"
|
||||
|
||||
puppeteer-core@24.9.0:
|
||||
version "24.9.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.9.0.tgz#fc489e83bf65db1dc72e53a78140ee567efd847e"
|
||||
integrity sha512-HFdCeH/wx6QPz8EncafbCqJBqaCG1ENW75xg3cLFMRUoqZDgByT6HSueiumetT2uClZxwqj0qS4qMVZwLHRHHw==
|
||||
dependencies:
|
||||
"@puppeteer/browsers" "2.10.5"
|
||||
chromium-bidi "5.1.0"
|
||||
debug "^4.4.1"
|
||||
devtools-protocol "0.0.1439962"
|
||||
typed-query-selector "^2.12.0"
|
||||
ws "^8.18.2"
|
||||
|
||||
puppeteer@^24.9.0:
|
||||
version "24.9.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-24.9.0.tgz#1d3f805e0170ca481b637a47c71a09b815594dae"
|
||||
integrity sha512-L0pOtALIx8rgDt24Y+COm8X52v78gNtBOW6EmUcEPci0TYD72SAuaXKqasRIx4JXxmg2Tkw5ySKcpPOwN8xXnQ==
|
||||
dependencies:
|
||||
"@puppeteer/browsers" "2.10.5"
|
||||
chromium-bidi "5.1.0"
|
||||
cosmiconfig "^9.0.0"
|
||||
devtools-protocol "0.0.1439962"
|
||||
puppeteer-core "24.9.0"
|
||||
typed-query-selector "^2.12.0"
|
||||
|
||||
require-directory@^2.1.1:
|
||||
version "2.1.1"
|
||||
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
|
||||
integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==
|
||||
|
||||
resolve-from@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
|
||||
integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
|
||||
|
||||
semver@^7.7.2:
|
||||
version "7.7.2"
|
||||
resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.2.tgz#67d99fdcd35cec21e6f8b87a7fd515a33f982b58"
|
||||
integrity sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==
|
||||
|
||||
smart-buffer@^4.2.0:
|
||||
version "4.2.0"
|
||||
resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae"
|
||||
integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==
|
||||
|
||||
socks-proxy-agent@^8.0.5:
|
||||
version "8.0.5"
|
||||
resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz#b9cdb4e7e998509d7659d689ce7697ac21645bee"
|
||||
integrity sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==
|
||||
dependencies:
|
||||
agent-base "^7.1.2"
|
||||
debug "^4.3.4"
|
||||
socks "^2.8.3"
|
||||
|
||||
socks@^2.8.3:
|
||||
version "2.8.4"
|
||||
resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.4.tgz#07109755cdd4da03269bda4725baa061ab56d5cc"
|
||||
integrity sha512-D3YaD0aRxR3mEcqnidIs7ReYJFVzWdd6fXJYUM8ixcQcJRGTka/b3saV0KflYhyVJXKhb947GndU35SxYNResQ==
|
||||
dependencies:
|
||||
ip-address "^9.0.5"
|
||||
smart-buffer "^4.2.0"
|
||||
|
||||
source-map@~0.6.1:
|
||||
version "0.6.1"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
|
||||
integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
|
||||
|
||||
sprintf-js@^1.1.3:
|
||||
version "1.1.3"
|
||||
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a"
|
||||
integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==
|
||||
|
||||
streamx@^2.15.0, streamx@^2.21.0:
|
||||
version "2.22.0"
|
||||
resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.22.0.tgz#cd7b5e57c95aaef0ff9b2aef7905afa62ec6e4a7"
|
||||
integrity sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==
|
||||
dependencies:
|
||||
fast-fifo "^1.3.2"
|
||||
text-decoder "^1.1.0"
|
||||
optionalDependencies:
|
||||
bare-events "^2.2.0"
|
||||
|
||||
string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
|
||||
version "4.2.3"
|
||||
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
|
||||
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
|
||||
dependencies:
|
||||
emoji-regex "^8.0.0"
|
||||
is-fullwidth-code-point "^3.0.0"
|
||||
strip-ansi "^6.0.1"
|
||||
|
||||
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
|
||||
version "6.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
|
||||
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
|
||||
dependencies:
|
||||
ansi-regex "^5.0.1"
|
||||
|
||||
tar-fs@^3.0.8:
|
||||
version "3.0.9"
|
||||
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.9.tgz#d570793c6370d7078926c41fa422891566a0b617"
|
||||
integrity sha512-XF4w9Xp+ZQgifKakjZYmFdkLoSWd34VGKcsTCwlNWM7QG3ZbaxnTsaBwnjFZqHRf/rROxaR8rXnbtwdvaDI+lA==
|
||||
dependencies:
|
||||
pump "^3.0.0"
|
||||
tar-stream "^3.1.5"
|
||||
optionalDependencies:
|
||||
bare-fs "^4.0.1"
|
||||
bare-path "^3.0.0"
|
||||
|
||||
tar-stream@^3.1.5:
|
||||
version "3.1.7"
|
||||
resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.7.tgz#24b3fb5eabada19fe7338ed6d26e5f7c482e792b"
|
||||
integrity sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==
|
||||
dependencies:
|
||||
b4a "^1.6.4"
|
||||
fast-fifo "^1.2.0"
|
||||
streamx "^2.15.0"
|
||||
|
||||
text-decoder@^1.1.0:
|
||||
version "1.2.3"
|
||||
resolved "https://registry.yarnpkg.com/text-decoder/-/text-decoder-1.2.3.tgz#b19da364d981b2326d5f43099c310cc80d770c65"
|
||||
integrity sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==
|
||||
dependencies:
|
||||
b4a "^1.6.4"
|
||||
|
||||
tslib@^2.0.1:
|
||||
version "2.8.1"
|
||||
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f"
|
||||
integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==
|
||||
|
||||
typed-query-selector@^2.12.0:
|
||||
version "2.12.0"
|
||||
resolved "https://registry.yarnpkg.com/typed-query-selector/-/typed-query-selector-2.12.0.tgz#92b65dbc0a42655fccf4aeb1a08b1dddce8af5f2"
|
||||
integrity sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==
|
||||
|
||||
undici-types@~6.21.0:
|
||||
version "6.21.0"
|
||||
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb"
|
||||
integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==
|
||||
|
||||
wrap-ansi@^7.0.0:
|
||||
version "7.0.0"
|
||||
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
|
||||
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
|
||||
dependencies:
|
||||
ansi-styles "^4.0.0"
|
||||
string-width "^4.1.0"
|
||||
strip-ansi "^6.0.0"
|
||||
|
||||
wrappy@1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
|
||||
integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
|
||||
|
||||
ws@^8.18.2:
|
||||
version "8.18.2"
|
||||
resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.2.tgz#42738b2be57ced85f46154320aabb51ab003705a"
|
||||
integrity sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==
|
||||
|
||||
y18n@^5.0.5:
|
||||
version "5.0.8"
|
||||
resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
|
||||
integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
|
||||
|
||||
yargs-parser@^21.1.1:
|
||||
version "21.1.1"
|
||||
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35"
|
||||
integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==
|
||||
|
||||
yargs@^17.7.2:
|
||||
version "17.7.2"
|
||||
resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269"
|
||||
integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==
|
||||
dependencies:
|
||||
cliui "^8.0.1"
|
||||
escalade "^3.1.1"
|
||||
get-caller-file "^2.0.5"
|
||||
require-directory "^2.1.1"
|
||||
string-width "^4.2.3"
|
||||
y18n "^5.0.5"
|
||||
yargs-parser "^21.1.1"
|
||||
|
||||
yauzl@^2.10.0:
|
||||
version "2.10.0"
|
||||
resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"
|
||||
integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==
|
||||
dependencies:
|
||||
buffer-crc32 "~0.2.3"
|
||||
fd-slicer "~1.1.0"
|
||||
|
||||
zod@^3.24.1:
|
||||
version "3.25.32"
|
||||
resolved "https://registry.yarnpkg.com/zod/-/zod-3.25.32.tgz#769cc684072df780fc8f38130b0cd9283a8d3818"
|
||||
integrity sha512-OSm2xTIRfW8CV5/QKgngwmQW/8aPfGdaQFlrGoErlgg/Epm7cjb6K6VEyExfe65a3VybUOnu381edLb0dfJl0g==
|
||||
2
ebay_storage/rust/.gitignore
vendored
Normal file
2
ebay_storage/rust/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
target
|
||||
|
||||
2237
ebay_storage/rust/Cargo.lock
generated
Normal file
2237
ebay_storage/rust/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
19
ebay_storage/rust/Cargo.toml
Normal file
19
ebay_storage/rust/Cargo.toml
Normal file
@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "ebay_scraper_rust"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4.4", features = ["derive"] }
|
||||
reqwest = { version = "0.11", features = ["json", "stream"] } # Removed "blocking" as we use tokio
|
||||
scraper = "0.18"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
regex = "1.10"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
url = "2.5"
|
||||
# path-slash is not strictly needed if using std::path::PathBuf correctly
|
||||
bytes = "1.5"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
lazy_static = "1.4.0"
|
||||
futures = "0.3" # For join_all on async tasks
|
||||
547
ebay_storage/rust/src/main.rs
Normal file
547
ebay_storage/rust/src/main.rs
Normal file
@ -0,0 +1,547 @@
|
||||
// main.rs
|
||||
|
||||
// Import necessary crates
|
||||
use clap::Parser;
|
||||
use regex::Regex;
|
||||
use scraper::{Html, Selector};
|
||||
use serde::Serialize;
|
||||
use std::fs::{self, File};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::error::Error;
|
||||
use chrono::{DateTime, Utc};
|
||||
use lazy_static::lazy_static;
|
||||
use url::Url;
|
||||
|
||||
// Define constants
|
||||
const PARSER_ENGINE_VERSION: i32 = 1;
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36";
|
||||
|
||||
// --- Lazy static Regex definitions ---
|
||||
lazy_static! {
|
||||
// Regex for parsing quantity from title (e.g., "LOT OF 10", "5-PACK")
|
||||
static ref EXPLICIT_QTY_PATTERNS: Vec<Regex> = vec![
|
||||
Regex::new(r"\b(?:LOT\s+OF|LOT)\s*\(?\s*(\d+)\s*\)?").unwrap(),
|
||||
Regex::new(r"\b(?:LOT\s+OF|LOT)\s*\*\s*(\d+)").unwrap(),
|
||||
Regex::new(r"\b(?:PACK\s+OF|PACK|BULK)\s*\(?\s*(\d+)\s*\)?").unwrap(),
|
||||
Regex::new(r"\b(\d+)\s*-\s*PACK\b").unwrap(),
|
||||
Regex::new(r"\b(\d+)\s*COUNT\b").unwrap(),
|
||||
];
|
||||
// Regex for parsing size from title (e.g., "500GB", "2TB")
|
||||
static ref SIZE_REGEX: Regex = Regex::new(r"(\d+(?:\.\d+)?)\s*(TB|GB)\b").unwrap();
|
||||
// Regex for titles indicating a range of sizes or mixed items
|
||||
static ref SIZE_RANGE_REGEX: Regex = Regex::new(r"\d+(?:\.\d+)?\s*(?:GB|TB)\s*(?:-|&|OR|TO)\s*\d+(?:\.\d+)?\s*(?:GB|TB)").unwrap();
|
||||
// Regex for extracting item ID from URL
|
||||
static ref ITEM_ID_REGEX: Regex = Regex::new(r"/itm/(\d+)").unwrap();
|
||||
// Regex for parsing price, potentially a range
|
||||
static ref PRICE_REGEX: Regex = Regex::new(r"\$?([\d,]+\.?\d*)").unwrap();
|
||||
// Regex for "NEW LISTING" prefix - case-insensitive to better match JS /i flag
|
||||
static ref NEW_LISTING_REGEX: Regex = Regex::new(r"(?i)^\s*NEW LISTING\s*[:\-\s]*").unwrap();
|
||||
}
|
||||
|
||||
// --- Command Line Argument Parsing (using clap) ---
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(name = "ebay-scraper-rust", version = "0.1.0", about = "Scrapes eBay search results for SSD/HDD cost per TB.")]
|
||||
struct Cli {
|
||||
#[clap(subcommand)]
|
||||
command: Option<Commands>,
|
||||
|
||||
/// The full eBay search URL to scrape.
|
||||
url: Option<String>,
|
||||
|
||||
/// Save scraped HTML to a file (and download images if fetching from URL).
|
||||
#[clap(long)]
|
||||
save: Option<String>,
|
||||
|
||||
/// Load HTML from a file (disables network). Image download will not occur with --load.
|
||||
#[clap(long)]
|
||||
load: Option<String>,
|
||||
|
||||
/// Suppress informational logs, output only final JSON.
|
||||
#[clap(long)]
|
||||
only_json: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
enum Commands {
|
||||
/// Scrapes latest listings.
|
||||
Latest(LatestArgs),
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
struct LatestArgs {
|
||||
/// Items per page (60, 120, or 240)
|
||||
#[clap(long, default_value = "60")]
|
||||
per_page: String, // Keep as string for validation
|
||||
|
||||
/// Minimum cost (e.g., 50.00)
|
||||
#[clap(long, default_value = "0.00")]
|
||||
minimum_cost: f64,
|
||||
}
|
||||
|
||||
// --- Data Structures for Scraped Items (using serde) ---
|
||||
#[derive(Serialize, Debug)]
|
||||
struct EbayItem {
|
||||
title: String,
|
||||
#[serde(rename = "itemId")]
|
||||
item_id: String,
|
||||
#[serde(rename = "dateFound")]
|
||||
date_found: DateTime<Utc>,
|
||||
#[serde(rename = "currentBidPrice")]
|
||||
current_bid_price: Option<f64>,
|
||||
#[serde(rename = "buyItNowPrice", skip_serializing_if = "Option::is_none")] // Keep skip for this one if JS does it
|
||||
buy_it_now_price: Option<f64>,
|
||||
#[serde(rename = "hasBestOffer")]
|
||||
has_best_offer: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")] // Keep skip for this one if JS does it
|
||||
image_url: Option<String>,
|
||||
parsed: ParsedItemData,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
struct ParsedItemData {
|
||||
#[serde(rename = "itemCount")]
|
||||
item_count: i32,
|
||||
// MODIFIED: Removed skip_serializing_if to always include the field, even if null
|
||||
#[serde(rename = "sizePerItemTB")]
|
||||
size_per_item_tb: Option<f64>,
|
||||
#[serde(rename = "totalTB")]
|
||||
total_tb: Option<f64>,
|
||||
#[serde(rename = "costPerTB")]
|
||||
cost_per_tb: Option<f64>,
|
||||
#[serde(rename = "needed_description_check")]
|
||||
needed_description_check: bool,
|
||||
#[serde(rename = "parser_engine")]
|
||||
parser_engine: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SizeQuantityInfo {
|
||||
total_tb: f64,
|
||||
quantity: i32,
|
||||
individual_size_tb: f64,
|
||||
needed_description_check: bool,
|
||||
}
|
||||
|
||||
// --- Logging ---
|
||||
fn log_message(message: &str, quiet_mode: bool) {
|
||||
if !quiet_mode {
|
||||
eprintln!("{}", message);
|
||||
}
|
||||
}
|
||||
|
||||
fn log_error(message: &str, quiet_mode: bool) {
|
||||
if !quiet_mode {
|
||||
eprintln!("ERROR: {}", message);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// --- Parsing Logic ---
|
||||
mod parser {
|
||||
use super::*;
|
||||
|
||||
/// Parses size and quantity information from an item title.
|
||||
pub fn parse_size_and_quantity(title: &str) -> SizeQuantityInfo {
|
||||
let upper_title = title.to_uppercase();
|
||||
let mut total_tb = 0.0;
|
||||
let mut quantity = 1;
|
||||
let mut needed_description_check = false;
|
||||
let mut individual_size_tb = 0.0;
|
||||
|
||||
for pattern in EXPLICIT_QTY_PATTERNS.iter() {
|
||||
if let Some(caps) = pattern.captures(&upper_title) {
|
||||
if let Some(qty_match) = caps.get(1) {
|
||||
if let Ok(parsed_qty) = qty_match.as_str().parse::<i32>() {
|
||||
if parsed_qty > 0 && parsed_qty < 500 {
|
||||
quantity = parsed_qty;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut size_matches: Vec<(f64, String)> = Vec::new();
|
||||
for caps in SIZE_REGEX.captures_iter(&upper_title) {
|
||||
if let (Some(val_str), Some(unit_str)) = (caps.get(1), caps.get(2)) {
|
||||
if let Ok(val) = val_str.as_str().parse::<f64>() {
|
||||
size_matches.push((val, unit_str.as_str().to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !size_matches.is_empty() {
|
||||
let mut unique_sizes_tb: Vec<f64> = size_matches.iter()
|
||||
.map(|(val, unit)| if unit == "GB" { *val / 1000.0 } else { *val })
|
||||
.collect();
|
||||
unique_sizes_tb.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
unique_sizes_tb.dedup();
|
||||
|
||||
if !unique_sizes_tb.is_empty() {
|
||||
individual_size_tb = unique_sizes_tb[0];
|
||||
if unique_sizes_tb.len() > 1 {
|
||||
needed_description_check = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if SIZE_RANGE_REGEX.is_match(&upper_title) {
|
||||
needed_description_check = true;
|
||||
}
|
||||
if quantity > 1 && upper_title.contains("MIXED") {
|
||||
needed_description_check = true;
|
||||
}
|
||||
if upper_title.contains("CHECK THE DESCRIPTION") || upper_title.contains("CHECK DESCRIPTION") || upper_title.contains("SEE DESCRIPTION") {
|
||||
if quantity > 1 || size_matches.is_empty() || size_matches.len() > 1 {
|
||||
needed_description_check = true;
|
||||
}
|
||||
}
|
||||
|
||||
if individual_size_tb > 0.0 {
|
||||
total_tb = individual_size_tb * quantity as f64;
|
||||
}
|
||||
|
||||
if quantity > 1 && total_tb == 0.0 && !size_matches.is_empty() {
|
||||
needed_description_check = true;
|
||||
}
|
||||
|
||||
if quantity == 1 && size_matches.len() == 1 && !needed_description_check {
|
||||
// This condition is implicitly handled
|
||||
}
|
||||
|
||||
SizeQuantityInfo {
|
||||
total_tb: (total_tb * 10000.0).round() / 10000.0,
|
||||
quantity,
|
||||
individual_size_tb: (individual_size_tb * 10000.0).round() / 10000.0,
|
||||
needed_description_check,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses price from a string, taking the first price if it's a range.
|
||||
pub fn parse_price(price_text: &str) -> Option<f64> {
|
||||
let lower_price_text = price_text.to_lowercase();
|
||||
if lower_price_text.contains(" to ") {
|
||||
if let Some(first_part) = lower_price_text.split(" to ").next() {
|
||||
if let Some(caps) = PRICE_REGEX.captures(first_part) {
|
||||
if let Some(price_match) = caps.get(1) {
|
||||
return price_match.as_str().replace(',', "").parse().ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(caps) = PRICE_REGEX.captures(price_text) {
|
||||
if let Some(price_match) = caps.get(1) {
|
||||
return price_match.as_str().replace(',', "").parse().ok();
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// --- HTML Scraping Logic ---
|
||||
mod html_scraper {
|
||||
use super::*;
|
||||
|
||||
/// Extracts item data from HTML content.
|
||||
pub fn extract_data_from_html(html_content: &str, quiet_mode: bool) -> Result<Vec<EbayItem>, Box<dyn Error>> {
|
||||
let document = Html::parse_document(html_content);
|
||||
let mut items = Vec::new();
|
||||
let today = Utc::now();
|
||||
|
||||
let item_selector = Selector::parse("li.s-item, li.srp-results__item, div.s-item[role='listitem']").unwrap();
|
||||
let title_selector = Selector::parse(".s-item__title, .srp-results__title").unwrap();
|
||||
let price_selector = Selector::parse(".s-item__price").unwrap();
|
||||
let image_selector = Selector::parse(".s-item__image-wrapper img.s-item__image-img, .s-item__image img").unwrap();
|
||||
let link_selector = Selector::parse("a.s-item__link[href*='/itm/'], .s-item__info > a[href*='/itm/']").unwrap();
|
||||
let bid_count_selector = Selector::parse(".s-item__bid-count").unwrap();
|
||||
let best_offer_selector = Selector::parse(".s-item__purchase-options--bo, .s-item__best-offer").unwrap();
|
||||
let secondary_info_selector = Selector::parse(".s-item__subtitle, .s-item__secondary-text, .s-item__detail--secondary").unwrap();
|
||||
let auction_bin_price_selector = Selector::parse(".s-item__buy-it-now-price").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let raw_title_text = element.select(&title_selector).next().map(|el| el.text().collect::<String>().trim().to_string());
|
||||
let price_text = element.select(&price_selector).next().map(|el| el.text().collect::<String>().trim().to_string());
|
||||
|
||||
let item_id = element.select(&link_selector).next()
|
||||
.and_then(|link_el| link_el.value().attr("href"))
|
||||
.and_then(|href| ITEM_ID_REGEX.captures(href))
|
||||
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()));
|
||||
|
||||
if raw_title_text.is_none() || price_text.is_none() || item_id.is_none() {
|
||||
log_message("Skipping item due to missing title, price, or item ID.", quiet_mode);
|
||||
continue;
|
||||
}
|
||||
let raw_title = raw_title_text.unwrap();
|
||||
let price_text = price_text.unwrap();
|
||||
let item_id = item_id.unwrap();
|
||||
|
||||
let cleaned_title = NEW_LISTING_REGEX.replace(&raw_title, "").trim().to_string();
|
||||
|
||||
let primary_display_price = parser::parse_price(&price_text);
|
||||
|
||||
let mut current_bid_price: Option<f64> = None;
|
||||
let mut final_buy_it_now_price: Option<f64> = None;
|
||||
let mut has_best_offer = false;
|
||||
let mut item_is_auction = false;
|
||||
|
||||
if let Some(bid_el) = element.select(&bid_count_selector).next() {
|
||||
if bid_el.text().collect::<String>().to_lowercase().contains("bid") {
|
||||
item_is_auction = true;
|
||||
}
|
||||
}
|
||||
|
||||
if element.select(&best_offer_selector).next().is_some() {
|
||||
has_best_offer = true;
|
||||
} else {
|
||||
for el in element.select(&secondary_info_selector) {
|
||||
if el.text().collect::<String>().to_lowercase().contains("or best offer") {
|
||||
has_best_offer = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if item_is_auction {
|
||||
current_bid_price = primary_display_price;
|
||||
if let Some(bin_el) = element.select(&auction_bin_price_selector).next() {
|
||||
final_buy_it_now_price = parser::parse_price(&bin_el.text().collect::<String>());
|
||||
}
|
||||
} else {
|
||||
final_buy_it_now_price = primary_display_price;
|
||||
}
|
||||
|
||||
let image_url_val = element.select(&image_selector).next()
|
||||
.and_then(|img_el| {
|
||||
img_el.value().attr("data-src").or(img_el.value().attr("src"))
|
||||
})
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let parsed_size_info = parser::parse_size_and_quantity(&cleaned_title);
|
||||
|
||||
let cost_per_tb = if let Some(price) = primary_display_price {
|
||||
if parsed_size_info.total_tb > 0.0 {
|
||||
Some(((price / parsed_size_info.total_tb) * 100.0).round() / 100.0)
|
||||
} else { None }
|
||||
} else { None };
|
||||
|
||||
let parsed_data = ParsedItemData {
|
||||
item_count: parsed_size_info.quantity,
|
||||
size_per_item_tb: if parsed_size_info.individual_size_tb > 0.0 { Some(parsed_size_info.individual_size_tb) } else { None },
|
||||
total_tb: if parsed_size_info.total_tb > 0.0 { Some(parsed_size_info.total_tb) } else { None },
|
||||
cost_per_tb, // This will be None if conditions aren't met, and serialized as null
|
||||
needed_description_check: parsed_size_info.needed_description_check,
|
||||
parser_engine: PARSER_ENGINE_VERSION,
|
||||
};
|
||||
|
||||
items.push(EbayItem {
|
||||
title: cleaned_title,
|
||||
item_id,
|
||||
date_found: today,
|
||||
current_bid_price,
|
||||
buy_it_now_price: final_buy_it_now_price,
|
||||
has_best_offer,
|
||||
image_url: image_url_val,
|
||||
parsed: parsed_data,
|
||||
});
|
||||
}
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Downloads an image from a URL and saves it, preserving path structure.
|
||||
pub async fn download_image(image_url_str: &str, base_save_directory: &Path, quiet_mode: bool) -> Result<(), Box<dyn Error>> {
|
||||
if image_url_str.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let parsed_url = Url::parse(image_url_str)?;
|
||||
|
||||
let image_path_from_url = parsed_url.path().trim_start_matches('/');
|
||||
if image_path_from_url.is_empty() {
|
||||
return Err("Image URL has no path component".into());
|
||||
}
|
||||
|
||||
let full_local_image_path = base_save_directory.join(image_path_from_url);
|
||||
|
||||
if let Some(parent_dir) = full_local_image_path.parent() {
|
||||
fs::create_dir_all(parent_dir)?;
|
||||
log_message(&format!("Ensured image directory exists: {}", parent_dir.display()), quiet_mode);
|
||||
}
|
||||
|
||||
let client = reqwest::Client::builder().user_agent(USER_AGENT).build()?;
|
||||
let response = client.get(image_url_str).send().await?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("Failed to download image {}. Status: {}", image_url_str, response.status()).into());
|
||||
}
|
||||
|
||||
let mut file = File::create(&full_local_image_path)?;
|
||||
let content = response.bytes().await?;
|
||||
file.write_all(&content)?;
|
||||
|
||||
log_message(&format!("Downloaded image: {}", full_local_image_path.display()), quiet_mode);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// --- Main Application Logic ---
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn Error>> {
|
||||
let cli = Cli::parse();
|
||||
let quiet_mode = cli.only_json;
|
||||
|
||||
log_message("Starting scraping process...", quiet_mode);
|
||||
|
||||
let html_content_to_parse: String;
|
||||
let mut should_download_images = false;
|
||||
let mut image_base_save_dir: Option<PathBuf> = None;
|
||||
|
||||
|
||||
if let Some(html_file) = &cli.load {
|
||||
log_message(&format!("Loading HTML from {}...", html_file), quiet_mode);
|
||||
html_content_to_parse = fs::read_to_string(html_file)?;
|
||||
log_message("HTML loaded. Network requests for page content disabled.", quiet_mode);
|
||||
} else {
|
||||
let url_to_fetch = match (&cli.command, &cli.url) {
|
||||
(Some(Commands::Latest(latest_args)), _) => {
|
||||
let valid_per_page = ["60", "120", "240"];
|
||||
if !valid_per_page.contains(&latest_args.per_page.as_str()) {
|
||||
let err_msg = format!("--per_page must be one of {}, got {}", valid_per_page.join(", "), latest_args.per_page);
|
||||
log_error(&err_msg, quiet_mode);
|
||||
return Err(err_msg.into());
|
||||
}
|
||||
if latest_args.minimum_cost < 0.0 {
|
||||
let err_msg = "--minimum_cost must be a non-negative number.";
|
||||
log_error(err_msg, quiet_mode);
|
||||
return Err(err_msg.into());
|
||||
}
|
||||
let base_url = "https://www.ebay.com/sch/i.html?_nkw=&_sacat=175669&_from=R40&_fsrp=1&LH_PrefLoc=3&imm=1&_sop=10";
|
||||
let url = format!("{}&_ipg={}&_udlo={:.2}", base_url, latest_args.per_page, latest_args.minimum_cost);
|
||||
log_message(&format!("Constructed URL for 'latest': {}", url), quiet_mode);
|
||||
url
|
||||
}
|
||||
(None, Some(url_arg)) => {
|
||||
url_arg.clone()
|
||||
}
|
||||
(None, None) => {
|
||||
let err_msg = "No URL provided and no command specified. Use --help for usage.";
|
||||
log_error(err_msg, true);
|
||||
return Err(err_msg.into());
|
||||
}
|
||||
};
|
||||
|
||||
log_message(&format!("Navigating to {}...", url_to_fetch), quiet_mode);
|
||||
let client = reqwest::Client::builder().user_agent(USER_AGENT).build()?;
|
||||
let response = client.get(&url_to_fetch).send().await?;
|
||||
if !response.status().is_success() {
|
||||
let err_msg = format!("Failed to fetch URL: {} - Status: {}", url_to_fetch, response.status());
|
||||
log_error(&err_msg, quiet_mode);
|
||||
return Err(err_msg.into());
|
||||
}
|
||||
html_content_to_parse = response.text().await?;
|
||||
log_message("Navigation successful. Page content retrieved.", quiet_mode);
|
||||
|
||||
if let Some(save_path_str) = &cli.save {
|
||||
log_message(&format!("Saving HTML to {}...", save_path_str), quiet_mode);
|
||||
let mut file = File::create(save_path_str)?;
|
||||
file.write_all(html_content_to_parse.as_bytes())?;
|
||||
log_message("HTML saved.", quiet_mode);
|
||||
|
||||
should_download_images = true;
|
||||
let save_file_path = PathBuf::from(save_path_str);
|
||||
let base_name = save_file_path.file_stem().unwrap_or_default().to_string_lossy().to_string();
|
||||
if let Some(parent_dir) = save_file_path.parent() {
|
||||
image_base_save_dir = Some(parent_dir.join(base_name));
|
||||
} else {
|
||||
image_base_save_dir = Some(PathBuf::from(base_name));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log_message("Extracting data...", quiet_mode);
|
||||
let extracted_results = html_scraper::extract_data_from_html(&html_content_to_parse, quiet_mode)?;
|
||||
log_message(&format!("Data extraction complete. Found {} items.", extracted_results.len()), quiet_mode);
|
||||
|
||||
if should_download_images && !extracted_results.is_empty() {
|
||||
if let Some(img_base_dir) = image_base_save_dir {
|
||||
log_message(&format!("Downloading images into subdirectories of {}...", img_base_dir.display()), quiet_mode);
|
||||
|
||||
let mut download_futures = Vec::new();
|
||||
for item in &extracted_results {
|
||||
if let Some(img_url) = &item.image_url {
|
||||
let img_base_dir_clone = img_base_dir.clone();
|
||||
let img_url_clone = img_url.clone();
|
||||
let item_id_clone = item.item_id.clone();
|
||||
|
||||
download_futures.push(async move {
|
||||
if let Err(e) = html_scraper::download_image(&img_url_clone, &img_base_dir_clone, quiet_mode).await {
|
||||
log_error(&format!("Skipping image download for item ID {} (URL: {}) due to error: {}", item_id_clone, img_url_clone, e), quiet_mode);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
futures::future::join_all(download_futures).await;
|
||||
log_message("Image download process finished.", quiet_mode);
|
||||
}
|
||||
}
|
||||
|
||||
if quiet_mode {
|
||||
println!("{}", serde_json::to_string(&extracted_results)?);
|
||||
} else {
|
||||
println!("{}", serde_json::to_string_pretty(&extracted_results)?);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// --- Unit tests for parser functions (optional, but good practice) ---
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parser::*;
|
||||
use super::SizeQuantityInfo;
|
||||
|
||||
fn assert_sq_info_eq(actual: SizeQuantityInfo, expected_total_tb: f64, expected_quantity: i32, expected_ind_size_tb: f64, expected_check: bool) {
|
||||
assert!((actual.total_tb - expected_total_tb).abs() < 0.0001, "TotalTB mismatch. Expected: {}, Got: {}", expected_total_tb, actual.total_tb);
|
||||
assert_eq!(actual.quantity, expected_quantity, "Quantity mismatch");
|
||||
assert!((actual.individual_size_tb - expected_ind_size_tb).abs() < 0.0001, "IndividualSizeTB mismatch. Expected: {}, Got: {}", expected_ind_size_tb, actual.individual_size_tb);
|
||||
assert_eq!(actual.needed_description_check, expected_check, "NeededDescriptionCheck mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_size_and_quantity() {
|
||||
let test_cases = vec![
|
||||
("LOT OF (9) MAJOR BRAND 2.5\" 7MM SSD * Kingston, Samsung, SanDisk& PNY*120-250GB", 1.080, 9, 0.120, true),
|
||||
("Lot of 10 Intel 256 GB 2.5\" SATA SSD different Model check the Description", 2.560, 10, 0.256, true),
|
||||
("Bulk 5 Lot Samsung 870 EVO 500GB SSD SATA - Used - Tested Passed Smart Test", 2.500, 5, 0.500, false),
|
||||
("Samsung 1.6TB NVME PCIe 3.0 x8 2.75\" SSD MZPLK1T6HCHP PM1725 Series TLC", 1.6, 1, 1.6, false),
|
||||
("Micron 5100 MAX 1.84TB SATA 6Gb/s 2.5\" SSD MTFDDAK1T9TCC-1AR1ZABYY", 1.84, 1, 1.84, false),
|
||||
("10-PACK 1TB SSD", 10.0, 10, 1.0, false),
|
||||
("2TB SSD NVMe", 2.0, 1, 2.0, false),
|
||||
("WD Blue 500GB Internal SSD SATA III 6Gb/s", 0.5, 1, 0.5, false),
|
||||
("Lot of 2 Mixed Capacity SSDs (120GB, 240GB) CHECK DESCRIPTION", 0.24, 2, 0.12, true),
|
||||
("Single Drive 1TB", 1.0, 1, 1.0, false),
|
||||
("Lot of 3 - CHECK DESCRIPTION - Mixed SSDs", 0.0, 3, 0.0, true),
|
||||
];
|
||||
|
||||
for (title, total_tb, quantity, ind_size_tb, check) in test_cases {
|
||||
println!("Testing title: {}", title);
|
||||
let result = parse_size_and_quantity(title);
|
||||
assert_sq_info_eq(result, total_tb, quantity, ind_size_tb, check);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_price() {
|
||||
assert_eq!(parse_price("$19.99"), Some(19.99));
|
||||
assert_eq!(parse_price("USD 150.00"), Some(150.00));
|
||||
assert_eq!(parse_price("$1,234.56"), Some(1234.56));
|
||||
assert_eq!(parse_price("Free"), None);
|
||||
assert_eq!(parse_price("$10.00 to $20.00"), Some(10.00));
|
||||
assert_eq!(parse_price("EUR 25.50"), Some(25.50));
|
||||
assert_eq!(parse_price("25.50"), Some(25.50));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user