fix: filter scraper results by search query
Run Tests on Branches / Detect Changes (push) Successful in 13s
Run Tests on Branches / Backend Tests (push) Failing after 16s
Run Tests on Branches / Frontend Tests (push) Failing after 16s
Run Tests on Branches / Frontend Mobile Tests (push) Failing after 17s
Run Tests on Branches / Parapharmacy API Tests (push) Has been skipped
Run Tests on Branches / Detect Changes (push) Successful in 13s
Run Tests on Branches / Backend Tests (push) Failing after 16s
Run Tests on Branches / Frontend Tests (push) Failing after 16s
Run Tests on Branches / Frontend Mobile Tests (push) Failing after 17s
Run Tests on Branches / Parapharmacy API Tests (push) Has been skipped
The scraper was extracting ALL products from the page including navigation menu items. Now it filters products to only include those that contain the search query in their name. Before: Found 47 products (mostly navigation items) After: Found 2 Capricare products (filtered by query)
This commit is contained in:
@@ -107,10 +107,13 @@ async function scrapeSource(browser, source, query) {
|
||||
});
|
||||
console.log(` 🔍 Product info:`, JSON.stringify(productInfo, null, 2));
|
||||
|
||||
const products = await page.evaluate((selectors) => {
|
||||
const products = await page.evaluate((selectors, searchQuery) => {
|
||||
const results = [];
|
||||
const cards = document.querySelectorAll(selectors.product);
|
||||
|
||||
// Get the search query to filter relevant products
|
||||
const query = searchQuery.toLowerCase();
|
||||
|
||||
cards.forEach(card => {
|
||||
// Try data attributes first (Promofarma style)
|
||||
let name = card.getAttribute('data-name');
|
||||
@@ -132,7 +135,9 @@ async function scrapeSource(browser, source, query) {
|
||||
if (name && priceStr) {
|
||||
const price = parseFloat(priceStr.replace(/[^0-9.,]/g, '').replace(',', '.')) || 0;
|
||||
|
||||
if (name && price > 0 && price < 1000) {
|
||||
// Filter: only include products that contain the search query
|
||||
const nameLower = name.toLowerCase();
|
||||
if (name && price > 0 && price < 1000 && nameLower.includes(query)) {
|
||||
results.push({
|
||||
name: name.substring(0, 200),
|
||||
price,
|
||||
@@ -144,7 +149,7 @@ async function scrapeSource(browser, source, query) {
|
||||
});
|
||||
|
||||
return results;
|
||||
}, config.selectors);
|
||||
}, config.selectors, query);
|
||||
|
||||
console.log(` 📦 Found ${products.length} products`);
|
||||
return products.slice(0, 10).map(p => ({
|
||||
|
||||
Reference in New Issue
Block a user