feat: working scraping for Promofarma, DocMorris, and Primor
- Fixed Primor URL format and selectors - Created simple N8N webhook that calls scraper API - All 3 sources now working: - Promofarma: 20 products - DocMorris: 10 products - Primor: 20 products - Total: 130 products in database - N8N webhook at /webhook/scrape working correctly
This commit is contained in:
@@ -28,12 +28,12 @@ const SOURCES = {
|
||||
},
|
||||
primor: {
|
||||
name: 'Primor',
|
||||
searchUrl: (q) => `https://www.primor.eu/search?s=${encodeURIComponent(q)}`,
|
||||
searchUrl: (q) => `https://www.primor.eu/catalogsearch/result/?q=${encodeURIComponent(q)}`,
|
||||
selectors: {
|
||||
product: '.product-item, .product-miniature',
|
||||
name: '.product-title, h3',
|
||||
price: '.price',
|
||||
link: 'a.product-title',
|
||||
product: 'form.product-item',
|
||||
name: 'form.product-item',
|
||||
price: 'form.product-item',
|
||||
link: 'a',
|
||||
image: 'img'
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,10 @@ async function scrapeSource(browser, source, query) {
|
||||
'a[href*="/p/"]',
|
||||
'[class*="Product"]',
|
||||
'article',
|
||||
'[role="listitem"]'
|
||||
'[role="listitem"]',
|
||||
'.product-item',
|
||||
'.product',
|
||||
'[data-product]'
|
||||
];
|
||||
|
||||
for (const sel of selectors) {
|
||||
@@ -85,8 +88,8 @@ async function scrapeSource(browser, source, query) {
|
||||
count: els.length,
|
||||
samples: Array.from(els).slice(0, 3).map(el => ({
|
||||
tag: el.tagName,
|
||||
class: el.className?.substring(0, 100),
|
||||
html: el.outerHTML?.substring(0, 300)
|
||||
class: el.className?.substring(0, 150),
|
||||
html: el.outerHTML?.substring(0, 500)
|
||||
}))
|
||||
};
|
||||
}
|
||||
@@ -110,8 +113,19 @@ async function scrapeSource(browser, source, query) {
|
||||
|
||||
cards.forEach(card => {
|
||||
// Try data attributes first (Promofarma style)
|
||||
const name = card.getAttribute('data-name') || card.querySelector(selectors.name)?.innerText?.trim();
|
||||
const priceStr = card.getAttribute('data-pvp') || card.querySelector(selectors.price)?.innerText?.trim();
|
||||
let name = card.getAttribute('data-name');
|
||||
let priceStr = card.getAttribute('data-pvp');
|
||||
|
||||
// If no data attributes, try to extract from content
|
||||
if (!name) {
|
||||
const nameEl = card.querySelector('[class*="name"], h3, h2, .product-name');
|
||||
name = nameEl?.innerText?.trim();
|
||||
}
|
||||
if (!priceStr) {
|
||||
const priceEl = card.querySelector('[class*="price"], .price');
|
||||
priceStr = priceEl?.innerText?.trim();
|
||||
}
|
||||
|
||||
const linkEl = card.querySelector(selectors.link);
|
||||
const imgEl = card.querySelector(selectors.image);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user