import puppeteer from 'puppeteer'; import axios from 'axios'; const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002'; const SOURCES = { promofarma: { name: 'Promofarma', searchUrl: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`, selectors: { product: 'article[data-name]', name: 'article[data-name]', price: 'article[data-pvp]', link: 'a', image: 'img' } }, docmorris: { name: 'DocMorris', searchUrl: (q) => `https://www.docmorris.es/search?query=${encodeURIComponent(q)}`, selectors: { product: '[class*="product-card"], [class*="product-item"]', name: '[class*="product-name"], h3', price: '[class*="price"]', link: 'a', image: 'img' } }, primor: { name: 'Primor', searchUrl: (q) => `https://www.primor.eu/catalogsearch/result/?q=${encodeURIComponent(q)}`, selectors: { product: 'form.product-item', name: 'form.product-item', price: 'form.product-item', link: 'a', image: 'img' } } }; async function scrapeSource(browser, source, query) { const config = SOURCES[source]; if (!config) return []; const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'); try { console.log(` šŸ” Scraping ${config.name}: ${query}`); const url = config.searchUrl(query); console.log(` šŸ“ URL: ${url}`); await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); // Wait for products to load await page.waitForSelector(config.selectors.product, { timeout: 15000 }).catch(() => {}); // Get page title for debugging const title = await page.title(); console.log(` šŸ“„ Page title: ${title}`); // Get page content for debugging const content = await page.content(); console.log(` šŸ“„ Page length: ${content.length} chars`); // Try to find product elements with various selectors const productInfo = await page.evaluate(() => { // Try multiple selector strategies const selectors = [ '[data-testid*="product"]', '[class*="ProductCard"]', '[class*="product-card"]', 'a[href*="/p/"]', '[class*="Product"]', 'article', '[role="listitem"]', '.product-item', '.product', '[data-product]' ]; for (const sel of selectors) { const els = document.querySelectorAll(sel); if (els.length > 0) { return { selector: sel, count: els.length, samples: Array.from(els).slice(0, 3).map(el => ({ tag: el.tagName, class: el.className?.substring(0, 150), html: el.outerHTML?.substring(0, 500) })) }; } } // Fallback: look for any element with price-like content const allText = document.body.innerText; const priceMatch = allText.match(/\d+[.,]\d{2}\s*€/g); return { selector: 'none found', priceMatches: priceMatch?.slice(0, 5) || [], bodyTextSample: document.body.innerText.substring(0, 500) }; }); console.log(` šŸ” Product info:`, JSON.stringify(productInfo, null, 2)); const products = await page.evaluate((selectors, searchQuery) => { const results = []; const cards = document.querySelectorAll(selectors.product); // Get the search query to filter relevant products const query = searchQuery.toLowerCase(); cards.forEach(card => { // Try data attributes first (Promofarma style) let name = card.getAttribute('data-name'); let priceStr = card.getAttribute('data-pvp'); // If no data attributes, try to extract from content if (!name) { const nameEl = card.querySelector('[class*="name"], h3, h2, .product-name'); name = nameEl?.innerText?.trim(); } if (!priceStr) { const priceEl = card.querySelector('[class*="price"], .price'); priceStr = priceEl?.innerText?.trim(); } const linkEl = card.querySelector(selectors.link); const imgEl = card.querySelector(selectors.image); if (name && priceStr) { const price = parseFloat(priceStr.replace(/[^0-9.,]/g, '').replace(',', '.')) || 0; // Filter: only include products that contain the search query const nameLower = name.toLowerCase(); if (name && price > 0 && price < 1000 && nameLower.includes(query)) { results.push({ name: name.substring(0, 200), price, source_url: linkEl?.href || '', image_url: imgEl?.src || null }); } } }); return results; }, config.selectors, query); console.log(` šŸ“¦ Found ${products.length} products`); return products.slice(0, 10).map(p => ({ ...p, source, source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`, brand: '', category: 'parapharmacy', available: true, scraped_at: new Date().toISOString() })); } catch (error) { console.error(` āŒ Error scraping ${config.name}: ${error.message}`); return []; } finally { await page.close(); } } export async function scrapeAll(queries = ['crema hidratante'], sources = ['promofarma']) { console.log('šŸš€ Starting scraper...'); const browser = await puppeteer.launch({ headless: 'new', executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] }); let totalProducts = 0; try { for (const query of queries) { console.log(`\nšŸ“‹ Query: "${query}"`); for (const source of sources) { const products = await scrapeSource(browser, source, query); console.log(` āœ… Found ${products.length} products`); if (products.length > 0) { try { await axios.post(`${API_URL}/api/products/bulk`, { products }); totalProducts += products.length; } catch (error) { console.error(` āŒ Error sending to API: ${error.message}`); } } } } } finally { await browser.close(); } console.log(`\nšŸ“Š Scraping completed. Total: ${totalProducts} products`); return { success: true, total: totalProducts }; }