feat: working Puppeteer scraper for Promofarma

- Added Puppeteer with Chrome in Docker - Scraper extracts products using data attributes - Added /api/scrape endpoint - Tested: 10 products scraped from Promofarma The scraper now works with Promofarma's HTML structure which uses data-name, data-pvp attributes on article elements.
2026-07-16 14:54:51 +02:00
parent 2ad4210221
commit a5a75d3249
6 changed files with 390 additions and 1 deletions
@@ -0,0 +1,189 @@
+import puppeteer from 'puppeteer';
+import axios from 'axios';
+
+const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002';
+
+const SOURCES = {
+  promofarma: {
+    name: 'Promofarma',
+    searchUrl: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`,
+    selectors: {
+      product: 'article[data-name]',
+      name: 'article[data-name]',
+      price: 'article[data-pvp]',
+      link: 'a',
+      image: 'img'
+    }
+  },
+  docmorris: {
+    name: 'DocMorris',
+    searchUrl: (q) => `https://www.docmorris.es/search?query=${encodeURIComponent(q)}`,
+    selectors: {
+      product: '[class*="product-card"], [class*="product-item"]',
+      name: '[class*="product-name"], h3',
+      price: '[class*="price"]',
+      link: 'a',
+      image: 'img'
+    }
+  },
+  primor: {
+    name: 'Primor',
+    searchUrl: (q) => `https://www.primor.eu/search?s=${encodeURIComponent(q)}`,
+    selectors: {
+      product: '.product-item, .product-miniature',
+      name: '.product-title, h3',
+      price: '.price',
+      link: 'a.product-title',
+      image: 'img'
+    }
+  }
+};
+
+async function scrapeSource(browser, source, query) {
+  const config = SOURCES[source];
+  if (!config) return [];
+
+  const page = await browser.newPage();
+  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
+
+  try {
+    console.log(`  🔍 Scraping ${config.name}: ${query}`);
+    const url = config.searchUrl(query);
+    console.log(`  📍 URL: ${url}`);
+
+    await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 });
+
+    // Wait for products to load
+    await page.waitForSelector(config.selectors.product, { timeout: 15000 }).catch(() => {});
+
+    // Get page title for debugging
+    const title = await page.title();
+    console.log(`  📄 Page title: ${title}`);
+
+    // Get page content for debugging
+    const content = await page.content();
+    console.log(`  📄 Page length: ${content.length} chars`);
+
+    // Try to find product elements with various selectors
+    const productInfo = await page.evaluate(() => {
+      // Try multiple selector strategies
+      const selectors = [
+        '[data-testid*="product"]',
+        '[class*="ProductCard"]',
+        '[class*="product-card"]',
+        'a[href*="/p/"]',
+        '[class*="Product"]',
+        'article',
+        '[role="listitem"]'
+      ];
+
+      for (const sel of selectors) {
+        const els = document.querySelectorAll(sel);
+        if (els.length > 0) {
+          return {
+            selector: sel,
+            count: els.length,
+            samples: Array.from(els).slice(0, 3).map(el => ({
+              tag: el.tagName,
+              class: el.className?.substring(0, 100),
+              html: el.outerHTML?.substring(0, 300)
+            }))
+          };
+        }
+      }
+
+      // Fallback: look for any element with price-like content
+      const allText = document.body.innerText;
+      const priceMatch = allText.match(/\d+[.,]\d{2}\s*€/g);
+
+      return {
+        selector: 'none found',
+        priceMatches: priceMatch?.slice(0, 5) || [],
+        bodyTextSample: document.body.innerText.substring(0, 500)
+      };
+    });
+    console.log(`  🔍 Product info:`, JSON.stringify(productInfo, null, 2));
+
+    const products = await page.evaluate((selectors) => {
+      const results = [];
+      const cards = document.querySelectorAll(selectors.product);
+
+      cards.forEach(card => {
+        // Try data attributes first (Promofarma style)
+        const name = card.getAttribute('data-name') || card.querySelector(selectors.name)?.innerText?.trim();
+        const priceStr = card.getAttribute('data-pvp') || card.querySelector(selectors.price)?.innerText?.trim();
+        const linkEl = card.querySelector(selectors.link);
+        const imgEl = card.querySelector(selectors.image);
+
+        if (name && priceStr) {
+          const price = parseFloat(priceStr.replace(/[^0-9.,]/g, '').replace(',', '.')) || 0;
+
+          if (name && price > 0 && price < 1000) {
+            results.push({
+              name: name.substring(0, 200),
+              price,
+              source_url: linkEl?.href || '',
+              image_url: imgEl?.src || null
+            });
+          }
+        }
+      });
+
+      return results;
+    }, config.selectors);
+
+    console.log(`  📦 Found ${products.length} products`);
+    return products.slice(0, 10).map(p => ({
+      ...p,
+      source,
+      source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,
+      brand: '',
+      category: 'parapharmacy',
+      available: true,
+      scraped_at: new Date().toISOString()
+    }));
+
+  } catch (error) {
+    console.error(`  ❌ Error scraping ${config.name}: ${error.message}`);
+    return [];
+  } finally {
+    await page.close();
+  }
+}
+
+export async function scrapeAll(queries = ['crema hidratante'], sources = ['promofarma']) {
+  console.log('🚀 Starting scraper...');
+
+  const browser = await puppeteer.launch({
+    headless: 'new',
+    executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
+    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
+  });
+
+  let totalProducts = 0;
+
+  try {
+    for (const query of queries) {
+      console.log(`\n📋 Query: "${query}"`);
+
+      for (const source of sources) {
+        const products = await scrapeSource(browser, source, query);
+        console.log(`  ✅ Found ${products.length} products`);
+
+        if (products.length > 0) {
+          try {
+            await axios.post(`${API_URL}/api/products/bulk`, { products });
+            totalProducts += products.length;
+          } catch (error) {
+            console.error(`  ❌ Error sending to API: ${error.message}`);
+          }
+        }
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+
+  console.log(`\n📊 Scraping completed. Total: ${totalProducts} products`);
+  return { success: true, total: totalProducts };
+}