feat: working Puppeteer scraper for Promofarma

- Added Puppeteer with Chrome in Docker - Scraper extracts products using data attributes - Added /api/scrape endpoint - Tested: 10 products scraped from Promofarma The scraper now works with Promofarma's HTML structure which uses data-name, data-pvp attributes on article elements.
2026-07-16 14:54:51 +02:00
parent 2ad4210221
commit a5a75d3249
6 changed files with 390 additions and 1 deletions
@@ -1,7 +1,32 @@
-FROM node:20-alpine
+FROM node:20-slim

 WORKDIR /app

+# Install Chrome and dependencies for Puppeteer
+RUN apt-get update && apt-get install -y \
+    chromium \
+    fonts-liberation \
+    libappindicator3-1 \
+    libasound2 \
+    libatk-bridge2.0-0 \
+    libatk1.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libgdk-pixbuf2.0-0 \
+    libnspr4 \
+    libnss3 \
+    libx11-xcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxrandr2 \
+    xdg-utils \
+    --no-install-recommends && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set Puppeteer to use installed Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+
 # Copy package files
 COPY package*.json ./

@@ -8,6 +8,7 @@
    "start": "node --env-file-if-exists=.env src/server.js",
    "dev": "node --env-file-if-exists=.env --watch src/server.js",
    "seed": "node --env-file-if-exists=.env scripts/seed.js",
+    "scrape": "node --env-file-if-exists=.env scripts/scrape-puppeteer.js",
    "test": "NODE_OPTIONS='--experimental-vm-modules' npx jest --ci --forceExit --forceExitTimeout=30000"
  },
  "dependencies": {
@@ -16,6 +17,7 @@
    "express-rate-limit": "^8.5.2",
    "mongoose": "^8.8.0",
    "morgan": "^1.10.0",
+    "puppeteer": "^22.0.0",
    "swagger-jsdoc": "^6.2.8",
    "swagger-ui-express": "^5.0.0",
    "axios": "^1.6.0"
@@ -0,0 +1,26 @@
+import { Router } from 'express';
+import { scrapeAll } from '../scraper.js';
+
+const router = Router();
+
+// Trigger scraping
+router.post('/scrape', async (req, res) => {
+  try {
+    const { queries = ['crema hidratante'], sources = ['promofarma'] } = req.body;
+
+    console.log('[Scraper] Starting scrape...');
+    console.log(`[Scraper] Queries: ${queries.join(', ')}`);
+    console.log(`[Scraper] Sources: ${sources.join(', ')}`);
+
+    const result = await scrapeAll(queries, sources);
+
+    console.log(`[Scraper] Completed. Total: ${result.total} products`);
+
+    res.json(result);
+  } catch (error) {
+    console.error('[Scraper] Error:', error.message);
+    res.status(500).json({ error: 'Scraping failed', message: error.message });
+  }
+});
+
+export default router;
@@ -0,0 +1,189 @@
+import puppeteer from 'puppeteer';
+import axios from 'axios';
+
+const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002';
+
+const SOURCES = {
+  promofarma: {
+    name: 'Promofarma',
+    searchUrl: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`,
+    selectors: {
+      product: 'article[data-name]',
+      name: 'article[data-name]',
+      price: 'article[data-pvp]',
+      link: 'a',
+      image: 'img'
+    }
+  },
+  docmorris: {
+    name: 'DocMorris',
+    searchUrl: (q) => `https://www.docmorris.es/search?query=${encodeURIComponent(q)}`,
+    selectors: {
+      product: '[class*="product-card"], [class*="product-item"]',
+      name: '[class*="product-name"], h3',
+      price: '[class*="price"]',
+      link: 'a',
+      image: 'img'
+    }
+  },
+  primor: {
+    name: 'Primor',
+    searchUrl: (q) => `https://www.primor.eu/search?s=${encodeURIComponent(q)}`,
+    selectors: {
+      product: '.product-item, .product-miniature',
+      name: '.product-title, h3',
+      price: '.price',
+      link: 'a.product-title',
+      image: 'img'
+    }
+  }
+};
+
+async function scrapeSource(browser, source, query) {
+  const config = SOURCES[source];
+  if (!config) return [];
+
+  const page = await browser.newPage();
+  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
+
+  try {
+    console.log(`  🔍 Scraping ${config.name}: ${query}`);
+    const url = config.searchUrl(query);
+    console.log(`  📍 URL: ${url}`);
+
+    await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 });
+
+    // Wait for products to load
+    await page.waitForSelector(config.selectors.product, { timeout: 15000 }).catch(() => {});
+
+    // Get page title for debugging
+    const title = await page.title();
+    console.log(`  📄 Page title: ${title}`);
+
+    // Get page content for debugging
+    const content = await page.content();
+    console.log(`  📄 Page length: ${content.length} chars`);
+
+    // Try to find product elements with various selectors
+    const productInfo = await page.evaluate(() => {
+      // Try multiple selector strategies
+      const selectors = [
+        '[data-testid*="product"]',
+        '[class*="ProductCard"]',
+        '[class*="product-card"]',
+        'a[href*="/p/"]',
+        '[class*="Product"]',
+        'article',
+        '[role="listitem"]'
+      ];
+
+      for (const sel of selectors) {
+        const els = document.querySelectorAll(sel);
+        if (els.length > 0) {
+          return {
+            selector: sel,
+            count: els.length,
+            samples: Array.from(els).slice(0, 3).map(el => ({
+              tag: el.tagName,
+              class: el.className?.substring(0, 100),
+              html: el.outerHTML?.substring(0, 300)
+            }))
+          };
+        }
+      }
+
+      // Fallback: look for any element with price-like content
+      const allText = document.body.innerText;
+      const priceMatch = allText.match(/\d+[.,]\d{2}\s*€/g);
+
+      return {
+        selector: 'none found',
+        priceMatches: priceMatch?.slice(0, 5) || [],
+        bodyTextSample: document.body.innerText.substring(0, 500)
+      };
+    });
+    console.log(`  🔍 Product info:`, JSON.stringify(productInfo, null, 2));
+
+    const products = await page.evaluate((selectors) => {
+      const results = [];
+      const cards = document.querySelectorAll(selectors.product);
+
+      cards.forEach(card => {
+        // Try data attributes first (Promofarma style)
+        const name = card.getAttribute('data-name') || card.querySelector(selectors.name)?.innerText?.trim();
+        const priceStr = card.getAttribute('data-pvp') || card.querySelector(selectors.price)?.innerText?.trim();
+        const linkEl = card.querySelector(selectors.link);
+        const imgEl = card.querySelector(selectors.image);
+
+        if (name && priceStr) {
+          const price = parseFloat(priceStr.replace(/[^0-9.,]/g, '').replace(',', '.')) || 0;
+
+          if (name && price > 0 && price < 1000) {
+            results.push({
+              name: name.substring(0, 200),
+              price,
+              source_url: linkEl?.href || '',
+              image_url: imgEl?.src || null
+            });
+          }
+        }
+      });
+
+      return results;
+    }, config.selectors);
+
+    console.log(`  📦 Found ${products.length} products`);
+    return products.slice(0, 10).map(p => ({
+      ...p,
+      source,
+      source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,
+      brand: '',
+      category: 'parapharmacy',
+      available: true,
+      scraped_at: new Date().toISOString()
+    }));
+
+  } catch (error) {
+    console.error(`  ❌ Error scraping ${config.name}: ${error.message}`);
+    return [];
+  } finally {
+    await page.close();
+  }
+}
+
+export async function scrapeAll(queries = ['crema hidratante'], sources = ['promofarma']) {
+  console.log('🚀 Starting scraper...');
+
+  const browser = await puppeteer.launch({
+    headless: 'new',
+    executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
+    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
+  });
+
+  let totalProducts = 0;
+
+  try {
+    for (const query of queries) {
+      console.log(`\n📋 Query: "${query}"`);
+
+      for (const source of sources) {
+        const products = await scrapeSource(browser, source, query);
+        console.log(`  ✅ Found ${products.length} products`);
+
+        if (products.length > 0) {
+          try {
+            await axios.post(`${API_URL}/api/products/bulk`, { products });
+            totalProducts += products.length;
+          } catch (error) {
+            console.error(`  ❌ Error sending to API: ${error.message}`);
+          }
+        }
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+
+  console.log(`\n📊 Scraping completed. Total: ${totalProducts} products`);
+  return { success: true, total: totalProducts };
+}
@@ -7,6 +7,7 @@ import swaggerJsdoc from 'swagger-jsdoc';
 import swaggerUi from 'swagger-ui-express';
 import { config } from './config.js';
 import productsRouter from './routes/products.js';
+import scraperRouter from './routes/scraper.js';

 const app = express();

@@ -69,6 +70,7 @@ app.use('/api/docs', swaggerUi.serve, swaggerUi.setup(swaggerSpec, {

 // Routes
 app.use('/api/products', productsRouter);
+app.use('/api', scraperRouter);

 // Health check
 app.get('/api/health', (req, res) => {
@@ -0,0 +1,145 @@
+import { workflow, node, trigger } from '@n8n/workflow-sdk';
+
+const webhook = trigger({
+  type: 'n8n-nodes-base.webhook',
+  version: 2,
+  config: {
+    name: 'Webhook',
+    position: [240, 300],
+    parameters: {
+      httpMethod: 'POST',
+      path: 'scrape-fixed',
+      responseMode: 'lastNode'
+    }
+  },
+  output: [{}]
+});
+
+const parseInput = node({
+  type: 'n8n-nodes-base.code',
+  version: 2,
+  config: {
+    name: 'Parse Input',
+    position: [440, 300],
+    parameters: {
+      jsCode: `const body = $input.first().json.body || {};
+return [{ json: { queries: body.queries || 'crema hidratante', sources: body.sources || ['promofarma'] } }];`
+    }
+  },
+  output: [{}]
+});
+
+const generateTasks = node({
+  type: 'n8n-nodes-base.code',
+  version: 2,
+  config: {
+    name: 'Generate Tasks',
+    position: [640, 300],
+    parameters: {
+      jsCode: `const input = $input.first().json;
+const queries = input.queries.split(',').map(q => q.trim());
+const tasks = [];
+for (const q of queries) {
+  tasks.push({ query: q, source: 'promofarma', url: 'https://www.promofarma.com/es/search?q=' + encodeURIComponent(q) });
+}
+return tasks.map(t => ({ json: t }));`
+    }
+  },
+  output: [{}]
+});
+
+const scrape = node({
+  type: 'n8n-nodes-base.httpRequest',
+  version: 4.2,
+  config: {
+    name: 'Scrape',
+    position: [840, 300],
+    parameters: {
+      method: 'GET',
+      url: '={{ $json.url }}',
+      options: { timeout: 30000 }
+    }
+  },
+  output: [{}]
+});
+
+const extractProducts = node({
+  type: 'n8n-nodes-base.code',
+  version: 2,
+  config: {
+    name: 'Extract Products',
+    position: [1040, 300],
+    parameters: {
+      jsCode: `const data = $input.first().json;
+const html = data.data || '';
+const source = data.source;
+const products = [];
+const cardRegex = /<(?:div|article)[^>]*class="[^"]*product[^"]*"[^>]*>([\\s\\S]*?)<\\/(?:div|article)>/gi;
+const nameRegex = /<h[23][^>]*>([^<]+)<\\/h[23]>/i;
+const priceRegex = /class="[^"]*price[^"]*"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;
+let match;
+while ((match = cardRegex.exec(html)) !== null) {
+  const card = match[1];
+  const name = nameRegex.exec(card)?.[1]?.trim();
+  const priceStr = priceRegex.exec(card)?.[1]?.trim();
+  if (name && name.length > 3) {
+    const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;
+    if (price > 0 && price < 1000) {
+      products.push({
+        name: name.substring(0, 200),
+        price,
+        source,
+        source_product_id: source + '_' + Date.now() + '_' + Math.random().toString(36).substr(2, 6),
+        source_url: 'https://www.' + source + '.com',
+        brand: '',
+        category: 'parapharmacy',
+        available: true,
+        scraped_at: new Date().toISOString()
+      });
+    }
+  }
+}
+return products.slice(0, 5).map(p => ({ json: p }));`
+    }
+  },
+  output: [{}]
+});
+
+const sendToApi = node({
+  type: 'n8n-nodes-base.httpRequest',
+  version: 4.2,
+  config: {
+    name: 'Send to API',
+    position: [1240, 300],
+    parameters: {
+      method: 'POST',
+      url: 'http://parapharmacy-api:3002/api/products/bulk',
+      sendBody: true,
+      specifyBody: 'json',
+      jsonBody: '={{ JSON.stringify({ products: $input.all().map(i => i.json) }) }}'
+    }
+  },
+  output: [{}]
+});
+
+const response = node({
+  type: 'n8n-nodes-base.code',
+  version: 2,
+  config: {
+    name: 'Response',
+    position: [1440, 300],
+    parameters: {
+      jsCode: `return [{ json: { success: true, message: 'Scraping completed', products: $input.all().length } }];`
+    }
+  },
+  output: [{}]
+});
+
+export default workflow('scrape-fixed', 'Parapharmacy Scraper Fixed')
+  .add(webhook)
+  .to(parseInput)
+  .to(generateTasks)
+  .to(scrape)
+  .to(extractProducts)
+  .to(sendToApi)
+  .to(response);