FarmaFinder/apps/parapharmacy-api/src/scraper.js

import puppeteer from 'puppeteer';
import axios from 'axios';

const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002';

const SOURCES = {
  promofarma: {
    name: 'Promofarma',
    searchUrl: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`,
    selectors: {
      product: 'article[data-name]',
      name: 'article[data-name]',
      price: 'article[data-pvp]',
      link: 'a',
      image: 'img'
    }
  },
  docmorris: {
    name: 'DocMorris',
    searchUrl: (q) => `https://www.docmorris.es/search?query=${encodeURIComponent(q)}`,
    selectors: {
      product: '[class*="product-card"], [class*="product-item"]',
      name: '[class*="product-name"], h3',
      price: '[class*="price"]',
      link: 'a',
      image: 'img'
    }
  },
  primor: {
    name: 'Primor',
    searchUrl: (q) => `https://www.primor.eu/catalogsearch/result/?q=${encodeURIComponent(q)}`,
    selectors: {
      product: 'form.product-item',
      name: 'form.product-item',
      price: 'form.product-item',
      link: 'a',
      image: 'img'
    }
  }
};

async function scrapeSource(browser, source, query) {
  const config = SOURCES[source];
  if (!config) return [];

  const page = await browser.newPage();
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');

  try {
    console.log(`  🔍 Scraping ${config.name}: ${query}`);
    const url = config.searchUrl(query);
    console.log(`  📍 URL: ${url}`);

    await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 });

    // Wait for products to load
    await page.waitForSelector(config.selectors.product, { timeout: 15000 }).catch(() => {});

    // Get page title for debugging
    const title = await page.title();
    console.log(`  📄 Page title: ${title}`);

    // Get page content for debugging
    const content = await page.content();
    console.log(`  📄 Page length: ${content.length} chars`);

    // Try to find product elements with various selectors
    const productInfo = await page.evaluate(() => {
      // Try multiple selector strategies
      const selectors = [
        '[data-testid*="product"]',
        '[class*="ProductCard"]',
        '[class*="product-card"]',
        'a[href*="/p/"]',
        '[class*="Product"]',
        'article',
        '[role="listitem"]',
        '.product-item',
        '.product',
        '[data-product]'
      ];

      for (const sel of selectors) {
        const els = document.querySelectorAll(sel);
        if (els.length > 0) {
          return {
            selector: sel,
            count: els.length,
            samples: Array.from(els).slice(0, 3).map(el => ({
              tag: el.tagName,
              class: el.className?.substring(0, 150),
              html: el.outerHTML?.substring(0, 500)
            }))
          };
        }
      }

      // Fallback: look for any element with price-like content
      const allText = document.body.innerText;
      const priceMatch = allText.match(/\d+[.,]\d{2}\s*€/g);

      return {
        selector: 'none found',
        priceMatches: priceMatch?.slice(0, 5) || [],
        bodyTextSample: document.body.innerText.substring(0, 500)
      };
    });
    console.log(`  🔍 Product info:`, JSON.stringify(productInfo, null, 2));

    const products = await page.evaluate((selectors, searchQuery) => {
      const results = [];
      const cards = document.querySelectorAll(selectors.product);

      // Get the search query to filter relevant products
      const query = searchQuery.toLowerCase();

      cards.forEach(card => {
        // Try data attributes first (Promofarma style)
        let name = card.getAttribute('data-name');
        let priceStr = card.getAttribute('data-pvp');

        // If no data attributes, try to extract from content
        if (!name) {
          const nameEl = card.querySelector('[class*="name"], h3, h2, .product-name');
          name = nameEl?.innerText?.trim();
        }
        if (!priceStr) {
          const priceEl = card.querySelector('[class*="price"], .price');
          priceStr = priceEl?.innerText?.trim();
        }

        const linkEl = card.querySelector(selectors.link);
        const imgEl = card.querySelector(selectors.image);

        if (name && priceStr) {
          const price = parseFloat(priceStr.replace(/[^0-9.,]/g, '').replace(',', '.')) || 0;

          // Filter: only include products that contain the search query
          const nameLower = name.toLowerCase();
          if (name && price > 0 && price < 1000 && nameLower.includes(query)) {
            results.push({
              name: name.substring(0, 200),
              price,
              source_url: linkEl?.href || '',
              image_url: imgEl?.src || null
            });
          }
        }
      });

      return results;
    }, config.selectors, query);

    console.log(`  📦 Found ${products.length} products`);
    return products.slice(0, 10).map(p => ({
      ...p,
      source,
      source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,
      brand: '',
      category: 'parapharmacy',
      available: true,
      scraped_at: new Date().toISOString()
    }));

  } catch (error) {
    console.error(`  ❌ Error scraping ${config.name}: ${error.message}`);
    return [];
  } finally {
    await page.close();
  }
}

export async function scrapeAll(queries = ['crema hidratante'], sources = ['promofarma']) {
  console.log('🚀 Starting scraper...');

  const browser = await puppeteer.launch({
    headless: 'new',
    executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
  });

  let totalProducts = 0;

  try {
    for (const query of queries) {
      console.log(`\n📋 Query: "${query}"`);

      for (const source of sources) {
        const products = await scrapeSource(browser, source, query);
        console.log(`  ✅ Found ${products.length} products`);

        if (products.length > 0) {
          try {
            await axios.post(`${API_URL}/api/products/bulk`, { products });
            totalProducts += products.length;
          } catch (error) {
            console.error(`  ❌ Error sending to API: ${error.message}`);
          }
        }
      }
    }
  } finally {
    await browser.close();
  }

  console.log(`\n📊 Scraping completed. Total: ${totalProducts} products`);
  return { success: true, total: totalProducts };
}