feat: add scrape script and fix N8N webhook workflow

- Add direct scrape.js script for testing - Fix N8N webhook workflow (lastNode response mode) - Note: Real scraping requires Puppeteer for anti-bot sites
2026-07-16 13:14:56 +02:00
parent e2dd3474a4
commit 2ad4210221
2 changed files with 141 additions and 11 deletions
@@ -0,0 +1,130 @@
+import axios from 'axios';
+
+const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002';
+
+const QUERIES = ['crema hidratante', 'protector solar', 'vitaminas', 'capricare'];
+
+const SOURCES = {
+  promofarma: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`,
+  pharmarket: (q) => `https://www.pharmarket.es/catalogsearch/result/?q=${encodeURIComponent(q)}`,
+};
+
+async function scrapeSource(source, url) {
+  try {
+    console.log(`  🔍 Scraping ${source}: ${url}`);
+    const response = await axios.get(url, {
+      timeout: 15000,
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+      }
+    });
+    return response.data;
+  } catch (error) {
+    console.error(`  ❌ Error scraping ${source}: ${error.message}`);
+    return null;
+  }
+}
+
+function extractProducts(html, source) {
+  const products = [];
+  
+  // Try multiple patterns
+  const patterns = [
+    // Promofarma patterns
+    /<div[^>]*class="[^"]*product-card[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/gi,
+    /<article[^>]*class="[^"]*product[^"]*"[^>]*>([\s\S]*?)<\/article>/gi,
+    // Generic patterns
+    /<(?:div|li)[^>]*class="[^"]*(?:product|item)[^"]*"[^>]*>([\s\S]*?)<\/(?:div|li)>/gi,
+  ];
+  
+  const namePatterns = [
+    /<h[23][^>]*>([^<]+)<\/h[23]>/i,
+    /class="[^"]*(?:name|title)[^"]*"[^>]*>([^<]+)</i,
+  ];
+  
+  const pricePatterns = [
+    /class="[^"]*price[^"]*"[^>]*>([^<]*\d+[.,]\d+[^<]*)</i,
+    /(\d+[.,]\d+)\s*€/i,
+  ];
+  
+  for (const pattern of patterns) {
+    let match;
+    while ((match = pattern.exec(html)) !== null) {
+      const card = match[1];
+      
+      let name = null;
+      for (const np of namePatterns) {
+        const m = np.exec(card);
+        if (m) { name = m[1].trim(); break; }
+      }
+      
+      let price = 0;
+      for (const pp of pricePatterns) {
+        const m = pp.exec(card);
+        if (m) {
+          price = parseFloat(m[1].replace(/[^0-9.,]/g, '').replace(',', '.'));
+          break;
+        }
+      }
+      
+      if (name && name.length > 3 && name.length < 200 && price > 0 && price < 1000) {
+        products.push({
+          name: name.substring(0, 200),
+          price,
+          source,
+          source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,
+          source_url: `https://www.${source}.com`,
+          brand: '',
+          category: 'parapharmacy',
+          available: true,
+          scraped_at: new Date().toISOString()
+        });
+      }
+    }
+  }
+  
+  // Deduplicate
+  const seen = new Set();
+  return products.filter(p => {
+    const key = `${source}:${p.name.toLowerCase()}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  }).slice(0, 5);
+}
+
+async function main() {
+  console.log('🚀 Starting parapharmacy scraper...\n');
+  
+  let totalProducts = 0;
+  
+  for (const query of QUERIES) {
+    console.log(`\n📋 Query: "${query}"`);
+    
+    for (const [source, urlFn] of Object.entries(SOURCES)) {
+      const url = urlFn(query);
+      const html = await scrapeSource(source, url);
+      
+      if (html) {
+        const products = extractProducts(html, source);
+        console.log(`  ✅ Found ${products.length} products`);
+        
+        if (products.length > 0) {
+          try {
+            await axios.post(`${API_URL}/api/products/bulk`, { products });
+            totalProducts += products.length;
+          } catch (error) {
+            console.error(`  ❌ Error sending to API: ${error.message}`);
+          }
+        }
+      }
+    }
+  }
+  
+  console.log(`\n📊 Scraping completed. Total products: ${totalProducts}`);
+}
+
+main().catch(error => {
+  console.error('❌ Scraper failed:', error.message);
+  process.exit(1);
+});