diff --git a/apps/parapharmacy-api/scripts/scrape.js b/apps/parapharmacy-api/scripts/scrape.js new file mode 100644 index 0000000..7c28e45 --- /dev/null +++ b/apps/parapharmacy-api/scripts/scrape.js @@ -0,0 +1,130 @@ +import axios from 'axios'; + +const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002'; + +const QUERIES = ['crema hidratante', 'protector solar', 'vitaminas', 'capricare']; + +const SOURCES = { + promofarma: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`, + pharmarket: (q) => `https://www.pharmarket.es/catalogsearch/result/?q=${encodeURIComponent(q)}`, +}; + +async function scrapeSource(source, url) { + try { + console.log(` šŸ” Scraping ${source}: ${url}`); + const response = await axios.get(url, { + timeout: 15000, + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' + } + }); + return response.data; + } catch (error) { + console.error(` āŒ Error scraping ${source}: ${error.message}`); + return null; + } +} + +function extractProducts(html, source) { + const products = []; + + // Try multiple patterns + const patterns = [ + // Promofarma patterns + /]*class="[^"]*product-card[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/gi, + /]*class="[^"]*product[^"]*"[^>]*>([\s\S]*?)<\/article>/gi, + // Generic patterns + /<(?:div|li)[^>]*class="[^"]*(?:product|item)[^"]*"[^>]*>([\s\S]*?)<\/(?:div|li)>/gi, + ]; + + const namePatterns = [ + /]*>([^<]+)<\/h[23]>/i, + /class="[^"]*(?:name|title)[^"]*"[^>]*>([^<]+)]*>([^<]*\d+[.,]\d+[^<]*) 3 && name.length < 200 && price > 0 && price < 1000) { + products.push({ + name: name.substring(0, 200), + price, + source, + source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`, + source_url: `https://www.${source}.com`, + brand: '', + category: 'parapharmacy', + available: true, + scraped_at: new Date().toISOString() + }); + } + } + } + + // Deduplicate + const seen = new Set(); + return products.filter(p => { + const key = `${source}:${p.name.toLowerCase()}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }).slice(0, 5); +} + +async function main() { + console.log('šŸš€ Starting parapharmacy scraper...\n'); + + let totalProducts = 0; + + for (const query of QUERIES) { + console.log(`\nšŸ“‹ Query: "${query}"`); + + for (const [source, urlFn] of Object.entries(SOURCES)) { + const url = urlFn(query); + const html = await scrapeSource(source, url); + + if (html) { + const products = extractProducts(html, source); + console.log(` āœ… Found ${products.length} products`); + + if (products.length > 0) { + try { + await axios.post(`${API_URL}/api/products/bulk`, { products }); + totalProducts += products.length; + } catch (error) { + console.error(` āŒ Error sending to API: ${error.message}`); + } + } + } + } + } + + console.log(`\nšŸ“Š Scraping completed. Total products: ${totalProducts}`); +} + +main().catch(error => { + console.error('āŒ Scraper failed:', error.message); + process.exit(1); +}); diff --git a/n8n/workflows/parapharmacy-webhook-scraper.json b/n8n/workflows/parapharmacy-webhook-scraper.json index c9bac10..68305a6 100644 --- a/n8n/workflows/parapharmacy-webhook-scraper.json +++ b/n8n/workflows/parapharmacy-webhook-scraper.json @@ -1,10 +1,11 @@ { - "name": "Parapharmacy Manual Scraper (All Sources)", + "name": "Parapharmacy Manual Scraper", "nodes": [ { "parameters": { "httpMethod": "POST", "path": "scrape-all", + "responseMode": "lastNode", "options": {} }, "id": "webhook", @@ -16,7 +17,7 @@ }, { "parameters": { - "jsCode": "const body = $input.first().json.body || {};\nconst queries = body.queries || 'crema hidratante,protector solar,vitaminas,paracetamol';\nconst sources = body.sources || ['promofarma', 'pharmarket', 'docmorris', '1001farma', 'primor', 'mifarma'];\n\nreturn [{ json: { queries, sources } }];" + "jsCode": "const body = $input.first().json.body || {};\nconst queries = body.queries || 'crema hidratante';\nconst sources = body.sources || ['promofarma'];\n\nreturn [{ json: { queries, sources } }];" }, "id": "parse-input", "name": "Parse Input", @@ -50,7 +51,7 @@ }, { "parameters": { - "jsCode": "const data = $input.first().json;\nconst html = data.data || '';\nconst source = data.source;\nconst query = data.query;\n\nconst products = [];\n\n// Generic extraction\nconst cardRegex = /<(?:div|article|li)[^>]*class=\"[^\"]*(?:product|item)[^\"]*\"[^>]*>([\\s\\S]*?)<\\/(?:div|article|li)>/gi;\nconst nameRegex = /<(?:h[23]|a|span)[^>]*class=\"[^\"]*(?:name|title)[^\"]*\"[^>]*>([^<]+)<\\/[^>]+>/i;\nconst priceRegex = /class=\"[^\"]*price[^\"]*\"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;\nconst linkRegex = /]*href=\"(https?:\\/\\/[^\"]+)\"/i;\nconst imageRegex = /]*src=\"(https?:\\/\\/[^\"\\.]+\\.(?:jpg|jpeg|png|webp)[^\"]*)\"/i;\n\nlet match;\nwhile ((match = cardRegex.exec(html)) !== null) {\n const card = match[1];\n const name = nameRegex.exec(card)?.[1]?.trim();\n const priceStr = priceRegex.exec(card)?.[1]?.trim();\n const link = linkRegex.exec(card)?.[1];\n const image = imageRegex.exec(card)?.[1];\n \n if (name && name.length > 3 && name.length < 200) {\n const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;\n if (price > 0 && price < 1000) {\n products.push({\n name: name.substring(0, 200),\n price,\n source_url: link || `https://${source}.com`,\n image_url: image || null,\n source,\n source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,\n brand: '',\n category: 'parapharmacy',\n available: true,\n scraped_at: new Date().toISOString()\n });\n }\n }\n}\n\n// Deduplicate\nconst seen = new Set();\nconst unique = products.filter(p => {\n const key = `${source}:${p.name.toLowerCase()}`;\n if (seen.has(key)) return false;\n seen.add(key);\n return true;\n});\n\nreturn unique.slice(0, 5).map(p => ({ json: p }));" + "jsCode": "const data = $input.first().json;\nconst html = data.data || '';\nconst source = data.source;\n\nconst products = [];\n\n// Generic extraction\nconst cardRegex = /<(?:div|article|li)[^>]*class=\"[^\"]*(?:product|item)[^\"]*\"[^>]*>([\\s\\S]*?)<\\/(?:div|article|li)>/gi;\nconst nameRegex = /<(?:h[23]|a|span)[^>]*class=\"[^\"]*(?:name|title)[^\"]*\"[^>]*>([^<]+)<\\/[^>]+>/i;\nconst priceRegex = /class=\"[^\"]*price[^\"]*\"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;\nconst linkRegex = /]*href=\"(https?:\\/\\/[^\"]+)\"/i;\nconst imageRegex = /]*src=\"(https?:\\/\\/[^\"\\.]+\\.(?:jpg|jpeg|png|webp)[^\"]*)\"/i;\n\nlet match;\nwhile ((match = cardRegex.exec(html)) !== null) {\n const card = match[1];\n const name = nameRegex.exec(card)?.[1]?.trim();\n const priceStr = priceRegex.exec(card)?.[1]?.trim();\n const link = linkRegex.exec(card)?.[1];\n const image = imageRegex.exec(card)?.[1];\n \n if (name && name.length > 3 && name.length < 200) {\n const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;\n if (price > 0 && price < 1000) {\n products.push({\n name: name.substring(0, 200),\n price,\n source_url: link || `https://${source}.com`,\n image_url: image || null,\n source,\n source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,\n brand: '',\n category: 'parapharmacy',\n available: true,\n scraped_at: new Date().toISOString()\n });\n }\n }\n}\n\n// Deduplicate\nconst seen = new Set();\nconst unique = products.filter(p => {\n const key = `${source}:${p.name.toLowerCase()}`;\n if (seen.has(key)) return false;\n seen.add(key);\n return true;\n});\n\nreturn unique.slice(0, 5).map(p => ({ json: p }));" }, "id": "extract", "name": "Extract Products", @@ -75,13 +76,12 @@ }, { "parameters": { - "respondWith": "json", - "responseBody": "={{ { success: true, message: 'Scraping completed', products: $input.all().length } }}" + "jsCode": "const result = $input.first().json;\nreturn [{ json: { success: true, message: 'Scraping completed', result } }];" }, - "id": "respond", - "name": "Respond", - "type": "n8n-nodes-base.respondToWebhook", - "typeVersion": 1.1, + "id": "format-response", + "name": "Format Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, "position": [1540, 300] } ], @@ -91,9 +91,9 @@ "Generate Tasks": { "main": [[{ "node": "Scrape", "type": "main", "index": 0 }]] }, "Scrape": { "main": [[{ "node": "Extract Products", "type": "main", "index": 0 }]] }, "Extract Products": { "main": [[{ "node": "Send to API", "type": "main", "index": 0 }]] }, - "Send to API": { "main": [[{ "node": "Respond", "type": "main", "index": 0 }]] } + "Send to API": { "main": [[{ "node": "Format Response", "type": "main", "index": 0 }]] } }, "active": true, "settings": { "executionOrder": "v1" }, - "tags": [{ "name": "parapharmacy" }, { "name": "scraper" }, { "name": "webhook" }] + "tags": [{ "name": "parapharmacy" }, { "name": "scraper" }] }