feat: complete N8N scrapers for 6 sources and seed script
- Create comprehensive N8N workflow for all 6 parapharmacy sources - Create webhook-triggered manual scraping workflow - Add seed script with 20 sample products - Update n8n/README.md with complete documentation - Update docs/parapharmacy.md with seeding instructions - Add seed script to package.json Sources configured: - Promofarma - Pharmarket - DocMorris - 1001Farma - Primor - MiFarma
This commit is contained in:
Antoni Nuñez Romeu
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"name": "Parapharmacy Manual Scraper (All Sources)",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {
|
||||
"httpMethod": "POST",
|
||||
"path": "scrape-all",
|
||||
"options": {}
|
||||
},
|
||||
"id": "webhook",
|
||||
"name": "Webhook",
|
||||
"type": "n8n-nodes-base.webhook",
|
||||
"typeVersion": 2,
|
||||
"position": [220, 300],
|
||||
"webhookId": "scrape-all"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "const body = $input.first().json.body || {};\nconst queries = body.queries || 'crema hidratante,protector solar,vitaminas,paracetamol';\nconst sources = body.sources || ['promofarma', 'pharmarket', 'docmorris', '1001farma', 'primor', 'mifarma'];\n\nreturn [{ json: { queries, sources } }];"
|
||||
},
|
||||
"id": "parse-input",
|
||||
"name": "Parse Input",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [440, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "const input = $input.first().json;\nconst queries = input.queries.split(',').map(q => q.trim());\nconst sources = input.sources;\n\nconst tasks = [];\nfor (const query of queries) {\n for (const source of sources) {\n const urls = {\n promofarma: `https://www.promofarma.com/es/search?q=${encodeURIComponent(query)}`,\n pharmarket: `https://www.pharmarket.es/catalogsearch/result/?q=${encodeURIComponent(query)}`,\n docmorris: `https://www.docmorris.es/search?query=${encodeURIComponent(query)}`,\n '1001farma': `https://www.1001farma.net/buscar?s=${encodeURIComponent(query)}`,\n primor: `https://www.primor.eu/search?s=${encodeURIComponent(query)}`,\n mifarma: `https://www.mifarma.es/buscador?q=${encodeURIComponent(query)}`\n };\n \n tasks.push({\n query,\n source,\n url: urls[source]\n });\n }\n}\n\nreturn tasks.map(t => ({ json: t }));"
|
||||
},
|
||||
"id": "generate-tasks",
|
||||
"name": "Generate Tasks",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [660, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"method": "GET",
|
||||
"url": "={{ $json.url }}",
|
||||
"options": {
|
||||
"timeout": 30000
|
||||
}
|
||||
},
|
||||
"id": "scrape",
|
||||
"name": "Scrape",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [880, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "const data = $input.first().json;\nconst html = data.data || '';\nconst source = data.source;\nconst query = data.query;\n\nconst products = [];\n\n// Generic extraction\nconst cardRegex = /<(?:div|article|li)[^>]*class=\"[^\"]*(?:product|item)[^\"]*\"[^>]*>([\\s\\S]*?)<\\/(?:div|article|li)>/gi;\nconst nameRegex = /<(?:h[23]|a|span)[^>]*class=\"[^\"]*(?:name|title)[^\"]*\"[^>]*>([^<]+)<\\/[^>]+>/i;\nconst priceRegex = /class=\"[^\"]*price[^\"]*\"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;\nconst linkRegex = /<a[^>]*href=\"(https?:\\/\\/[^\"]+)\"/i;\nconst imageRegex = /<img[^>]*src=\"(https?:\\/\\/[^\"\\.]+\\.(?:jpg|jpeg|png|webp)[^\"]*)\"/i;\n\nlet match;\nwhile ((match = cardRegex.exec(html)) !== null) {\n const card = match[1];\n const name = nameRegex.exec(card)?.[1]?.trim();\n const priceStr = priceRegex.exec(card)?.[1]?.trim();\n const link = linkRegex.exec(card)?.[1];\n const image = imageRegex.exec(card)?.[1];\n \n if (name && name.length > 3 && name.length < 200) {\n const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;\n if (price > 0 && price < 1000) {\n products.push({\n name: name.substring(0, 200),\n price,\n source_url: link || `https://${source}.com`,\n image_url: image || null,\n source,\n source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,\n brand: '',\n category: 'parapharmacy',\n available: true,\n scraped_at: new Date().toISOString()\n });\n }\n }\n}\n\n// Deduplicate\nconst seen = new Set();\nconst unique = products.filter(p => {\n const key = `${source}:${p.name.toLowerCase()}`;\n if (seen.has(key)) return false;\n seen.add(key);\n return true;\n});\n\nreturn unique.slice(0, 5).map(p => ({ json: p }));"
|
||||
},
|
||||
"id": "extract",
|
||||
"name": "Extract Products",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [1100, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"method": "POST",
|
||||
"url": "http://parapharmacy-api:3002/api/products/bulk",
|
||||
"sendBody": true,
|
||||
"specifyBody": "json",
|
||||
"jsonBody": "={{ JSON.stringify({ products: $input.all().map(i => i.json) }) }}",
|
||||
"options": {}
|
||||
},
|
||||
"id": "send-bulk",
|
||||
"name": "Send to API",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [1320, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"respondWith": "json",
|
||||
"responseBody": "={{ { success: true, message: 'Scraping completed', products: $input.all().length } }}"
|
||||
},
|
||||
"id": "respond",
|
||||
"name": "Respond",
|
||||
"type": "n8n-nodes-base.respondToWebhook",
|
||||
"typeVersion": 1.1,
|
||||
"position": [1540, 300]
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Webhook": { "main": [[{ "node": "Parse Input", "type": "main", "index": 0 }]] },
|
||||
"Parse Input": { "main": [[{ "node": "Generate Tasks", "type": "main", "index": 0 }]] },
|
||||
"Generate Tasks": { "main": [[{ "node": "Scrape", "type": "main", "index": 0 }]] },
|
||||
"Scrape": { "main": [[{ "node": "Extract Products", "type": "main", "index": 0 }]] },
|
||||
"Extract Products": { "main": [[{ "node": "Send to API", "type": "main", "index": 0 }]] },
|
||||
"Send to API": { "main": [[{ "node": "Respond", "type": "main", "index": 0 }]] }
|
||||
},
|
||||
"active": true,
|
||||
"settings": { "executionOrder": "v1" },
|
||||
"tags": [{ "name": "parapharmacy" }, { "name": "scraper" }, { "name": "webhook" }]
|
||||
}
|
||||
Reference in New Issue
Block a user