feat: add scrape script and fix N8N webhook workflow
- Add direct scrape.js script for testing - Fix N8N webhook workflow (lastNode response mode) - Note: Real scraping requires Puppeteer for anti-bot sites
This commit is contained in:
Antoni Nuñez Romeu
@@ -0,0 +1,130 @@
|
||||
import axios from 'axios';
|
||||
|
||||
const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002';
|
||||
|
||||
const QUERIES = ['crema hidratante', 'protector solar', 'vitaminas', 'capricare'];
|
||||
|
||||
const SOURCES = {
|
||||
promofarma: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`,
|
||||
pharmarket: (q) => `https://www.pharmarket.es/catalogsearch/result/?q=${encodeURIComponent(q)}`,
|
||||
};
|
||||
|
||||
async function scrapeSource(source, url) {
|
||||
try {
|
||||
console.log(` 🔍 Scraping ${source}: ${url}`);
|
||||
const response = await axios.get(url, {
|
||||
timeout: 15000,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
}
|
||||
});
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
console.error(` ❌ Error scraping ${source}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractProducts(html, source) {
|
||||
const products = [];
|
||||
|
||||
// Try multiple patterns
|
||||
const patterns = [
|
||||
// Promofarma patterns
|
||||
/<div[^>]*class="[^"]*product-card[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/gi,
|
||||
/<article[^>]*class="[^"]*product[^"]*"[^>]*>([\s\S]*?)<\/article>/gi,
|
||||
// Generic patterns
|
||||
/<(?:div|li)[^>]*class="[^"]*(?:product|item)[^"]*"[^>]*>([\s\S]*?)<\/(?:div|li)>/gi,
|
||||
];
|
||||
|
||||
const namePatterns = [
|
||||
/<h[23][^>]*>([^<]+)<\/h[23]>/i,
|
||||
/class="[^"]*(?:name|title)[^"]*"[^>]*>([^<]+)</i,
|
||||
];
|
||||
|
||||
const pricePatterns = [
|
||||
/class="[^"]*price[^"]*"[^>]*>([^<]*\d+[.,]\d+[^<]*)</i,
|
||||
/(\d+[.,]\d+)\s*€/i,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
let match;
|
||||
while ((match = pattern.exec(html)) !== null) {
|
||||
const card = match[1];
|
||||
|
||||
let name = null;
|
||||
for (const np of namePatterns) {
|
||||
const m = np.exec(card);
|
||||
if (m) { name = m[1].trim(); break; }
|
||||
}
|
||||
|
||||
let price = 0;
|
||||
for (const pp of pricePatterns) {
|
||||
const m = pp.exec(card);
|
||||
if (m) {
|
||||
price = parseFloat(m[1].replace(/[^0-9.,]/g, '').replace(',', '.'));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (name && name.length > 3 && name.length < 200 && price > 0 && price < 1000) {
|
||||
products.push({
|
||||
name: name.substring(0, 200),
|
||||
price,
|
||||
source,
|
||||
source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,
|
||||
source_url: `https://www.${source}.com`,
|
||||
brand: '',
|
||||
category: 'parapharmacy',
|
||||
available: true,
|
||||
scraped_at: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate
|
||||
const seen = new Set();
|
||||
return products.filter(p => {
|
||||
const key = `${source}:${p.name.toLowerCase()}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
}).slice(0, 5);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('🚀 Starting parapharmacy scraper...\n');
|
||||
|
||||
let totalProducts = 0;
|
||||
|
||||
for (const query of QUERIES) {
|
||||
console.log(`\n📋 Query: "${query}"`);
|
||||
|
||||
for (const [source, urlFn] of Object.entries(SOURCES)) {
|
||||
const url = urlFn(query);
|
||||
const html = await scrapeSource(source, url);
|
||||
|
||||
if (html) {
|
||||
const products = extractProducts(html, source);
|
||||
console.log(` ✅ Found ${products.length} products`);
|
||||
|
||||
if (products.length > 0) {
|
||||
try {
|
||||
await axios.post(`${API_URL}/api/products/bulk`, { products });
|
||||
totalProducts += products.length;
|
||||
} catch (error) {
|
||||
console.error(` ❌ Error sending to API: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n📊 Scraping completed. Total products: ${totalProducts}`);
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error('❌ Scraper failed:', error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,10 +1,11 @@
|
||||
{
|
||||
"name": "Parapharmacy Manual Scraper (All Sources)",
|
||||
"name": "Parapharmacy Manual Scraper",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {
|
||||
"httpMethod": "POST",
|
||||
"path": "scrape-all",
|
||||
"responseMode": "lastNode",
|
||||
"options": {}
|
||||
},
|
||||
"id": "webhook",
|
||||
@@ -16,7 +17,7 @@
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "const body = $input.first().json.body || {};\nconst queries = body.queries || 'crema hidratante,protector solar,vitaminas,paracetamol';\nconst sources = body.sources || ['promofarma', 'pharmarket', 'docmorris', '1001farma', 'primor', 'mifarma'];\n\nreturn [{ json: { queries, sources } }];"
|
||||
"jsCode": "const body = $input.first().json.body || {};\nconst queries = body.queries || 'crema hidratante';\nconst sources = body.sources || ['promofarma'];\n\nreturn [{ json: { queries, sources } }];"
|
||||
},
|
||||
"id": "parse-input",
|
||||
"name": "Parse Input",
|
||||
@@ -50,7 +51,7 @@
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "const data = $input.first().json;\nconst html = data.data || '';\nconst source = data.source;\nconst query = data.query;\n\nconst products = [];\n\n// Generic extraction\nconst cardRegex = /<(?:div|article|li)[^>]*class=\"[^\"]*(?:product|item)[^\"]*\"[^>]*>([\\s\\S]*?)<\\/(?:div|article|li)>/gi;\nconst nameRegex = /<(?:h[23]|a|span)[^>]*class=\"[^\"]*(?:name|title)[^\"]*\"[^>]*>([^<]+)<\\/[^>]+>/i;\nconst priceRegex = /class=\"[^\"]*price[^\"]*\"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;\nconst linkRegex = /<a[^>]*href=\"(https?:\\/\\/[^\"]+)\"/i;\nconst imageRegex = /<img[^>]*src=\"(https?:\\/\\/[^\"\\.]+\\.(?:jpg|jpeg|png|webp)[^\"]*)\"/i;\n\nlet match;\nwhile ((match = cardRegex.exec(html)) !== null) {\n const card = match[1];\n const name = nameRegex.exec(card)?.[1]?.trim();\n const priceStr = priceRegex.exec(card)?.[1]?.trim();\n const link = linkRegex.exec(card)?.[1];\n const image = imageRegex.exec(card)?.[1];\n \n if (name && name.length > 3 && name.length < 200) {\n const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;\n if (price > 0 && price < 1000) {\n products.push({\n name: name.substring(0, 200),\n price,\n source_url: link || `https://${source}.com`,\n image_url: image || null,\n source,\n source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,\n brand: '',\n category: 'parapharmacy',\n available: true,\n scraped_at: new Date().toISOString()\n });\n }\n }\n}\n\n// Deduplicate\nconst seen = new Set();\nconst unique = products.filter(p => {\n const key = `${source}:${p.name.toLowerCase()}`;\n if (seen.has(key)) return false;\n seen.add(key);\n return true;\n});\n\nreturn unique.slice(0, 5).map(p => ({ json: p }));"
|
||||
"jsCode": "const data = $input.first().json;\nconst html = data.data || '';\nconst source = data.source;\n\nconst products = [];\n\n// Generic extraction\nconst cardRegex = /<(?:div|article|li)[^>]*class=\"[^\"]*(?:product|item)[^\"]*\"[^>]*>([\\s\\S]*?)<\\/(?:div|article|li)>/gi;\nconst nameRegex = /<(?:h[23]|a|span)[^>]*class=\"[^\"]*(?:name|title)[^\"]*\"[^>]*>([^<]+)<\\/[^>]+>/i;\nconst priceRegex = /class=\"[^\"]*price[^\"]*\"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;\nconst linkRegex = /<a[^>]*href=\"(https?:\\/\\/[^\"]+)\"/i;\nconst imageRegex = /<img[^>]*src=\"(https?:\\/\\/[^\"\\.]+\\.(?:jpg|jpeg|png|webp)[^\"]*)\"/i;\n\nlet match;\nwhile ((match = cardRegex.exec(html)) !== null) {\n const card = match[1];\n const name = nameRegex.exec(card)?.[1]?.trim();\n const priceStr = priceRegex.exec(card)?.[1]?.trim();\n const link = linkRegex.exec(card)?.[1];\n const image = imageRegex.exec(card)?.[1];\n \n if (name && name.length > 3 && name.length < 200) {\n const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;\n if (price > 0 && price < 1000) {\n products.push({\n name: name.substring(0, 200),\n price,\n source_url: link || `https://${source}.com`,\n image_url: image || null,\n source,\n source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,\n brand: '',\n category: 'parapharmacy',\n available: true,\n scraped_at: new Date().toISOString()\n });\n }\n }\n}\n\n// Deduplicate\nconst seen = new Set();\nconst unique = products.filter(p => {\n const key = `${source}:${p.name.toLowerCase()}`;\n if (seen.has(key)) return false;\n seen.add(key);\n return true;\n});\n\nreturn unique.slice(0, 5).map(p => ({ json: p }));"
|
||||
},
|
||||
"id": "extract",
|
||||
"name": "Extract Products",
|
||||
@@ -75,13 +76,12 @@
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"respondWith": "json",
|
||||
"responseBody": "={{ { success: true, message: 'Scraping completed', products: $input.all().length } }}"
|
||||
"jsCode": "const result = $input.first().json;\nreturn [{ json: { success: true, message: 'Scraping completed', result } }];"
|
||||
},
|
||||
"id": "respond",
|
||||
"name": "Respond",
|
||||
"type": "n8n-nodes-base.respondToWebhook",
|
||||
"typeVersion": 1.1,
|
||||
"id": "format-response",
|
||||
"name": "Format Response",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [1540, 300]
|
||||
}
|
||||
],
|
||||
@@ -91,9 +91,9 @@
|
||||
"Generate Tasks": { "main": [[{ "node": "Scrape", "type": "main", "index": 0 }]] },
|
||||
"Scrape": { "main": [[{ "node": "Extract Products", "type": "main", "index": 0 }]] },
|
||||
"Extract Products": { "main": [[{ "node": "Send to API", "type": "main", "index": 0 }]] },
|
||||
"Send to API": { "main": [[{ "node": "Respond", "type": "main", "index": 0 }]] }
|
||||
"Send to API": { "main": [[{ "node": "Format Response", "type": "main", "index": 0 }]] }
|
||||
},
|
||||
"active": true,
|
||||
"settings": { "executionOrder": "v1" },
|
||||
"tags": [{ "name": "parapharmacy" }, { "name": "scraper" }, { "name": "webhook" }]
|
||||
"tags": [{ "name": "parapharmacy" }, { "name": "scraper" }]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user