a5a75d3249
- Added Puppeteer with Chrome in Docker - Scraper extracts products using data attributes - Added /api/scrape endpoint - Tested: 10 products scraped from Promofarma The scraper now works with Promofarma's HTML structure which uses data-name, data-pvp attributes on article elements.
146 lines
3.6 KiB
JavaScript
146 lines
3.6 KiB
JavaScript
import { workflow, node, trigger } from '@n8n/workflow-sdk';
|
|
|
|
const webhook = trigger({
|
|
type: 'n8n-nodes-base.webhook',
|
|
version: 2,
|
|
config: {
|
|
name: 'Webhook',
|
|
position: [240, 300],
|
|
parameters: {
|
|
httpMethod: 'POST',
|
|
path: 'scrape-fixed',
|
|
responseMode: 'lastNode'
|
|
}
|
|
},
|
|
output: [{}]
|
|
});
|
|
|
|
const parseInput = node({
|
|
type: 'n8n-nodes-base.code',
|
|
version: 2,
|
|
config: {
|
|
name: 'Parse Input',
|
|
position: [440, 300],
|
|
parameters: {
|
|
jsCode: `const body = $input.first().json.body || {};
|
|
return [{ json: { queries: body.queries || 'crema hidratante', sources: body.sources || ['promofarma'] } }];`
|
|
}
|
|
},
|
|
output: [{}]
|
|
});
|
|
|
|
const generateTasks = node({
|
|
type: 'n8n-nodes-base.code',
|
|
version: 2,
|
|
config: {
|
|
name: 'Generate Tasks',
|
|
position: [640, 300],
|
|
parameters: {
|
|
jsCode: `const input = $input.first().json;
|
|
const queries = input.queries.split(',').map(q => q.trim());
|
|
const tasks = [];
|
|
for (const q of queries) {
|
|
tasks.push({ query: q, source: 'promofarma', url: 'https://www.promofarma.com/es/search?q=' + encodeURIComponent(q) });
|
|
}
|
|
return tasks.map(t => ({ json: t }));`
|
|
}
|
|
},
|
|
output: [{}]
|
|
});
|
|
|
|
const scrape = node({
|
|
type: 'n8n-nodes-base.httpRequest',
|
|
version: 4.2,
|
|
config: {
|
|
name: 'Scrape',
|
|
position: [840, 300],
|
|
parameters: {
|
|
method: 'GET',
|
|
url: '={{ $json.url }}',
|
|
options: { timeout: 30000 }
|
|
}
|
|
},
|
|
output: [{}]
|
|
});
|
|
|
|
const extractProducts = node({
|
|
type: 'n8n-nodes-base.code',
|
|
version: 2,
|
|
config: {
|
|
name: 'Extract Products',
|
|
position: [1040, 300],
|
|
parameters: {
|
|
jsCode: `const data = $input.first().json;
|
|
const html = data.data || '';
|
|
const source = data.source;
|
|
const products = [];
|
|
const cardRegex = /<(?:div|article)[^>]*class="[^"]*product[^"]*"[^>]*>([\\s\\S]*?)<\\/(?:div|article)>/gi;
|
|
const nameRegex = /<h[23][^>]*>([^<]+)<\\/h[23]>/i;
|
|
const priceRegex = /class="[^"]*price[^"]*"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;
|
|
let match;
|
|
while ((match = cardRegex.exec(html)) !== null) {
|
|
const card = match[1];
|
|
const name = nameRegex.exec(card)?.[1]?.trim();
|
|
const priceStr = priceRegex.exec(card)?.[1]?.trim();
|
|
if (name && name.length > 3) {
|
|
const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;
|
|
if (price > 0 && price < 1000) {
|
|
products.push({
|
|
name: name.substring(0, 200),
|
|
price,
|
|
source,
|
|
source_product_id: source + '_' + Date.now() + '_' + Math.random().toString(36).substr(2, 6),
|
|
source_url: 'https://www.' + source + '.com',
|
|
brand: '',
|
|
category: 'parapharmacy',
|
|
available: true,
|
|
scraped_at: new Date().toISOString()
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return products.slice(0, 5).map(p => ({ json: p }));`
|
|
}
|
|
},
|
|
output: [{}]
|
|
});
|
|
|
|
const sendToApi = node({
|
|
type: 'n8n-nodes-base.httpRequest',
|
|
version: 4.2,
|
|
config: {
|
|
name: 'Send to API',
|
|
position: [1240, 300],
|
|
parameters: {
|
|
method: 'POST',
|
|
url: 'http://parapharmacy-api:3002/api/products/bulk',
|
|
sendBody: true,
|
|
specifyBody: 'json',
|
|
jsonBody: '={{ JSON.stringify({ products: $input.all().map(i => i.json) }) }}'
|
|
}
|
|
},
|
|
output: [{}]
|
|
});
|
|
|
|
const response = node({
|
|
type: 'n8n-nodes-base.code',
|
|
version: 2,
|
|
config: {
|
|
name: 'Response',
|
|
position: [1440, 300],
|
|
parameters: {
|
|
jsCode: `return [{ json: { success: true, message: 'Scraping completed', products: $input.all().length } }];`
|
|
}
|
|
},
|
|
output: [{}]
|
|
});
|
|
|
|
export default workflow('scrape-fixed', 'Parapharmacy Scraper Fixed')
|
|
.add(webhook)
|
|
.to(parseInput)
|
|
.to(generateTasks)
|
|
.to(scrape)
|
|
.to(extractProducts)
|
|
.to(sendToApi)
|
|
.to(response);
|