feat: working Puppeteer scraper for Promofarma
- Added Puppeteer with Chrome in Docker - Scraper extracts products using data attributes - Added /api/scrape endpoint - Tested: 10 products scraped from Promofarma The scraper now works with Promofarma's HTML structure which uses data-name, data-pvp attributes on article elements.
This commit is contained in:
@@ -0,0 +1,145 @@
|
||||
import { workflow, node, trigger } from '@n8n/workflow-sdk';
|
||||
|
||||
const webhook = trigger({
|
||||
type: 'n8n-nodes-base.webhook',
|
||||
version: 2,
|
||||
config: {
|
||||
name: 'Webhook',
|
||||
position: [240, 300],
|
||||
parameters: {
|
||||
httpMethod: 'POST',
|
||||
path: 'scrape-fixed',
|
||||
responseMode: 'lastNode'
|
||||
}
|
||||
},
|
||||
output: [{}]
|
||||
});
|
||||
|
||||
const parseInput = node({
|
||||
type: 'n8n-nodes-base.code',
|
||||
version: 2,
|
||||
config: {
|
||||
name: 'Parse Input',
|
||||
position: [440, 300],
|
||||
parameters: {
|
||||
jsCode: `const body = $input.first().json.body || {};
|
||||
return [{ json: { queries: body.queries || 'crema hidratante', sources: body.sources || ['promofarma'] } }];`
|
||||
}
|
||||
},
|
||||
output: [{}]
|
||||
});
|
||||
|
||||
const generateTasks = node({
|
||||
type: 'n8n-nodes-base.code',
|
||||
version: 2,
|
||||
config: {
|
||||
name: 'Generate Tasks',
|
||||
position: [640, 300],
|
||||
parameters: {
|
||||
jsCode: `const input = $input.first().json;
|
||||
const queries = input.queries.split(',').map(q => q.trim());
|
||||
const tasks = [];
|
||||
for (const q of queries) {
|
||||
tasks.push({ query: q, source: 'promofarma', url: 'https://www.promofarma.com/es/search?q=' + encodeURIComponent(q) });
|
||||
}
|
||||
return tasks.map(t => ({ json: t }));`
|
||||
}
|
||||
},
|
||||
output: [{}]
|
||||
});
|
||||
|
||||
const scrape = node({
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
version: 4.2,
|
||||
config: {
|
||||
name: 'Scrape',
|
||||
position: [840, 300],
|
||||
parameters: {
|
||||
method: 'GET',
|
||||
url: '={{ $json.url }}',
|
||||
options: { timeout: 30000 }
|
||||
}
|
||||
},
|
||||
output: [{}]
|
||||
});
|
||||
|
||||
const extractProducts = node({
|
||||
type: 'n8n-nodes-base.code',
|
||||
version: 2,
|
||||
config: {
|
||||
name: 'Extract Products',
|
||||
position: [1040, 300],
|
||||
parameters: {
|
||||
jsCode: `const data = $input.first().json;
|
||||
const html = data.data || '';
|
||||
const source = data.source;
|
||||
const products = [];
|
||||
const cardRegex = /<(?:div|article)[^>]*class="[^"]*product[^"]*"[^>]*>([\\s\\S]*?)<\\/(?:div|article)>/gi;
|
||||
const nameRegex = /<h[23][^>]*>([^<]+)<\\/h[23]>/i;
|
||||
const priceRegex = /class="[^"]*price[^"]*"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i;
|
||||
let match;
|
||||
while ((match = cardRegex.exec(html)) !== null) {
|
||||
const card = match[1];
|
||||
const name = nameRegex.exec(card)?.[1]?.trim();
|
||||
const priceStr = priceRegex.exec(card)?.[1]?.trim();
|
||||
if (name && name.length > 3) {
|
||||
const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0;
|
||||
if (price > 0 && price < 1000) {
|
||||
products.push({
|
||||
name: name.substring(0, 200),
|
||||
price,
|
||||
source,
|
||||
source_product_id: source + '_' + Date.now() + '_' + Math.random().toString(36).substr(2, 6),
|
||||
source_url: 'https://www.' + source + '.com',
|
||||
brand: '',
|
||||
category: 'parapharmacy',
|
||||
available: true,
|
||||
scraped_at: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return products.slice(0, 5).map(p => ({ json: p }));`
|
||||
}
|
||||
},
|
||||
output: [{}]
|
||||
});
|
||||
|
||||
const sendToApi = node({
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
version: 4.2,
|
||||
config: {
|
||||
name: 'Send to API',
|
||||
position: [1240, 300],
|
||||
parameters: {
|
||||
method: 'POST',
|
||||
url: 'http://parapharmacy-api:3002/api/products/bulk',
|
||||
sendBody: true,
|
||||
specifyBody: 'json',
|
||||
jsonBody: '={{ JSON.stringify({ products: $input.all().map(i => i.json) }) }}'
|
||||
}
|
||||
},
|
||||
output: [{}]
|
||||
});
|
||||
|
||||
const response = node({
|
||||
type: 'n8n-nodes-base.code',
|
||||
version: 2,
|
||||
config: {
|
||||
name: 'Response',
|
||||
position: [1440, 300],
|
||||
parameters: {
|
||||
jsCode: `return [{ json: { success: true, message: 'Scraping completed', products: $input.all().length } }];`
|
||||
}
|
||||
},
|
||||
output: [{}]
|
||||
});
|
||||
|
||||
export default workflow('scrape-fixed', 'Parapharmacy Scraper Fixed')
|
||||
.add(webhook)
|
||||
.to(parseInput)
|
||||
.to(generateTasks)
|
||||
.to(scrape)
|
||||
.to(extractProducts)
|
||||
.to(sendToApi)
|
||||
.to(response);
|
||||
Reference in New Issue
Block a user