feat: working Puppeteer scraper for Promofarma
- Added Puppeteer with Chrome in Docker - Scraper extracts products using data attributes - Added /api/scrape endpoint - Tested: 10 products scraped from Promofarma The scraper now works with Promofarma's HTML structure which uses data-name, data-pvp attributes on article elements.
This commit is contained in:
Antoni Nuñez Romeu
@@ -1,7 +1,32 @@
|
||||
FROM node:20-alpine
|
||||
FROM node:20-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install Chrome and dependencies for Puppeteer
|
||||
RUN apt-get update && apt-get install -y \
|
||||
chromium \
|
||||
fonts-liberation \
|
||||
libappindicator3-1 \
|
||||
libasound2 \
|
||||
libatk-bridge2.0-0 \
|
||||
libatk1.0-0 \
|
||||
libcups2 \
|
||||
libdbus-1-3 \
|
||||
libgdk-pixbuf2.0-0 \
|
||||
libnspr4 \
|
||||
libnss3 \
|
||||
libx11-xcb1 \
|
||||
libxcomposite1 \
|
||||
libxdamage1 \
|
||||
libxrandr2 \
|
||||
xdg-utils \
|
||||
--no-install-recommends && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set Puppeteer to use installed Chrome
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
|
||||
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
|
||||
|
||||
# Copy package files
|
||||
COPY package*.json ./
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"start": "node --env-file-if-exists=.env src/server.js",
|
||||
"dev": "node --env-file-if-exists=.env --watch src/server.js",
|
||||
"seed": "node --env-file-if-exists=.env scripts/seed.js",
|
||||
"scrape": "node --env-file-if-exists=.env scripts/scrape-puppeteer.js",
|
||||
"test": "NODE_OPTIONS='--experimental-vm-modules' npx jest --ci --forceExit --forceExitTimeout=30000"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -16,6 +17,7 @@
|
||||
"express-rate-limit": "^8.5.2",
|
||||
"mongoose": "^8.8.0",
|
||||
"morgan": "^1.10.0",
|
||||
"puppeteer": "^22.0.0",
|
||||
"swagger-jsdoc": "^6.2.8",
|
||||
"swagger-ui-express": "^5.0.0",
|
||||
"axios": "^1.6.0"
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { Router } from 'express';
|
||||
import { scrapeAll } from '../scraper.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// Trigger scraping
|
||||
router.post('/scrape', async (req, res) => {
|
||||
try {
|
||||
const { queries = ['crema hidratante'], sources = ['promofarma'] } = req.body;
|
||||
|
||||
console.log('[Scraper] Starting scrape...');
|
||||
console.log(`[Scraper] Queries: ${queries.join(', ')}`);
|
||||
console.log(`[Scraper] Sources: ${sources.join(', ')}`);
|
||||
|
||||
const result = await scrapeAll(queries, sources);
|
||||
|
||||
console.log(`[Scraper] Completed. Total: ${result.total} products`);
|
||||
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error('[Scraper] Error:', error.message);
|
||||
res.status(500).json({ error: 'Scraping failed', message: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,189 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
import axios from 'axios';
|
||||
|
||||
const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002';
|
||||
|
||||
const SOURCES = {
|
||||
promofarma: {
|
||||
name: 'Promofarma',
|
||||
searchUrl: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`,
|
||||
selectors: {
|
||||
product: 'article[data-name]',
|
||||
name: 'article[data-name]',
|
||||
price: 'article[data-pvp]',
|
||||
link: 'a',
|
||||
image: 'img'
|
||||
}
|
||||
},
|
||||
docmorris: {
|
||||
name: 'DocMorris',
|
||||
searchUrl: (q) => `https://www.docmorris.es/search?query=${encodeURIComponent(q)}`,
|
||||
selectors: {
|
||||
product: '[class*="product-card"], [class*="product-item"]',
|
||||
name: '[class*="product-name"], h3',
|
||||
price: '[class*="price"]',
|
||||
link: 'a',
|
||||
image: 'img'
|
||||
}
|
||||
},
|
||||
primor: {
|
||||
name: 'Primor',
|
||||
searchUrl: (q) => `https://www.primor.eu/search?s=${encodeURIComponent(q)}`,
|
||||
selectors: {
|
||||
product: '.product-item, .product-miniature',
|
||||
name: '.product-title, h3',
|
||||
price: '.price',
|
||||
link: 'a.product-title',
|
||||
image: 'img'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
async function scrapeSource(browser, source, query) {
|
||||
const config = SOURCES[source];
|
||||
if (!config) return [];
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
|
||||
|
||||
try {
|
||||
console.log(` 🔍 Scraping ${config.name}: ${query}`);
|
||||
const url = config.searchUrl(query);
|
||||
console.log(` 📍 URL: ${url}`);
|
||||
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 });
|
||||
|
||||
// Wait for products to load
|
||||
await page.waitForSelector(config.selectors.product, { timeout: 15000 }).catch(() => {});
|
||||
|
||||
// Get page title for debugging
|
||||
const title = await page.title();
|
||||
console.log(` 📄 Page title: ${title}`);
|
||||
|
||||
// Get page content for debugging
|
||||
const content = await page.content();
|
||||
console.log(` 📄 Page length: ${content.length} chars`);
|
||||
|
||||
// Try to find product elements with various selectors
|
||||
const productInfo = await page.evaluate(() => {
|
||||
// Try multiple selector strategies
|
||||
const selectors = [
|
||||
'[data-testid*="product"]',
|
||||
'[class*="ProductCard"]',
|
||||
'[class*="product-card"]',
|
||||
'a[href*="/p/"]',
|
||||
'[class*="Product"]',
|
||||
'article',
|
||||
'[role="listitem"]'
|
||||
];
|
||||
|
||||
for (const sel of selectors) {
|
||||
const els = document.querySelectorAll(sel);
|
||||
if (els.length > 0) {
|
||||
return {
|
||||
selector: sel,
|
||||
count: els.length,
|
||||
samples: Array.from(els).slice(0, 3).map(el => ({
|
||||
tag: el.tagName,
|
||||
class: el.className?.substring(0, 100),
|
||||
html: el.outerHTML?.substring(0, 300)
|
||||
}))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: look for any element with price-like content
|
||||
const allText = document.body.innerText;
|
||||
const priceMatch = allText.match(/\d+[.,]\d{2}\s*€/g);
|
||||
|
||||
return {
|
||||
selector: 'none found',
|
||||
priceMatches: priceMatch?.slice(0, 5) || [],
|
||||
bodyTextSample: document.body.innerText.substring(0, 500)
|
||||
};
|
||||
});
|
||||
console.log(` 🔍 Product info:`, JSON.stringify(productInfo, null, 2));
|
||||
|
||||
const products = await page.evaluate((selectors) => {
|
||||
const results = [];
|
||||
const cards = document.querySelectorAll(selectors.product);
|
||||
|
||||
cards.forEach(card => {
|
||||
// Try data attributes first (Promofarma style)
|
||||
const name = card.getAttribute('data-name') || card.querySelector(selectors.name)?.innerText?.trim();
|
||||
const priceStr = card.getAttribute('data-pvp') || card.querySelector(selectors.price)?.innerText?.trim();
|
||||
const linkEl = card.querySelector(selectors.link);
|
||||
const imgEl = card.querySelector(selectors.image);
|
||||
|
||||
if (name && priceStr) {
|
||||
const price = parseFloat(priceStr.replace(/[^0-9.,]/g, '').replace(',', '.')) || 0;
|
||||
|
||||
if (name && price > 0 && price < 1000) {
|
||||
results.push({
|
||||
name: name.substring(0, 200),
|
||||
price,
|
||||
source_url: linkEl?.href || '',
|
||||
image_url: imgEl?.src || null
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
}, config.selectors);
|
||||
|
||||
console.log(` 📦 Found ${products.length} products`);
|
||||
return products.slice(0, 10).map(p => ({
|
||||
...p,
|
||||
source,
|
||||
source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`,
|
||||
brand: '',
|
||||
category: 'parapharmacy',
|
||||
available: true,
|
||||
scraped_at: new Date().toISOString()
|
||||
}));
|
||||
|
||||
} catch (error) {
|
||||
console.error(` ❌ Error scraping ${config.name}: ${error.message}`);
|
||||
return [];
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function scrapeAll(queries = ['crema hidratante'], sources = ['promofarma']) {
|
||||
console.log('🚀 Starting scraper...');
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
|
||||
});
|
||||
|
||||
let totalProducts = 0;
|
||||
|
||||
try {
|
||||
for (const query of queries) {
|
||||
console.log(`\n📋 Query: "${query}"`);
|
||||
|
||||
for (const source of sources) {
|
||||
const products = await scrapeSource(browser, source, query);
|
||||
console.log(` ✅ Found ${products.length} products`);
|
||||
|
||||
if (products.length > 0) {
|
||||
try {
|
||||
await axios.post(`${API_URL}/api/products/bulk`, { products });
|
||||
totalProducts += products.length;
|
||||
} catch (error) {
|
||||
console.error(` ❌ Error sending to API: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log(`\n📊 Scraping completed. Total: ${totalProducts} products`);
|
||||
return { success: true, total: totalProducts };
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import swaggerJsdoc from 'swagger-jsdoc';
|
||||
import swaggerUi from 'swagger-ui-express';
|
||||
import { config } from './config.js';
|
||||
import productsRouter from './routes/products.js';
|
||||
import scraperRouter from './routes/scraper.js';
|
||||
|
||||
const app = express();
|
||||
|
||||
@@ -69,6 +70,7 @@ app.use('/api/docs', swaggerUi.serve, swaggerUi.setup(swaggerSpec, {
|
||||
|
||||
// Routes
|
||||
app.use('/api/products', productsRouter);
|
||||
app.use('/api', scraperRouter);
|
||||
|
||||
// Health check
|
||||
app.get('/api/health', (req, res) => {
|
||||
|
||||
Reference in New Issue
Block a user