diff --git a/apps/parapharmacy-api/Dockerfile b/apps/parapharmacy-api/Dockerfile index a29c043..87827e7 100644 --- a/apps/parapharmacy-api/Dockerfile +++ b/apps/parapharmacy-api/Dockerfile @@ -1,7 +1,32 @@ -FROM node:20-alpine +FROM node:20-slim WORKDIR /app +# Install Chrome and dependencies for Puppeteer +RUN apt-get update && apt-get install -y \ + chromium \ + fonts-liberation \ + libappindicator3-1 \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libcups2 \ + libdbus-1-3 \ + libgdk-pixbuf2.0-0 \ + libnspr4 \ + libnss3 \ + libx11-xcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxrandr2 \ + xdg-utils \ + --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +# Set Puppeteer to use installed Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium + # Copy package files COPY package*.json ./ diff --git a/apps/parapharmacy-api/package.json b/apps/parapharmacy-api/package.json index 79c42c4..14318e0 100644 --- a/apps/parapharmacy-api/package.json +++ b/apps/parapharmacy-api/package.json @@ -8,6 +8,7 @@ "start": "node --env-file-if-exists=.env src/server.js", "dev": "node --env-file-if-exists=.env --watch src/server.js", "seed": "node --env-file-if-exists=.env scripts/seed.js", + "scrape": "node --env-file-if-exists=.env scripts/scrape-puppeteer.js", "test": "NODE_OPTIONS='--experimental-vm-modules' npx jest --ci --forceExit --forceExitTimeout=30000" }, "dependencies": { @@ -16,6 +17,7 @@ "express-rate-limit": "^8.5.2", "mongoose": "^8.8.0", "morgan": "^1.10.0", + "puppeteer": "^22.0.0", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "axios": "^1.6.0" diff --git a/apps/parapharmacy-api/src/routes/scraper.js b/apps/parapharmacy-api/src/routes/scraper.js new file mode 100644 index 0000000..6145ed6 --- /dev/null +++ b/apps/parapharmacy-api/src/routes/scraper.js @@ -0,0 +1,26 @@ +import { Router } from 'express'; +import { scrapeAll } from '../scraper.js'; + +const router = Router(); + +// Trigger scraping +router.post('/scrape', async (req, res) => { + try { + const { queries = ['crema hidratante'], sources = ['promofarma'] } = req.body; + + console.log('[Scraper] Starting scrape...'); + console.log(`[Scraper] Queries: ${queries.join(', ')}`); + console.log(`[Scraper] Sources: ${sources.join(', ')}`); + + const result = await scrapeAll(queries, sources); + + console.log(`[Scraper] Completed. Total: ${result.total} products`); + + res.json(result); + } catch (error) { + console.error('[Scraper] Error:', error.message); + res.status(500).json({ error: 'Scraping failed', message: error.message }); + } +}); + +export default router; diff --git a/apps/parapharmacy-api/src/scraper.js b/apps/parapharmacy-api/src/scraper.js new file mode 100644 index 0000000..ca09f55 --- /dev/null +++ b/apps/parapharmacy-api/src/scraper.js @@ -0,0 +1,189 @@ +import puppeteer from 'puppeteer'; +import axios from 'axios'; + +const API_URL = process.env.PARAPHARMACY_API_URL || 'http://localhost:3002'; + +const SOURCES = { + promofarma: { + name: 'Promofarma', + searchUrl: (q) => `https://www.promofarma.com/es/search?q=${encodeURIComponent(q)}`, + selectors: { + product: 'article[data-name]', + name: 'article[data-name]', + price: 'article[data-pvp]', + link: 'a', + image: 'img' + } + }, + docmorris: { + name: 'DocMorris', + searchUrl: (q) => `https://www.docmorris.es/search?query=${encodeURIComponent(q)}`, + selectors: { + product: '[class*="product-card"], [class*="product-item"]', + name: '[class*="product-name"], h3', + price: '[class*="price"]', + link: 'a', + image: 'img' + } + }, + primor: { + name: 'Primor', + searchUrl: (q) => `https://www.primor.eu/search?s=${encodeURIComponent(q)}`, + selectors: { + product: '.product-item, .product-miniature', + name: '.product-title, h3', + price: '.price', + link: 'a.product-title', + image: 'img' + } + } +}; + +async function scrapeSource(browser, source, query) { + const config = SOURCES[source]; + if (!config) return []; + + const page = await browser.newPage(); + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'); + + try { + console.log(` šŸ” Scraping ${config.name}: ${query}`); + const url = config.searchUrl(query); + console.log(` šŸ“ URL: ${url}`); + + await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); + + // Wait for products to load + await page.waitForSelector(config.selectors.product, { timeout: 15000 }).catch(() => {}); + + // Get page title for debugging + const title = await page.title(); + console.log(` šŸ“„ Page title: ${title}`); + + // Get page content for debugging + const content = await page.content(); + console.log(` šŸ“„ Page length: ${content.length} chars`); + + // Try to find product elements with various selectors + const productInfo = await page.evaluate(() => { + // Try multiple selector strategies + const selectors = [ + '[data-testid*="product"]', + '[class*="ProductCard"]', + '[class*="product-card"]', + 'a[href*="/p/"]', + '[class*="Product"]', + 'article', + '[role="listitem"]' + ]; + + for (const sel of selectors) { + const els = document.querySelectorAll(sel); + if (els.length > 0) { + return { + selector: sel, + count: els.length, + samples: Array.from(els).slice(0, 3).map(el => ({ + tag: el.tagName, + class: el.className?.substring(0, 100), + html: el.outerHTML?.substring(0, 300) + })) + }; + } + } + + // Fallback: look for any element with price-like content + const allText = document.body.innerText; + const priceMatch = allText.match(/\d+[.,]\d{2}\s*€/g); + + return { + selector: 'none found', + priceMatches: priceMatch?.slice(0, 5) || [], + bodyTextSample: document.body.innerText.substring(0, 500) + }; + }); + console.log(` šŸ” Product info:`, JSON.stringify(productInfo, null, 2)); + + const products = await page.evaluate((selectors) => { + const results = []; + const cards = document.querySelectorAll(selectors.product); + + cards.forEach(card => { + // Try data attributes first (Promofarma style) + const name = card.getAttribute('data-name') || card.querySelector(selectors.name)?.innerText?.trim(); + const priceStr = card.getAttribute('data-pvp') || card.querySelector(selectors.price)?.innerText?.trim(); + const linkEl = card.querySelector(selectors.link); + const imgEl = card.querySelector(selectors.image); + + if (name && priceStr) { + const price = parseFloat(priceStr.replace(/[^0-9.,]/g, '').replace(',', '.')) || 0; + + if (name && price > 0 && price < 1000) { + results.push({ + name: name.substring(0, 200), + price, + source_url: linkEl?.href || '', + image_url: imgEl?.src || null + }); + } + } + }); + + return results; + }, config.selectors); + + console.log(` šŸ“¦ Found ${products.length} products`); + return products.slice(0, 10).map(p => ({ + ...p, + source, + source_product_id: `${source}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`, + brand: '', + category: 'parapharmacy', + available: true, + scraped_at: new Date().toISOString() + })); + + } catch (error) { + console.error(` āŒ Error scraping ${config.name}: ${error.message}`); + return []; + } finally { + await page.close(); + } +} + +export async function scrapeAll(queries = ['crema hidratante'], sources = ['promofarma']) { + console.log('šŸš€ Starting scraper...'); + + const browser = await puppeteer.launch({ + headless: 'new', + executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined, + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] + }); + + let totalProducts = 0; + + try { + for (const query of queries) { + console.log(`\nšŸ“‹ Query: "${query}"`); + + for (const source of sources) { + const products = await scrapeSource(browser, source, query); + console.log(` āœ… Found ${products.length} products`); + + if (products.length > 0) { + try { + await axios.post(`${API_URL}/api/products/bulk`, { products }); + totalProducts += products.length; + } catch (error) { + console.error(` āŒ Error sending to API: ${error.message}`); + } + } + } + } + } finally { + await browser.close(); + } + + console.log(`\nšŸ“Š Scraping completed. Total: ${totalProducts} products`); + return { success: true, total: totalProducts }; +} diff --git a/apps/parapharmacy-api/src/server.js b/apps/parapharmacy-api/src/server.js index 92664d4..3a705c7 100644 --- a/apps/parapharmacy-api/src/server.js +++ b/apps/parapharmacy-api/src/server.js @@ -7,6 +7,7 @@ import swaggerJsdoc from 'swagger-jsdoc'; import swaggerUi from 'swagger-ui-express'; import { config } from './config.js'; import productsRouter from './routes/products.js'; +import scraperRouter from './routes/scraper.js'; const app = express(); @@ -69,6 +70,7 @@ app.use('/api/docs', swaggerUi.serve, swaggerUi.setup(swaggerSpec, { // Routes app.use('/api/products', productsRouter); +app.use('/api', scraperRouter); // Health check app.get('/api/health', (req, res) => { diff --git a/n8n/workflow-sdk.js b/n8n/workflow-sdk.js new file mode 100644 index 0000000..cbe78b4 --- /dev/null +++ b/n8n/workflow-sdk.js @@ -0,0 +1,145 @@ +import { workflow, node, trigger } from '@n8n/workflow-sdk'; + +const webhook = trigger({ + type: 'n8n-nodes-base.webhook', + version: 2, + config: { + name: 'Webhook', + position: [240, 300], + parameters: { + httpMethod: 'POST', + path: 'scrape-fixed', + responseMode: 'lastNode' + } + }, + output: [{}] +}); + +const parseInput = node({ + type: 'n8n-nodes-base.code', + version: 2, + config: { + name: 'Parse Input', + position: [440, 300], + parameters: { + jsCode: `const body = $input.first().json.body || {}; +return [{ json: { queries: body.queries || 'crema hidratante', sources: body.sources || ['promofarma'] } }];` + } + }, + output: [{}] +}); + +const generateTasks = node({ + type: 'n8n-nodes-base.code', + version: 2, + config: { + name: 'Generate Tasks', + position: [640, 300], + parameters: { + jsCode: `const input = $input.first().json; +const queries = input.queries.split(',').map(q => q.trim()); +const tasks = []; +for (const q of queries) { + tasks.push({ query: q, source: 'promofarma', url: 'https://www.promofarma.com/es/search?q=' + encodeURIComponent(q) }); +} +return tasks.map(t => ({ json: t }));` + } + }, + output: [{}] +}); + +const scrape = node({ + type: 'n8n-nodes-base.httpRequest', + version: 4.2, + config: { + name: 'Scrape', + position: [840, 300], + parameters: { + method: 'GET', + url: '={{ $json.url }}', + options: { timeout: 30000 } + } + }, + output: [{}] +}); + +const extractProducts = node({ + type: 'n8n-nodes-base.code', + version: 2, + config: { + name: 'Extract Products', + position: [1040, 300], + parameters: { + jsCode: `const data = $input.first().json; +const html = data.data || ''; +const source = data.source; +const products = []; +const cardRegex = /<(?:div|article)[^>]*class="[^"]*product[^"]*"[^>]*>([\\s\\S]*?)<\\/(?:div|article)>/gi; +const nameRegex = /]*>([^<]+)<\\/h[23]>/i; +const priceRegex = /class="[^"]*price[^"]*"[^>]*>([^<]*\\d+[.,]\\d+[^<]*)<\\/[^>]+>/i; +let match; +while ((match = cardRegex.exec(html)) !== null) { + const card = match[1]; + const name = nameRegex.exec(card)?.[1]?.trim(); + const priceStr = priceRegex.exec(card)?.[1]?.trim(); + if (name && name.length > 3) { + const price = parseFloat(priceStr?.replace(/[^\\d.,]/g, '').replace(',', '.')) || 0; + if (price > 0 && price < 1000) { + products.push({ + name: name.substring(0, 200), + price, + source, + source_product_id: source + '_' + Date.now() + '_' + Math.random().toString(36).substr(2, 6), + source_url: 'https://www.' + source + '.com', + brand: '', + category: 'parapharmacy', + available: true, + scraped_at: new Date().toISOString() + }); + } + } +} +return products.slice(0, 5).map(p => ({ json: p }));` + } + }, + output: [{}] +}); + +const sendToApi = node({ + type: 'n8n-nodes-base.httpRequest', + version: 4.2, + config: { + name: 'Send to API', + position: [1240, 300], + parameters: { + method: 'POST', + url: 'http://parapharmacy-api:3002/api/products/bulk', + sendBody: true, + specifyBody: 'json', + jsonBody: '={{ JSON.stringify({ products: $input.all().map(i => i.json) }) }}' + } + }, + output: [{}] +}); + +const response = node({ + type: 'n8n-nodes-base.code', + version: 2, + config: { + name: 'Response', + position: [1440, 300], + parameters: { + jsCode: `return [{ json: { success: true, message: 'Scraping completed', products: $input.all().length } }];` + } + }, + output: [{}] +}); + +export default workflow('scrape-fixed', 'Parapharmacy Scraper Fixed') + .add(webhook) + .to(parseInput) + .to(generateTasks) + .to(scrape) + .to(extractProducts) + .to(sendToApi) + .to(response);