Antoni Nuñez Romeu
79 lines
2.8 KiB
JavaScript
79 lines
2.8 KiB
JavaScript
// OpenTelemetry SDK — must be required first to instrument http, dns, etc.
|
|
require('./tracing');
|
|
|
|
const { trace } = require('@opentelemetry/api');
|
|
const puppeteer = require('puppeteer-extra');
|
|
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
|
const fs = require('fs');
|
|
|
|
const scrapePromofarma = require('./src/promofarma');
|
|
const scrapeAmazon = require('./src/amazon');
|
|
const scrapeGoogleShopping = require('./src/googleshopping');
|
|
const scrapeOpenFoodFacts = require('./src/openfoodfacts');
|
|
|
|
puppeteer.use(StealthPlugin());
|
|
|
|
// Default terms related to parapharmacy
|
|
const QUERIES = [
|
|
'crema hidratante cara',
|
|
'ibuprofeno' // fallback basic test query
|
|
];
|
|
|
|
async function run() {
|
|
const tracer = trace.getTracer('farmaclic-scraper');
|
|
return tracer.startActiveSpan('scraper.run', { attributes: { 'scraper.queries': QUERIES.length } }, async (span) => {
|
|
try {
|
|
console.log('🚀 Starting Parapharmacy Scraper...');
|
|
const browser = await puppeteer.launch({
|
|
headless: 'new',
|
|
args: ['--no-sandbox', '--disable-setuid-sandbox']
|
|
});
|
|
|
|
const allResults = {};
|
|
|
|
for (const query of QUERIES) {
|
|
const querySpan = tracer.startSpan('scraper.query', { attributes: { 'scraper.query': query } });
|
|
console.log(`\n🔍 Searching for: "${query}"`);
|
|
|
|
const promofarmaResults = await scrapePromofarma(query, browser);
|
|
console.log(` ✅ Promofarma: found ${promofarmaResults.length} items`);
|
|
|
|
const amazonResults = await scrapeAmazon(query, browser);
|
|
console.log(` ✅ Amazon: found ${amazonResults.length} items`);
|
|
|
|
const googleShoppingResults = await scrapeGoogleShopping(query, browser);
|
|
console.log(` ✅ Google Shopping: found ${googleShoppingResults.length} items`);
|
|
|
|
const ofFactsResults = await scrapeOpenFoodFacts(query, browser);
|
|
console.log(` ✅ OpenFoodFacts: found ${ofFactsResults.length} items`);
|
|
|
|
allResults[query] = {
|
|
promofarma: promofarmaResults,
|
|
amazon: amazonResults,
|
|
googleshopping: googleShoppingResults,
|
|
openfoodfacts: ofFactsResults
|
|
};
|
|
querySpan.setAttribute('scraper.results.total', promofarmaResults.length + amazonResults.length + googleShoppingResults.length + ofFactsResults.length);
|
|
querySpan.end();
|
|
}
|
|
|
|
await browser.close();
|
|
|
|
fs.writeFileSync('results.json', JSON.stringify(allResults, null, 2));
|
|
console.log('\n💾 Scraping finished. Results saved to results.json');
|
|
span.setStatus({ code: 1 }); // OK
|
|
} catch (err) {
|
|
span.recordException(err);
|
|
span.setStatus({ code: 2, message: err.message }); // ERROR
|
|
throw err;
|
|
} finally {
|
|
span.end();
|
|
}
|
|
});
|
|
}
|
|
|
|
run().catch(err => {
|
|
console.error("Critical Error during scraping:", err);
|
|
process.exit(1);
|
|
});
|