fix(backend): add retry logic and remove custom User-Agent for OFF API
- Add retry logic with exponential backoff (2 retries) - Remove custom User-Agent that was being blocked by OFF - Fix syntax error (missing catch block) - OFF API blocks anonymous users during high demand
This commit is contained in:
+57
-32
@@ -3,8 +3,10 @@ import redisClient from './redis-client.js';
|
||||
|
||||
const OFF_API_BASE = 'https://world.openfoodfacts.org';
|
||||
const CACHE_TTL = 3600;
|
||||
const MAX_RETRIES = 2;
|
||||
const RETRY_DELAY_MS = 1000;
|
||||
|
||||
// Open Food Facts authentication (optional, improves rate limits)
|
||||
// Open Food Facts authentication (required for reliable access)
|
||||
const OFF_USERNAME = process.env.OFF_USERNAME;
|
||||
const OFF_PASSWORD = process.env.OFF_PASSWORD;
|
||||
|
||||
@@ -18,6 +20,10 @@ function getOffAuth() {
|
||||
return null;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function transformOFFProduct(offProduct) {
|
||||
if (!offProduct || !offProduct._id || !offProduct.product_name) {
|
||||
return null;
|
||||
@@ -68,41 +74,60 @@ export async function searchBabyProducts(query) {
|
||||
|
||||
console.log(`Fetching from OFF API: ${searchTerm}`);
|
||||
const auth = getOffAuth();
|
||||
const response = await axios.get(`${OFF_API_BASE}/cgi/search.pl`, {
|
||||
params: {
|
||||
search_terms: searchTerm,
|
||||
json: true,
|
||||
fields: 'product_name,brands,image_url,nutriscore_grade,ingredients_text,nova_group,ecoscore_grade,categories_tags',
|
||||
page_size: 20,
|
||||
action: 'process',
|
||||
},
|
||||
timeout: 10000,
|
||||
validateStatus: (status) => status === 200,
|
||||
...(auth && { auth }),
|
||||
});
|
||||
|
||||
// Check if response is actually JSON (OFF sometimes returns HTML on error)
|
||||
if (typeof response.data === 'string' || !response.data.products) {
|
||||
console.warn(`[OFF] Invalid response for "${searchTerm}": API may be down`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const products = response.data.products
|
||||
.map(transformOFFProduct)
|
||||
.filter(Boolean);
|
||||
|
||||
// Only cache non-empty results to avoid caching API failures
|
||||
if (products.length > 0) {
|
||||
let lastError;
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
await redisClient.setEx(cacheKey, CACHE_TTL, JSON.stringify(products));
|
||||
console.log(`Cached ${products.length} OFF products for: ${searchTerm}`);
|
||||
} catch (cacheErr) {
|
||||
// cache write failed, still return the data
|
||||
const response = await axios.get(`${OFF_API_BASE}/cgi/search.pl`, {
|
||||
params: {
|
||||
search_terms: searchTerm,
|
||||
json: true,
|
||||
fields: 'product_name,brands,image_url,nutriscore_grade,ingredients_text,nova_group,ecoscore_grade,categories_tags',
|
||||
page_size: 20,
|
||||
action: 'process',
|
||||
},
|
||||
timeout: 10000,
|
||||
validateStatus: (status) => status === 200,
|
||||
...(auth && { auth }),
|
||||
});
|
||||
|
||||
// Check if response is actually JSON (OFF sometimes returns HTML on error)
|
||||
if (typeof response.data === 'string' || !response.data.products) {
|
||||
console.warn(`[OFF] Invalid response for "${searchTerm}" (attempt ${attempt + 1}): API may be down`);
|
||||
if (attempt < MAX_RETRIES) {
|
||||
await sleep(RETRY_DELAY_MS * (attempt + 1));
|
||||
continue;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
const products = response.data.products
|
||||
.map(transformOFFProduct)
|
||||
.filter(Boolean);
|
||||
|
||||
// Only cache non-empty results to avoid caching API failures
|
||||
if (products.length > 0) {
|
||||
try {
|
||||
await redisClient.setEx(cacheKey, CACHE_TTL, JSON.stringify(products));
|
||||
console.log(`Cached ${products.length} OFF products for: ${searchTerm}`);
|
||||
} catch (cacheErr) {
|
||||
// cache write failed, still return the data
|
||||
}
|
||||
} else {
|
||||
console.log(`[OFF] No products found for "${searchTerm}", not caching empty result`);
|
||||
}
|
||||
return products;
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
console.warn(`[OFF] Request failed for "${searchTerm}" (attempt ${attempt + 1}): ${err.message}`);
|
||||
if (attempt < MAX_RETRIES) {
|
||||
await sleep(RETRY_DELAY_MS * (attempt + 1));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(`[OFF] No products found for "${searchTerm}", not caching empty result`);
|
||||
}
|
||||
return products;
|
||||
|
||||
console.error(`[OFF] All retries failed for "${searchTerm}": ${lastError?.message}`);
|
||||
return [];
|
||||
} catch (error) {
|
||||
console.error('Error searching baby products from OFF:', error.message);
|
||||
return [];
|
||||
|
||||
Reference in New Issue
Block a user