fix(backend): add retry logic and remove custom User-Agent for OFF API
- Add retry logic with exponential backoff (2 retries) - Remove custom User-Agent that was being blocked by OFF - Fix syntax error (missing catch block) - OFF API blocks anonymous users during high demand
This commit is contained in:
+57
-32
@@ -3,8 +3,10 @@ import redisClient from './redis-client.js';
|
|||||||
|
|
||||||
const OFF_API_BASE = 'https://world.openfoodfacts.org';
|
const OFF_API_BASE = 'https://world.openfoodfacts.org';
|
||||||
const CACHE_TTL = 3600;
|
const CACHE_TTL = 3600;
|
||||||
|
const MAX_RETRIES = 2;
|
||||||
|
const RETRY_DELAY_MS = 1000;
|
||||||
|
|
||||||
// Open Food Facts authentication (optional, improves rate limits)
|
// Open Food Facts authentication (required for reliable access)
|
||||||
const OFF_USERNAME = process.env.OFF_USERNAME;
|
const OFF_USERNAME = process.env.OFF_USERNAME;
|
||||||
const OFF_PASSWORD = process.env.OFF_PASSWORD;
|
const OFF_PASSWORD = process.env.OFF_PASSWORD;
|
||||||
|
|
||||||
@@ -18,6 +20,10 @@ function getOffAuth() {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function sleep(ms) {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
export function transformOFFProduct(offProduct) {
|
export function transformOFFProduct(offProduct) {
|
||||||
if (!offProduct || !offProduct._id || !offProduct.product_name) {
|
if (!offProduct || !offProduct._id || !offProduct.product_name) {
|
||||||
return null;
|
return null;
|
||||||
@@ -68,41 +74,60 @@ export async function searchBabyProducts(query) {
|
|||||||
|
|
||||||
console.log(`Fetching from OFF API: ${searchTerm}`);
|
console.log(`Fetching from OFF API: ${searchTerm}`);
|
||||||
const auth = getOffAuth();
|
const auth = getOffAuth();
|
||||||
const response = await axios.get(`${OFF_API_BASE}/cgi/search.pl`, {
|
|
||||||
params: {
|
|
||||||
search_terms: searchTerm,
|
|
||||||
json: true,
|
|
||||||
fields: 'product_name,brands,image_url,nutriscore_grade,ingredients_text,nova_group,ecoscore_grade,categories_tags',
|
|
||||||
page_size: 20,
|
|
||||||
action: 'process',
|
|
||||||
},
|
|
||||||
timeout: 10000,
|
|
||||||
validateStatus: (status) => status === 200,
|
|
||||||
...(auth && { auth }),
|
|
||||||
});
|
|
||||||
|
|
||||||
// Check if response is actually JSON (OFF sometimes returns HTML on error)
|
let lastError;
|
||||||
if (typeof response.data === 'string' || !response.data.products) {
|
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||||
console.warn(`[OFF] Invalid response for "${searchTerm}": API may be down`);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
const products = response.data.products
|
|
||||||
.map(transformOFFProduct)
|
|
||||||
.filter(Boolean);
|
|
||||||
|
|
||||||
// Only cache non-empty results to avoid caching API failures
|
|
||||||
if (products.length > 0) {
|
|
||||||
try {
|
try {
|
||||||
await redisClient.setEx(cacheKey, CACHE_TTL, JSON.stringify(products));
|
const response = await axios.get(`${OFF_API_BASE}/cgi/search.pl`, {
|
||||||
console.log(`Cached ${products.length} OFF products for: ${searchTerm}`);
|
params: {
|
||||||
} catch (cacheErr) {
|
search_terms: searchTerm,
|
||||||
// cache write failed, still return the data
|
json: true,
|
||||||
|
fields: 'product_name,brands,image_url,nutriscore_grade,ingredients_text,nova_group,ecoscore_grade,categories_tags',
|
||||||
|
page_size: 20,
|
||||||
|
action: 'process',
|
||||||
|
},
|
||||||
|
timeout: 10000,
|
||||||
|
validateStatus: (status) => status === 200,
|
||||||
|
...(auth && { auth }),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Check if response is actually JSON (OFF sometimes returns HTML on error)
|
||||||
|
if (typeof response.data === 'string' || !response.data.products) {
|
||||||
|
console.warn(`[OFF] Invalid response for "${searchTerm}" (attempt ${attempt + 1}): API may be down`);
|
||||||
|
if (attempt < MAX_RETRIES) {
|
||||||
|
await sleep(RETRY_DELAY_MS * (attempt + 1));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const products = response.data.products
|
||||||
|
.map(transformOFFProduct)
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
// Only cache non-empty results to avoid caching API failures
|
||||||
|
if (products.length > 0) {
|
||||||
|
try {
|
||||||
|
await redisClient.setEx(cacheKey, CACHE_TTL, JSON.stringify(products));
|
||||||
|
console.log(`Cached ${products.length} OFF products for: ${searchTerm}`);
|
||||||
|
} catch (cacheErr) {
|
||||||
|
// cache write failed, still return the data
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log(`[OFF] No products found for "${searchTerm}", not caching empty result`);
|
||||||
|
}
|
||||||
|
return products;
|
||||||
|
} catch (err) {
|
||||||
|
lastError = err;
|
||||||
|
console.warn(`[OFF] Request failed for "${searchTerm}" (attempt ${attempt + 1}): ${err.message}`);
|
||||||
|
if (attempt < MAX_RETRIES) {
|
||||||
|
await sleep(RETRY_DELAY_MS * (attempt + 1));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
console.log(`[OFF] No products found for "${searchTerm}", not caching empty result`);
|
|
||||||
}
|
}
|
||||||
return products;
|
|
||||||
|
console.error(`[OFF] All retries failed for "${searchTerm}": ${lastError?.message}`);
|
||||||
|
return [];
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error searching baby products from OFF:', error.message);
|
console.error('Error searching baby products from OFF:', error.message);
|
||||||
return [];
|
return [];
|
||||||
|
|||||||
Reference in New Issue
Block a user