wmantly
51e6817393
New controller/retry.js wraps async calls with infinite retries, capped at 30s with jitter. Applied to: TPB search, TMDB, Emby, Ollama chat, Transmission RPC, and the status/status interval calls inherit it via a Proxy around the Transmission client. Also handles ollama package returning pre-parsed JSON objects. Signed-off-by: William Mantly <wmantly@gmail.com>
86 lines
2.6 KiB
JavaScript
86 lines
2.6 KiB
JavaScript
'use strict';
|
|
|
|
// Retry an async function with exponential backoff up to a max delay.
|
|
// No attempt limit: keeps retrying on transient failures forever.
|
|
// Transient = network errors, 5xx, 429. 4xx client errors are NOT retried by default.
|
|
const TRANSIENT_NETWORK_CODES = new Set([
|
|
'ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN',
|
|
'ENOTFOUND', 'EPIPE', 'ERR_SOCKET_TIMEOUT', 'ETIMEOUT',
|
|
]);
|
|
|
|
function isRetryable(error){
|
|
// Network / DNS / socket errors
|
|
if(error && error.code && TRANSIENT_NETWORK_CODES.has(error.code)) return true;
|
|
|
|
// HTTP status codes worth retrying
|
|
let status = Number(error && error.status) || Number(error && error.status_code);
|
|
if(status >= 500 && status < 600) return true;
|
|
if(status === 429) return true;
|
|
if(status === 502 || status === 503 || status === 504) return true;
|
|
|
|
// ResponseError from the ollama package carries status_code
|
|
if(error && error.name === 'ResponseError' && status >= 500) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
async function sleep(ms){
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function withRetry(fn, options = {}){
|
|
let baseDelay = options.baseDelay || 1000;
|
|
let maxDelay = options.maxDelay || 30000;
|
|
let multiplier = options.multiplier || 2;
|
|
let jitter = options.jitter || 0.2;
|
|
let label = options.label || 'operation';
|
|
let shouldRetry = options.shouldRetry || isRetryable;
|
|
|
|
let attempt = 0;
|
|
let delay = baseDelay;
|
|
|
|
while(true){
|
|
try{
|
|
return await fn();
|
|
}catch(error){
|
|
attempt++;
|
|
if(!shouldRetry(error)){
|
|
console.error(`${label} attempt ${attempt} failed permanently:`, error.message);
|
|
throw error;
|
|
}
|
|
|
|
console.error(`${label} attempt ${attempt} failed, retrying in ${delay}ms:`, error.message);
|
|
let jittered = Math.round(delay * (1 + (Math.random() * 2 - 1) * jitter));
|
|
await sleep(jittered);
|
|
delay = Math.min(delay * multiplier, maxDelay);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Convenience: retry a fetch call. On success returns the Response.
|
|
async function fetchWithRetry(url, options = {}, retryOptions = {}){
|
|
return await withRetry(async () => {
|
|
let res = await fetch(url, options);
|
|
if(!res.ok){
|
|
let error = new Error(`UpstreamError`);
|
|
error.message = `Request to ${url.split('?')[0]} failed( ${res.status})`;
|
|
error.status = res.status;
|
|
throw error;
|
|
}
|
|
return res;
|
|
}, { label: `fetch ${url.split('?')[0]}`, ...retryOptions });
|
|
}
|
|
|
|
// Convenience: retry a fetch call and parse JSON response.
|
|
async function fetchJSONWithRetry(url, options = {}, retryOptions = {}){
|
|
let res = await fetchWithRetry(url, options, retryOptions);
|
|
return await res.json();
|
|
}
|
|
|
|
module.exports = {
|
|
withRetry,
|
|
fetchWithRetry,
|
|
fetchJSONWithRetry,
|
|
isRetryable,
|
|
};
|