Add exponential backoff retry to all downstream calls

New controller/retry.js wraps async calls with infinite retries, capped

at 30s with jitter. Applied to: TPB search, TMDB, Emby, Ollama chat,

Transmission RPC, and the status/status interval calls inherit it via

a Proxy around the Transmission client.

Also handles ollama package returning pre-parsed JSON objects.

Signed-off-by: William Mantly <wmantly@gmail.com>

controller/retry.js

@@ -0,0 +1,85 @@
+'use strict';
+
+// Retry an async function with exponential backoff up to a max delay.
+// No attempt limit: keeps retrying on transient failures forever.
+// Transient = network errors, 5xx, 429. 4xx client errors are NOT retried by default.
+const TRANSIENT_NETWORK_CODES = new Set([
+	'ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN',
+	'ENOTFOUND', 'EPIPE', 'ERR_SOCKET_TIMEOUT', 'ETIMEOUT',
+]);
+
+function isRetryable(error){
+	// Network / DNS / socket errors
+	if(error && error.code && TRANSIENT_NETWORK_CODES.has(error.code)) return true;
+
+	// HTTP status codes worth retrying
+	let status = Number(error && error.status) || Number(error && error.status_code);
+	if(status >= 500 && status < 600) return true;
+	if(status === 429) return true;
+	if(status === 502 || status === 503 || status === 504) return true;
+
+	// ResponseError from the ollama package carries status_code
+	if(error && error.name === 'ResponseError' && status >= 500) return true;
+
+	return false;
+}
+
+async function sleep(ms){
+	return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function withRetry(fn, options = {}){
+	let baseDelay = options.baseDelay || 1000;
+	let maxDelay = options.maxDelay || 30000;
+	let multiplier = options.multiplier || 2;
+	let jitter = options.jitter || 0.2;
+	let label = options.label || 'operation';
+	let shouldRetry = options.shouldRetry || isRetryable;
+
+	let attempt = 0;
+	let delay = baseDelay;
+
+	while(true){
+		try{
+			return await fn();
+		}catch(error){
+			attempt++;
+			if(!shouldRetry(error)){
+				console.error(`${label} attempt ${attempt} failed permanently:`, error.message);
+				throw error;
+			}
+
+			console.error(`${label} attempt ${attempt} failed, retrying in ${delay}ms:`, error.message);
+			let jittered = Math.round(delay * (1 + (Math.random() * 2 - 1) * jitter));
+			await sleep(jittered);
+			delay = Math.min(delay * multiplier, maxDelay);
+		}
+	}
+}
+
+// Convenience: retry a fetch call. On success returns the Response.
+async function fetchWithRetry(url, options = {}, retryOptions = {}){
+	return await withRetry(async () => {
+		let res = await fetch(url, options);
+		if(!res.ok){
+			let error = new Error(`UpstreamError`);
+			error.message = `Request to ${url.split('?')[0]} failed( ${res.status})`;
+			error.status = res.status;
+			throw error;
+		}
+		return res;
+	}, { label: `fetch ${url.split('?')[0]}`, ...retryOptions });
+}
+
+// Convenience: retry a fetch call and parse JSON response.
+async function fetchJSONWithRetry(url, options = {}, retryOptions = {}){
+	let res = await fetchWithRetry(url, options, retryOptions);
+	return await res.json();
+}
+
+module.exports = {
+	withRetry,
+	fetchWithRetry,
+	fetchJSONWithRetry,
+	isRetryable,
+};