Add exponential backoff retry to all downstream calls

New controller/retry.js wraps async calls with infinite retries, capped

at 30s with jitter. Applied to: TPB search, TMDB, Emby, Ollama chat,

Transmission RPC, and the status/status interval calls inherit it via

a Proxy around the Transmission client.

Also handles ollama package returning pre-parsed JSON objects.

Signed-off-by: William Mantly <wmantly@gmail.com>

models/sql/torrent.js

@@ -2,18 +2,33 @@
 
 const Transmission = require('transmission-promise');
 const conf = require('>/conf');
+const { withRetry } = require('>/controller/retry');
 
-const tr_client = new Transmission(conf.transmission)
+const tr_client = new Transmission(conf.transmission);
+
+// Wrap the raw Transmission client so every RPC call retries on transient network errors.
+const trClient = new Proxy(tr_client, {
+	get(target, prop){
+		let value = target[prop];
+		if(typeof value !== 'function') return value;
+		return async function(...args){
+			return await withRetry(
+				() => value.apply(target, args),
+				{ label: `transmission ${String(prop)}` }
+			);
+		};
+	}
+});
 
 const statusMap = [
   'STOPPED', // 0
-  'CHECK_WAIT', // 1  
+  'CHECK_WAIT', // 1
   'CHECK', // 2
-  'DOWNLOAD_WAIT', // 3 
-  'DOWNLOAD', // 4     
-  'SEED_WAIT', // 5 
-  'SEED', // 6   
-  'ISOLATED', // 7  
+  'DOWNLOAD_WAIT', // 3
+  'DOWNLOAD', // 4
+  'SEED_WAIT', // 5
+  'SEED', // 6
+  'ISOLATED', // 7
 ];
 
 module.exports = (sequelize, DataTypes, Model) => {
@@ -27,7 +42,7 @@ module.exports = (sequelize, DataTypes, Model) => {
       // define association here
     }
 
-    static trClient = tr_client;
+    static trClient = trClient;
 
     // Map a raw TPB category id( e.g. 207) to one of our buckets. TPB groups by the
     // hundreds digit; TV shows are the exception pulled out of the Video group.
@@ -57,7 +72,7 @@ module.exports = (sequelize, DataTypes, Model) => {
           'download-dir': data.isPrivate  ? `${conf.privateDownloadLocation}/${data.added_by}` : undefined,
         };
 
-        let res = await tr_client.addUrl(data.magnetLink, options);
+        let res = await trClient.addUrl(data.magnetLink, options);
 
         return await super.create({
           magnetLink: data.magnetLink,
@@ -80,7 +95,7 @@ module.exports = (sequelize, DataTypes, Model) => {
 
         if(this.percentDone === 1) return this.dataValues
 
-        let res = ( await tr_client.get(this.hashString, [
+        let res = ( await trClient.get(this.hashString, [
           "eta", "percentDone", "status", "rateDownload",
           "errorString", "hashString", 'name',
           'downloadDir',
@@ -104,7 +119,7 @@ module.exports = (sequelize, DataTypes, Model) => {
           throw e
         }
         // console.error(`Torrent ${this.hashString} getTorrentData error`, error);
-        throw error;
+        throw error
       }
     }