Compare commits
8 Commits
llm
...
debug-ollama-403
| Author | SHA1 | Date | |
|---|---|---|---|
| 8abc2a65e1 | |||
| a66982a00d | |||
| 51e6817393 | |||
| 803a6db033 | |||
| ce0e0bdd4f | |||
| bd9f058f26 | |||
| 47a930eddb | |||
| 40a39b2d14 |
+1
-1
@@ -28,7 +28,7 @@ module.exports = {
|
|||||||
apiKey: '__IN SRECREST FILE__',
|
apiKey: '__IN SRECREST FILE__',
|
||||||
},
|
},
|
||||||
ollama: {
|
ollama: {
|
||||||
url: 'https://ollama.com',
|
url: 'http://192.168.1.148:11434',
|
||||||
apiKey: '__IN SRECREST FILE__',
|
apiKey: '__IN SRECREST FILE__',
|
||||||
model: 'gemma4:31b-cloud',
|
model: 'gemma4:31b-cloud',
|
||||||
},
|
},
|
||||||
|
|||||||
+3
-8
@@ -1,18 +1,13 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const conf = require('>/conf');
|
const conf = require('>/conf');
|
||||||
|
const { fetchWithRetry } = require('./retry');
|
||||||
|
|
||||||
// Ask Emby to (re)scan its libraries so a newly filed item gets indexed.
|
// Ask Emby to (re)scan its libraries so a newly filed item gets indexed.
|
||||||
async function refresh(){
|
async function refresh(){
|
||||||
let res = await fetch(`${conf.emby.url}/Library/Refresh?api_key=${conf.emby.apiKey}`, {
|
let res = await fetchWithRetry(`${conf.emby.url}/Library/Refresh?api_key=${conf.emby.apiKey}`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
});
|
}, { label: 'emby refresh' });
|
||||||
if(!res.ok){
|
|
||||||
let error = new Error('EmbyError');
|
|
||||||
error.message = `Emby library refresh failed( ${res.status})`;
|
|
||||||
error.status = 502;
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+12
-11
@@ -1,6 +1,7 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const conf = require('>/conf');
|
const conf = require('>/conf');
|
||||||
|
const { fetchJSONWithRetry } = require('./retry');
|
||||||
|
|
||||||
// Resolution rank so we can compare "what you own" against a candidate release.
|
// Resolution rank so we can compare "what you own" against a candidate release.
|
||||||
function rankFromWidth(w){
|
function rankFromWidth(w){
|
||||||
@@ -25,9 +26,8 @@ function labelFromRank(rank){ return ['', '480p', '720p', '1080p', '4k'][rank] |
|
|||||||
|
|
||||||
async function embyGet(pathAndQuery){
|
async function embyGet(pathAndQuery){
|
||||||
let sep = pathAndQuery.includes('?') ? '&' : '?';
|
let sep = pathAndQuery.includes('?') ? '&' : '?';
|
||||||
let res = await fetch(`${conf.emby.url}${pathAndQuery}${sep}api_key=${conf.emby.apiKey}`);
|
let res = await fetchJSONWithRetry(`${conf.emby.url}${pathAndQuery}${sep}api_key=${conf.emby.apiKey}`, {}, { label: `emby ${pathAndQuery.split('?')[0]}` });
|
||||||
if(!res.ok) throw new Error(`Emby ${res.status}`);
|
return res;
|
||||||
return await res.json();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Is this movie already in the library, and at what best quality? Never throws — an
|
// Is this movie already in the library, and at what best quality? Never throws — an
|
||||||
@@ -44,25 +44,26 @@ async function movieInLibrary(tmdbId){
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Which episodes of a series are already present, grouped by season( season -> count).
|
// Which episodes of a series are already present, grouped by season.
|
||||||
|
// Returns { owned, seriesId, have: { season: Set(episode) }, counts: { season: n } }.
|
||||||
async function seriesInventory(tmdbId){
|
async function seriesInventory(tmdbId){
|
||||||
try{
|
try{
|
||||||
let s = await embyGet(`/Items?Recursive=true&IncludeItemTypes=Series&AnyProviderIdEquals=tmdb.${tmdbId}&Fields=ProviderIds`);
|
let s = await embyGet(`/Items?Recursive=true&IncludeItemTypes=Series&AnyProviderIdEquals=tmdb.${tmdbId}&Fields=ProviderIds`);
|
||||||
let series = (s.Items || [])[0];
|
let series = (s.Items || [])[0];
|
||||||
if(!series) return { owned: false, have: {} };
|
if(!series) return { owned: false, have: {}, counts: {} };
|
||||||
|
|
||||||
let ep = await embyGet(`/Shows/${series.Id}/Episodes`);
|
let ep = await embyGet(`/Shows/${series.Id}/Episodes`);
|
||||||
let sets = {};
|
let have = {};
|
||||||
for(let e of ep.Items || []){
|
for(let e of ep.Items || []){
|
||||||
let sn = e.ParentIndexNumber, en = e.IndexNumber;
|
let sn = e.ParentIndexNumber, en = e.IndexNumber;
|
||||||
if(sn == null || en == null || sn === 0) continue; // skip specials
|
if(sn == null || en == null || sn === 0) continue; // skip specials
|
||||||
(sets[sn] = sets[sn] || new Set()).add(en);
|
(have[sn] = have[sn] || new Set()).add(en);
|
||||||
}
|
}
|
||||||
let have = {};
|
let counts = {};
|
||||||
for(let sn of Object.keys(sets)) have[sn] = sets[sn].size;
|
for(let sn of Object.keys(have)) counts[sn] = have[sn].size;
|
||||||
return { owned: true, seriesId: series.Id, have };
|
return { owned: true, seriesId: series.Id, have, counts };
|
||||||
}catch(error){
|
}catch(error){
|
||||||
return { owned: false, have: {}, unknown: true };
|
return { owned: false, have: {}, counts: {}, unknown: true };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+27
-19
@@ -1,25 +1,38 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
|
const { Ollama } = require('ollama');
|
||||||
const conf = require('>/conf');
|
const conf = require('>/conf');
|
||||||
|
|
||||||
|
// Shared Ollama client. The npm package adds its own Accept / Content-Type / User-Agent
|
||||||
|
// headers and handles the Ollama chat protocol, which avoids the CDN-level 403s Node's
|
||||||
|
// bare fetch gets against ollama.com.
|
||||||
|
const client = new Ollama({
|
||||||
|
host: conf.ollama.url,
|
||||||
|
headers: {
|
||||||
|
'Authorization': `Bearer ${conf.ollama.apiKey}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
// Pull the first JSON object out of a model response( tolerates ```json fences / prose).
|
// Pull the first JSON object out of a model response( tolerates ```json fences / prose).
|
||||||
function extractJSON(content){
|
function extractJSON(content){
|
||||||
let text = String(content).replace(/```json/gi, '').replace(/```/g, '').trim();
|
let text = String(content).replace(/```json/gi, '').replace(/```/g, '').trim();
|
||||||
let start = text.indexOf('{');
|
let start = text.indexOf('{');
|
||||||
let end = text.lastIndexOf('}');
|
let end = text.lastIndexOf('}');
|
||||||
if(start === -1 || end === -1) throw new Error('No JSON in model response');
|
if(start === -1 || end === -1){
|
||||||
|
console.error('extractJSON: no JSON object found in:', content);
|
||||||
|
throw new Error('No JSON in model response');
|
||||||
|
}
|
||||||
|
try{
|
||||||
return JSON.parse(text.slice(start, end + 1));
|
return JSON.parse(text.slice(start, end + 1));
|
||||||
|
}catch(error){
|
||||||
|
console.error('extractJSON: parse failed. extracted:', text.slice(start, end + 1));
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// One-shot JSON chat against the Ollama cloud model.
|
// One-shot JSON chat against the configured Ollama model.
|
||||||
async function chatJSON(system, user){
|
async function chatJSON(system, user){
|
||||||
let res = await fetch(`${conf.ollama.url}/api/chat`, {
|
let res = await client.chat({
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Authorization': `Bearer ${conf.ollama.apiKey}`,
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
model: conf.ollama.model,
|
model: conf.ollama.model,
|
||||||
stream: false,
|
stream: false,
|
||||||
format: 'json',
|
format: 'json',
|
||||||
@@ -27,16 +40,11 @@ async function chatJSON(system, user){
|
|||||||
{ role: 'system', content: system },
|
{ role: 'system', content: system },
|
||||||
{ role: 'user', content: user },
|
{ role: 'user', content: user },
|
||||||
],
|
],
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
if(!res.ok){
|
// The ollama package may return message.content as a JSON string when format:'json' is used.
|
||||||
let error = new Error('OllamaError');
|
let content = res.message && res.message.content;
|
||||||
error.message = `Ollama chat failed( ${res.status})`;
|
if(typeof content === 'string') return extractJSON(content);
|
||||||
error.status = 502;
|
return content;
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
let data = await res.json();
|
|
||||||
return extractJSON(data.message && data.message.content);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { chatJSON, extractJSON };
|
module.exports = { client, chatJSON, extractJSON };
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ const ollama = require('>/controller/ollama');
|
|||||||
const tmdb = require('>/controller/tmdb');
|
const tmdb = require('>/controller/tmdb');
|
||||||
const emby = require('>/controller/emby');
|
const emby = require('>/controller/emby');
|
||||||
const ps = require('>/controller/pubsub');
|
const ps = require('>/controller/pubsub');
|
||||||
|
const { withRetry } = require('>/controller/retry');
|
||||||
|
|
||||||
const VIDEO_EXT = new Set(['.mkv', '.mp4', '.avi', '.m4v', '.ts', '.wmv', '.mov']);
|
const VIDEO_EXT = new Set(['.mkv', '.mp4', '.avi', '.m4v', '.ts', '.wmv', '.mov']);
|
||||||
const SUB_EXT = new Set(['.srt', '.ass', '.ssa', '.sub', '.vtt', '.idx']);
|
const SUB_EXT = new Set(['.srt', '.ass', '.ssa', '.sub', '.vtt', '.idx']);
|
||||||
@@ -57,7 +58,10 @@ function buildExtractPrompt(today){
|
|||||||
async function llmExtract({ name, category, files }){
|
async function llmExtract({ name, category, files }){
|
||||||
let today = new Date().toISOString().slice(0, 10);
|
let today = new Date().toISOString().slice(0, 10);
|
||||||
let user = JSON.stringify({ name, type: category, files });
|
let user = JSON.stringify({ name, type: category, files });
|
||||||
return await ollama.chatJSON(buildExtractPrompt(today), user);
|
return await withRetry(
|
||||||
|
() => ollama.chatJSON(buildExtractPrompt(today), user),
|
||||||
|
{ label: 'ollama llmExtract' }
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse + canonicalize. Returns a metadata object, or one carrying `organizeError`
|
// Parse + canonicalize. Returns a metadata object, or one carrying `organizeError`
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ async function tick(){
|
|||||||
}
|
}
|
||||||
}catch(error){
|
}catch(error){
|
||||||
// Transmission down / transient — just try again next tick.
|
// Transmission down / transient — just try again next tick.
|
||||||
|
console.error('organizeWatcher tick failed:', error.name, error.message);
|
||||||
|
if(error.stack) console.error(error.stack);
|
||||||
}finally{
|
}finally{
|
||||||
lock = false;
|
lock = false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
// Retry an async function with exponential backoff up to a max delay.
|
||||||
|
// No attempt limit: keeps retrying on transient failures forever.
|
||||||
|
// Transient = network errors, 5xx, 429. 4xx client errors are NOT retried by default.
|
||||||
|
const TRANSIENT_NETWORK_CODES = new Set([
|
||||||
|
'ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN',
|
||||||
|
'ENOTFOUND', 'EPIPE', 'ERR_SOCKET_TIMEOUT', 'ETIMEOUT',
|
||||||
|
]);
|
||||||
|
|
||||||
|
function isRetryable(error){
|
||||||
|
// Network / DNS / socket errors
|
||||||
|
if(error && error.code && TRANSIENT_NETWORK_CODES.has(error.code)) return true;
|
||||||
|
|
||||||
|
// HTTP status codes worth retrying
|
||||||
|
let status = Number(error && error.status) || Number(error && error.status_code);
|
||||||
|
if(status >= 500 && status < 600) return true;
|
||||||
|
if(status === 429) return true;
|
||||||
|
if(status === 502 || status === 503 || status === 504) return true;
|
||||||
|
|
||||||
|
// ResponseError from the ollama package carries status_code
|
||||||
|
if(error && error.name === 'ResponseError' && status >= 500) return true;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sleep(ms){
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function withRetry(fn, options = {}){
|
||||||
|
let baseDelay = options.baseDelay || 1000;
|
||||||
|
let maxDelay = options.maxDelay || 30000;
|
||||||
|
let multiplier = options.multiplier || 2;
|
||||||
|
let jitter = options.jitter || 0.2;
|
||||||
|
let label = options.label || 'operation';
|
||||||
|
let shouldRetry = options.shouldRetry || isRetryable;
|
||||||
|
|
||||||
|
let attempt = 0;
|
||||||
|
let delay = baseDelay;
|
||||||
|
|
||||||
|
while(true){
|
||||||
|
try{
|
||||||
|
return await fn();
|
||||||
|
}catch(error){
|
||||||
|
attempt++;
|
||||||
|
if(!shouldRetry(error)){
|
||||||
|
console.error(`${label} attempt ${attempt} failed permanently:`, error.message);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error(`${label} attempt ${attempt} failed, retrying in ${delay}ms:`, error.message);
|
||||||
|
let jittered = Math.round(delay * (1 + (Math.random() * 2 - 1) * jitter));
|
||||||
|
await sleep(jittered);
|
||||||
|
delay = Math.min(delay * multiplier, maxDelay);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convenience: retry a fetch call. On success returns the Response.
|
||||||
|
async function fetchWithRetry(url, options = {}, retryOptions = {}){
|
||||||
|
return await withRetry(async () => {
|
||||||
|
let res = await fetch(url, options);
|
||||||
|
if(!res.ok){
|
||||||
|
let error = new Error(`UpstreamError`);
|
||||||
|
error.message = `Request to ${url.split('?')[0]} failed( ${res.status})`;
|
||||||
|
error.status = res.status;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}, { label: `fetch ${url.split('?')[0]}`, ...retryOptions });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convenience: retry a fetch call and parse JSON response.
|
||||||
|
async function fetchJSONWithRetry(url, options = {}, retryOptions = {}){
|
||||||
|
let res = await fetchWithRetry(url, options, retryOptions);
|
||||||
|
return await res.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
withRetry,
|
||||||
|
fetchWithRetry,
|
||||||
|
fetchJSONWithRetry,
|
||||||
|
isRetryable,
|
||||||
|
};
|
||||||
+80
-50
@@ -5,6 +5,7 @@ const tmdb = require('>/controller/tmdb');
|
|||||||
const { tmdbSearch, tmdbDetails } = tmdb;
|
const { tmdbSearch, tmdbDetails } = tmdb;
|
||||||
const library = require('>/controller/library');
|
const library = require('>/controller/library');
|
||||||
const ollama = require('>/controller/ollama');
|
const ollama = require('>/controller/ollama');
|
||||||
|
const { fetchWithRetry, withRetry } = require('>/controller/retry');
|
||||||
|
|
||||||
// TPB numeric categories that count as Movies / TV( used for the q.php search and to
|
// TPB numeric categories that count as Movies / TV( used for the q.php search and to
|
||||||
// keep the LLM from ever seeing non-video results).
|
// keep the LLM from ever seeing non-video results).
|
||||||
@@ -22,17 +23,6 @@ function humanSize(bytes){
|
|||||||
return `${bytes.toFixed(bytes < 10 && i > 0 ? 1 : 0)} ${units[i]}`;
|
return `${bytes.toFixed(bytes < 10 && i > 0 ? 1 : 0)} ${units[i]}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchJSON(url, options){
|
|
||||||
let res = await fetch(url, options);
|
|
||||||
if(!res.ok){
|
|
||||||
let error = new Error(`UpstreamError`);
|
|
||||||
error.message = `Request to ${url.split('?')[0]} failed( ${res.status})`;
|
|
||||||
error.status = 502;
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
return await res.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Turn a raw TPB category id into 'movie' | 'tv' | null.
|
// Turn a raw TPB category id into 'movie' | 'tv' | null.
|
||||||
function videoKind(category){
|
function videoKind(category){
|
||||||
category = parseInt(category, 10);
|
category = parseInt(category, 10);
|
||||||
@@ -90,13 +80,7 @@ function parseTPBHtml(html){
|
|||||||
async function tpbSearch(title){
|
async function tpbSearch(title){
|
||||||
// Category 200 = all Video; we narrow to movie/tv subcats in prefilter().
|
// Category 200 = all Video; we narrow to movie/tv subcats in prefilter().
|
||||||
let url = `${conf.search.tpbBase}/search/${encodeURIComponent(title)}/1/99/200`;
|
let url = `${conf.search.tpbBase}/search/${encodeURIComponent(title)}/1/99/200`;
|
||||||
let res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
|
let res = await fetchWithRetry(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }, { label: 'tpb search' });
|
||||||
if(!res.ok){
|
|
||||||
let error = new Error('UpstreamError');
|
|
||||||
error.message = `TPB search failed( ${res.status})`;
|
|
||||||
error.status = 502;
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
return parseTPBHtml(await res.text());
|
return parseTPBHtml(await res.text());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,26 +163,13 @@ async function rankReleases(candidates, meta){
|
|||||||
imdb: t.imdb || null,
|
imdb: t.imdb || null,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
let body = {
|
let parsed = await withRetry(
|
||||||
model: conf.ollama.model,
|
() => ollama.chatJSON(
|
||||||
stream: false,
|
buildSystemPrompt(meta.title, meta.year, meta.today),
|
||||||
format: 'json',
|
JSON.stringify(compact)
|
||||||
messages: [
|
),
|
||||||
{ role: 'system', content: buildSystemPrompt(meta.title, meta.year, meta.today) },
|
{ label: 'ollama rankReleases' }
|
||||||
{ role: 'user', content: JSON.stringify(compact) },
|
);
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
let data = await fetchJSON(`${conf.ollama.url}/api/chat`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Authorization': `Bearer ${conf.ollama.apiKey}`,
|
|
||||||
},
|
|
||||||
body: JSON.stringify(body),
|
|
||||||
});
|
|
||||||
|
|
||||||
let parsed = extractJSON(data.message && data.message.content);
|
|
||||||
let options = Array.isArray(parsed.options) ? parsed.options : [];
|
let options = Array.isArray(parsed.options) ? parsed.options : [];
|
||||||
if(!options.length) throw new Error('Model returned no options');
|
if(!options.length) throw new Error('Model returned no options');
|
||||||
return options;
|
return options;
|
||||||
@@ -305,7 +276,10 @@ function buildSeasonPrompt(title, season, today){
|
|||||||
|
|
||||||
async function bestSeasonPack(candidates, meta){
|
async function bestSeasonPack(candidates, meta){
|
||||||
let compact = candidates.map(t => ({ name: t.name, sizeHuman: humanSize(t.size), seeders: Number(t.seeders), info_hash: t.info_hash }));
|
let compact = candidates.map(t => ({ name: t.name, sizeHuman: humanSize(t.size), seeders: Number(t.seeders), info_hash: t.info_hash }));
|
||||||
let parsed = await ollama.chatJSON(buildSeasonPrompt(meta.title, meta.season, meta.today), JSON.stringify(compact));
|
let parsed = await withRetry(
|
||||||
|
() => ollama.chatJSON(buildSeasonPrompt(meta.title, meta.season, meta.today), JSON.stringify(compact)),
|
||||||
|
{ label: 'ollama bestSeasonPack' }
|
||||||
|
);
|
||||||
return parsed && parsed.info_hash ? parsed : null;
|
return parsed && parsed.info_hash ? parsed : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -313,15 +287,44 @@ async function bestSeasonPack(candidates, meta){
|
|||||||
async function seasonPlan(tmdbId){
|
async function seasonPlan(tmdbId){
|
||||||
let [tv, inv] = await Promise.all([ tmdb.tmdbSeasons(tmdbId), library.seriesInventory(tmdbId) ]);
|
let [tv, inv] = await Promise.all([ tmdb.tmdbSeasons(tmdbId), library.seriesInventory(tmdbId) ]);
|
||||||
let seasons = tv.seasons.map(s => {
|
let seasons = tv.seasons.map(s => {
|
||||||
let have = Number(inv.have && inv.have[s.season]) || 0;
|
let haveSet = inv.have[s.season] || new Set();
|
||||||
let status = have === 0 ? 'missing' : (have >= s.episodeCount ? 'complete' : 'partial');
|
let haveCount = haveSet.size;
|
||||||
return { season: s.season, episodeCount: s.episodeCount, haveCount: have, status };
|
let status = haveCount === 0 ? 'missing' : (haveCount >= s.episodeCount ? 'complete' : 'partial');
|
||||||
|
let missingEps = [];
|
||||||
|
if(status === 'partial'){
|
||||||
|
for(let e = 1; e <= s.episodeCount; e++){
|
||||||
|
if(!haveSet.has(e)) missingEps.push(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { season: s.season, episodeCount: s.episodeCount, haveCount, status, missingEpisodes: missingEps };
|
||||||
});
|
});
|
||||||
return { title: tv.title, year: tv.year, seasons, missing: seasons.filter(s => s.status !== 'complete').map(s => s.season) };
|
return { title: tv.title, year: tv.year, seasons, missing: seasons.filter(s => s.status !== 'complete').map(s => s.season) };
|
||||||
}
|
}
|
||||||
|
|
||||||
// For every missing/incomplete season, pick the best complete-season pack. Returns the
|
// Best single episode torrent for a specific SxxExx.
|
||||||
// chosen packs( the route adds them to Transmission) plus any seasons it couldn't fill.
|
async function bestEpisodeTorrent(candidates, meta){
|
||||||
|
let compact = candidates.map(t => ({
|
||||||
|
name: t.name,
|
||||||
|
sizeHuman: humanSize(t.size),
|
||||||
|
seeders: Number(t.seeders),
|
||||||
|
info_hash: t.info_hash,
|
||||||
|
}));
|
||||||
|
let prompt = [
|
||||||
|
`You are a torrent selector. Pick the single best torrent for "${meta.title}"`,
|
||||||
|
`Season ${meta.season} Episode ${meta.episode}.`,
|
||||||
|
`It MUST be this exact episode (S${String(meta.season).padStart(2, '0')}E${String(meta.episode).padStart(2, '0')}), not a pack or a different episode.`,
|
||||||
|
`Prefer HEVC/x265, 1080p, English audio, reasonable size. Reject CAM/TS/telesync and non-English-only.`,
|
||||||
|
`Return STRICT JSON {"info_hash":"<from candidates>","why":"one short sentence"}; if nothing suitable, {"info_hash":null}.`,
|
||||||
|
].join(' ');
|
||||||
|
let parsed = await withRetry(
|
||||||
|
() => ollama.chatJSON(prompt, JSON.stringify(compact)),
|
||||||
|
{ label: 'ollama bestEpisodeTorrent' }
|
||||||
|
);
|
||||||
|
return parsed && parsed.info_hash ? parsed : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For every missing/incomplete season, first try a complete-season pack. If that fails,
|
||||||
|
// fall back to searching each missing individual episode and queue them separately.
|
||||||
async function fillMissing(tmdbId){
|
async function fillMissing(tmdbId){
|
||||||
let plan = await seasonPlan(tmdbId);
|
let plan = await seasonPlan(tmdbId);
|
||||||
let details = await tmdbDetails(tmdbId, 'tv');
|
let details = await tmdbDetails(tmdbId, 'tv');
|
||||||
@@ -329,16 +332,43 @@ async function fillMissing(tmdbId){
|
|||||||
let today = new Date().toISOString().slice(0, 10);
|
let today = new Date().toISOString().slice(0, 10);
|
||||||
|
|
||||||
let picks = [], skipped = [];
|
let picks = [], skipped = [];
|
||||||
for(let season of plan.missing){
|
for(let seasonInfo of plan.seasons.filter(s => s.status !== 'complete')){
|
||||||
let cands = prefilter(await tpbSearch(`${title} S${String(season).padStart(2, '0')}`), null);
|
let season = seasonInfo.season;
|
||||||
if(!cands.length){ skipped.push({ season, reason: 'no results' }); continue; }
|
|
||||||
|
// 1) Try a complete season pack first.
|
||||||
|
let seasonCands = prefilter(await tpbSearch(`${title} S${String(season).padStart(2, '0')}`), null);
|
||||||
|
if(seasonCands.length){
|
||||||
|
let pick = null;
|
||||||
|
try{ pick = await bestSeasonPack(seasonCands, { title, season, today }); }catch(error){ pick = null; }
|
||||||
|
let src = pick && seasonCands.find(c => c.info_hash.toLowerCase() === String(pick.info_hash).toLowerCase());
|
||||||
|
if(src){
|
||||||
|
picks.push({ type: 'season', season, name: src.name, info_hash: src.info_hash, category: Number(src.category), magnetLink: buildMagnet(src.info_hash, src.name), why: pick.why });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) No pack. If we know exactly which episodes are missing, grab them one by one.
|
||||||
|
let missingEps = seasonInfo.missingEpisodes.length ? seasonInfo.missingEpisodes : [];
|
||||||
|
if(!missingEps.length){
|
||||||
|
// Fallback: episode-by-episode for the whole season when Emby inventory is unavailable.
|
||||||
|
for(let e = 1; e <= seasonInfo.episodeCount; e++) missingEps.push(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
let seasonQueued = 0;
|
||||||
|
for(let episode of missingEps){
|
||||||
|
let seLabel = `S${String(season).padStart(2, '0')}E${String(episode).padStart(2, '0')}`;
|
||||||
|
let cands = prefilter(await tpbSearch(`${title} ${seLabel}`), null);
|
||||||
|
if(!cands.length){ skipped.push({ season, episode, reason: 'no results' }); continue; }
|
||||||
|
|
||||||
let pick = null;
|
let pick = null;
|
||||||
try{ pick = await bestSeasonPack(cands, { title, season, today }); }catch(error){ pick = null; }
|
try{ pick = await bestEpisodeTorrent(cands, { title, season, episode }); }catch(error){ pick = null; }
|
||||||
let src = pick && cands.find(c => c.info_hash.toLowerCase() === String(pick.info_hash).toLowerCase());
|
let src = pick && cands.find(c => c.info_hash.toLowerCase() === String(pick.info_hash).toLowerCase());
|
||||||
if(!src){ skipped.push({ season, reason: 'no suitable season pack' }); continue; }
|
if(!src){ skipped.push({ season, episode, reason: 'no suitable episode torrent' }); continue; }
|
||||||
|
|
||||||
picks.push({ season, name: src.name, info_hash: src.info_hash, category: Number(src.category), magnetLink: buildMagnet(src.info_hash, src.name), why: pick.why });
|
picks.push({ type: 'episode', season, episode, name: src.name, info_hash: src.info_hash, category: Number(src.category), magnetLink: buildMagnet(src.info_hash, src.name), why: pick.why });
|
||||||
|
seasonQueued++;
|
||||||
|
}
|
||||||
|
if(!seasonQueued) skipped.push({ season, reason: 'no suitable season pack or episodes' });
|
||||||
}
|
}
|
||||||
return { title, year: plan.year, picks, skipped };
|
return { title, year: plan.year, picks, skipped };
|
||||||
}
|
}
|
||||||
|
|||||||
+5
-15
@@ -1,23 +1,13 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const conf = require('>/conf');
|
const conf = require('>/conf');
|
||||||
|
const { fetchJSONWithRetry } = require('./retry');
|
||||||
async function fetchJSON(url, options){
|
|
||||||
let res = await fetch(url, options);
|
|
||||||
if(!res.ok){
|
|
||||||
let error = new Error('UpstreamError');
|
|
||||||
error.message = `Request to ${url.split('?')[0]} failed( ${res.status})`;
|
|
||||||
error.status = 502;
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
return await res.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function tmdbSearch(query){
|
async function tmdbSearch(query){
|
||||||
let url = `https://api.themoviedb.org/3/search/multi?include_adult=false`
|
let url = `https://api.themoviedb.org/3/search/multi?include_adult=false`
|
||||||
+ `&query=${encodeURIComponent(query)}&api_key=${conf.tmdb.apiKey}`;
|
+ `&query=${encodeURIComponent(query)}&api_key=${conf.tmdb.apiKey}`;
|
||||||
|
|
||||||
let data = await fetchJSON(url);
|
let data = await fetchJSONWithRetry(url);
|
||||||
|
|
||||||
return (data.results || [])
|
return (data.results || [])
|
||||||
.filter(item => item.media_type === 'movie' || item.media_type === 'tv')
|
.filter(item => item.media_type === 'movie' || item.media_type === 'tv')
|
||||||
@@ -38,7 +28,7 @@ async function tmdbDetails(tmdbId, mediaType){
|
|||||||
let url = `https://api.themoviedb.org/3/${mediaType}/${tmdbId}`
|
let url = `https://api.themoviedb.org/3/${mediaType}/${tmdbId}`
|
||||||
+ `?append_to_response=external_ids&api_key=${conf.tmdb.apiKey}`;
|
+ `?append_to_response=external_ids&api_key=${conf.tmdb.apiKey}`;
|
||||||
|
|
||||||
let data = await fetchJSON(url);
|
let data = await fetchJSONWithRetry(url);
|
||||||
let date = data.release_date || data.first_air_date || '';
|
let date = data.release_date || data.first_air_date || '';
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -71,7 +61,7 @@ async function tmdbFindBest(title, year, mediaType){
|
|||||||
|
|
||||||
// Per-season episode counts for a show( season 0 / specials excluded).
|
// Per-season episode counts for a show( season 0 / specials excluded).
|
||||||
async function tmdbSeasons(tmdbId){
|
async function tmdbSeasons(tmdbId){
|
||||||
let data = await fetchJSON(`https://api.themoviedb.org/3/tv/${tmdbId}?api_key=${conf.tmdb.apiKey}`);
|
let data = await fetchJSONWithRetry(`https://api.themoviedb.org/3/tv/${tmdbId}?api_key=${conf.tmdb.apiKey}`);
|
||||||
let date = data.first_air_date || '';
|
let date = data.first_air_date || '';
|
||||||
return {
|
return {
|
||||||
title: data.name,
|
title: data.name,
|
||||||
@@ -82,4 +72,4 @@ async function tmdbSeasons(tmdbId){
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { fetchJSON, tmdbSearch, tmdbDetails, tmdbFindBest, tmdbSeasons };
|
module.exports = { tmdbSearch, tmdbDetails, tmdbFindBest, tmdbSeasons };
|
||||||
|
|||||||
+44
-14
@@ -2,8 +2,23 @@
|
|||||||
|
|
||||||
const Transmission = require('transmission-promise');
|
const Transmission = require('transmission-promise');
|
||||||
const conf = require('>/conf');
|
const conf = require('>/conf');
|
||||||
|
const { withRetry } = require('>/controller/retry');
|
||||||
|
|
||||||
const tr_client = new Transmission(conf.transmission)
|
const tr_client = new Transmission(conf.transmission);
|
||||||
|
|
||||||
|
// Wrap the raw Transmission client so every RPC call retries on transient network errors.
|
||||||
|
const trClient = new Proxy(tr_client, {
|
||||||
|
get(target, prop){
|
||||||
|
let value = target[prop];
|
||||||
|
if(typeof value !== 'function') return value;
|
||||||
|
return async function(...args){
|
||||||
|
return await withRetry(
|
||||||
|
() => value.apply(target, args),
|
||||||
|
{ label: `transmission ${String(prop)}` }
|
||||||
|
);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
const statusMap = [
|
const statusMap = [
|
||||||
'STOPPED', // 0
|
'STOPPED', // 0
|
||||||
@@ -27,7 +42,7 @@ module.exports = (sequelize, DataTypes, Model) => {
|
|||||||
// define association here
|
// define association here
|
||||||
}
|
}
|
||||||
|
|
||||||
static trClient = tr_client;
|
static trClient = trClient;
|
||||||
|
|
||||||
// Map a raw TPB category id( e.g. 207) to one of our buckets. TPB groups by the
|
// Map a raw TPB category id( e.g. 207) to one of our buckets. TPB groups by the
|
||||||
// hundreds digit; TV shows are the exception pulled out of the Video group.
|
// hundreds digit; TV shows are the exception pulled out of the Video group.
|
||||||
@@ -46,29 +61,44 @@ module.exports = (sequelize, DataTypes, Model) => {
|
|||||||
|
|
||||||
static async create(data, ...args){
|
static async create(data, ...args){
|
||||||
try{
|
try{
|
||||||
|
|
||||||
// let instance = this.build(data);
|
|
||||||
// console.log('instance', instance)
|
|
||||||
await this.build(data).validate();
|
|
||||||
// console.log('validate', val);
|
|
||||||
data.isPrivate = data.isPrivate === 'true' ? true : false;
|
data.isPrivate = data.isPrivate === 'true' ? true : false;
|
||||||
|
|
||||||
let options = {
|
let options = {
|
||||||
'download-dir': data.isPrivate ? `${conf.privateDownloadLocation}/${data.added_by}` : undefined,
|
'download-dir': data.isPrivate ? `${conf.privateDownloadLocation}/${data.added_by}` : undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
let res = await tr_client.addUrl(data.magnetLink, options);
|
let res = await trClient.addUrl(data.magnetLink, options);
|
||||||
|
|
||||||
return await super.create({
|
// Transmission usually returns hashString, but if it doesn't (duplicate, malformed
|
||||||
|
// response, etc.), fall back to parsing the info hash from the magnet link.
|
||||||
|
let hashString = res && (res.hashString || res.id);
|
||||||
|
if(!hashString){
|
||||||
|
let match = data.magnetLink.match(/urn:btih:([A-Fa-f0-9]{40})/i);
|
||||||
|
if(match) hashString = match[1].toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!hashString){
|
||||||
|
console.error('Transmission addUrl response missing hashString:', res);
|
||||||
|
let error = new Error('TorrentCreateError');
|
||||||
|
error.message = 'Transmission did not return a hash for this torrent';
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
let createData = {
|
||||||
magnetLink: data.magnetLink,
|
magnetLink: data.magnetLink,
|
||||||
hashString: res.hashString,
|
hashString,
|
||||||
isPrivate: data.isPrivate,
|
isPrivate: data.isPrivate,
|
||||||
name: res.name,
|
name: res && res.name,
|
||||||
added_by: data.added_by,
|
added_by: data.added_by,
|
||||||
category: this.categoryFromTPB(data.category),
|
category: this.categoryFromTPB(data.category),
|
||||||
status: 0,
|
status: 0,
|
||||||
percentDone: 0,
|
percentDone: 0,
|
||||||
}, args);
|
};
|
||||||
|
|
||||||
|
// Validate only after hashString is populated.
|
||||||
|
await this.build(createData).validate();
|
||||||
|
|
||||||
|
return await super.create(createData, args);
|
||||||
}catch (error){
|
}catch (error){
|
||||||
console.log('Torrent create error', error);
|
console.log('Torrent create error', error);
|
||||||
throw error;
|
throw error;
|
||||||
@@ -80,7 +110,7 @@ module.exports = (sequelize, DataTypes, Model) => {
|
|||||||
|
|
||||||
if(this.percentDone === 1) return this.dataValues
|
if(this.percentDone === 1) return this.dataValues
|
||||||
|
|
||||||
let res = ( await tr_client.get(this.hashString, [
|
let res = ( await trClient.get(this.hashString, [
|
||||||
"eta", "percentDone", "status", "rateDownload",
|
"eta", "percentDone", "status", "rateDownload",
|
||||||
"errorString", "hashString", 'name',
|
"errorString", "hashString", 'name',
|
||||||
'downloadDir',
|
'downloadDir',
|
||||||
@@ -104,7 +134,7 @@ module.exports = (sequelize, DataTypes, Model) => {
|
|||||||
throw e
|
throw e
|
||||||
}
|
}
|
||||||
// console.error(`Torrent ${this.hashString} getTorrentData error`, error);
|
// console.error(`Torrent ${this.hashString} getTorrentData error`, error);
|
||||||
throw error;
|
throw error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Generated
+16
@@ -20,6 +20,7 @@
|
|||||||
"lru-native2": "^1.2.6",
|
"lru-native2": "^1.2.6",
|
||||||
"moment": "^2.30.1",
|
"moment": "^2.30.1",
|
||||||
"mustache": "^4.2.0",
|
"mustache": "^4.2.0",
|
||||||
|
"ollama": "^0.6.3",
|
||||||
"p2psub": "^0.1.9",
|
"p2psub": "^0.1.9",
|
||||||
"sequelize": "^6.35.2",
|
"sequelize": "^6.35.2",
|
||||||
"sequelize-cli": "^6.6.2",
|
"sequelize-cli": "^6.6.2",
|
||||||
@@ -2933,6 +2934,15 @@
|
|||||||
"url": "https://github.com/sponsors/ljharb"
|
"url": "https://github.com/sponsors/ljharb"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/ollama": {
|
||||||
|
"version": "0.6.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/ollama/-/ollama-0.6.3.tgz",
|
||||||
|
"integrity": "sha512-KEWEhIqE5wtfzEIZbDCLH51VFZ6Z3ZSa6sIOg/E/tBV8S51flyqBOXi+bRxlOYKDf8i327zG9eSTb8IJxvm3Zg==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"whatwg-fetch": "^3.6.20"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/on-finished": {
|
"node_modules/on-finished": {
|
||||||
"version": "2.4.1",
|
"version": "2.4.1",
|
||||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
|
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
|
||||||
@@ -4411,6 +4421,12 @@
|
|||||||
"node": ">=12"
|
"node": ">=12"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/whatwg-fetch": {
|
||||||
|
"version": "3.6.20",
|
||||||
|
"resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz",
|
||||||
|
"integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/whatwg-url": {
|
"node_modules/whatwg-url": {
|
||||||
"version": "14.2.0",
|
"version": "14.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
|
||||||
|
|||||||
@@ -23,6 +23,7 @@
|
|||||||
"lru-native2": "^1.2.6",
|
"lru-native2": "^1.2.6",
|
||||||
"moment": "^2.30.1",
|
"moment": "^2.30.1",
|
||||||
"mustache": "^4.2.0",
|
"mustache": "^4.2.0",
|
||||||
|
"ollama": "^0.6.3",
|
||||||
"p2psub": "^0.1.9",
|
"p2psub": "^0.1.9",
|
||||||
"sequelize": "^6.35.2",
|
"sequelize": "^6.35.2",
|
||||||
"sequelize-cli": "^6.6.2",
|
"sequelize-cli": "^6.6.2",
|
||||||
|
|||||||
+2
-2
@@ -78,9 +78,9 @@ router.post('/fill', async function(req, res, next){
|
|||||||
added_by: req.user.username,
|
added_by: req.user.username,
|
||||||
category: pick.category,
|
category: pick.category,
|
||||||
});
|
});
|
||||||
queued.push({ season: pick.season, name: pick.name });
|
queued.push({ type: pick.type, season: pick.season, episode: pick.episode, name: pick.name });
|
||||||
}catch(error){
|
}catch(error){
|
||||||
plan.skipped.push({ season: pick.season, reason: error.message });
|
plan.skipped.push({ season: pick.season, episode: pick.episode, reason: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
res.json({ title: plan.title, year: plan.year, queued, skipped: plan.skipped });
|
res.json({ title: plan.title, year: plan.year, queued, skipped: plan.skipped });
|
||||||
|
|||||||
Reference in New Issue
Block a user