359 lines
15 KiB
JavaScript
359 lines
15 KiB
JavaScript
const { Resource, ResourceEdge, ResourceGroup } = require('../models/resource');
|
|
const { WebhookEmitter } = require('./webhook_emitter');
|
|
const crypto = require('crypto');
|
|
|
|
// Is `candidateId` at or below `rootId` in the edge graph? Used to refuse an
|
|
// edge that would close a loop. Carries its own visited set so it terminates
|
|
// even if the stored graph already contains a cycle from an older release.
|
|
function isDescendant(candidateId, rootId, edges) {
|
|
const seen = new Set();
|
|
const stack = [rootId];
|
|
while (stack.length) {
|
|
const id = stack.pop();
|
|
if (id === candidateId) return true;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
for (const e of edges) if (e.parentId === id) stack.push(e.childId);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
class DiscoveryReconciler {
|
|
static async reconcile(sourceName, payload, options = {}) {
|
|
const { resources = [], edges = [] } = payload;
|
|
let newDevices = 0;
|
|
const location = options.location || options.site || null;
|
|
const autoPromote = !!options.autoPromote;
|
|
|
|
let targetSite = null;
|
|
if (location && String(location).trim()) {
|
|
const sites = await Resource.list({ where: { kind: 'site' } });
|
|
const locStr = String(location).trim().toLowerCase();
|
|
targetSite = sites.find(s => s.name.toLowerCase() === locStr || s.slug.toLowerCase() === locStr);
|
|
if (!targetSite) {
|
|
const locSlug = `site-${locStr.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')}`;
|
|
targetSite = await Resource.create({
|
|
id: crypto.randomUUID(),
|
|
kind: 'site',
|
|
name: String(location).trim(),
|
|
slug: locSlug,
|
|
created_on: Math.floor(Date.now() / 1000)
|
|
}).catch(() => null);
|
|
}
|
|
}
|
|
if (!targetSite) {
|
|
const sites = await Resource.list({ where: { kind: 'site' } });
|
|
if (sites && sites.length > 0) {
|
|
targetSite = sites[0];
|
|
} else {
|
|
targetSite = await Resource.create({
|
|
id: crypto.randomUUID(),
|
|
kind: 'site',
|
|
name: 'Default Site',
|
|
slug: 'site-default',
|
|
created_on: Math.floor(Date.now() / 1000)
|
|
}).catch(() => null);
|
|
}
|
|
}
|
|
|
|
const normalizeMac = (m) => (m || '').toLowerCase().replace(/[^a-f0-9]/g, '');
|
|
const normalizeHost = (h) => (h || '').toLowerCase().split('.')[0].trim();
|
|
|
|
// Read the inventory ONCE, not once per incoming resource. A Proxmox
|
|
// cluster reports ~55 resources against an inventory of similar size, so
|
|
// the per-iteration Resource.list() was doing quadratic full-table reads
|
|
// every discovery run. Newly created rows are pushed onto this list as we
|
|
// go, so later resources in the same payload still match against them.
|
|
const allRes = await Resource.list();
|
|
|
|
for (const res of resources) {
|
|
if (!res.metadata) res.metadata = {};
|
|
if (autoPromote) res.metadata.managed = true;
|
|
res._originalSlug = res.slug; // Keep track for edge mapping
|
|
|
|
let existing = null;
|
|
|
|
// A discovered device may only merge into a resource of the same kind
|
|
// (or into a placeholder from an earlier, kind-less discovery). Without
|
|
// this a VM called "gitea-runner" matches a hand-created *service* of
|
|
// the same name on rule 3 and silently overwrites it -- the discovered
|
|
// host's metadata lands on a service row, and the operator's entry is
|
|
// gone. `template` counts as `host`: a VM converted to a template is the
|
|
// same device, and it should update in place rather than fork a row.
|
|
const kindClass = (k) => (k === 'template' ? 'host' : k);
|
|
const incomingKind = kindClass(res.kind || 'unmanaged_device');
|
|
const kindCompatible = (r) => {
|
|
const k = kindClass(r.kind);
|
|
if (k === 'unmanaged_device' || incomingKind === 'unmanaged_device') return true;
|
|
return k === incomingKind;
|
|
};
|
|
const candidates = allRes.filter(kindCompatible);
|
|
|
|
// 1. Attempt matching by MAC (highest precision)
|
|
if (res.metadata.interfaces && res.metadata.interfaces.length > 0) {
|
|
const macs = res.metadata.interfaces.map(i => normalizeMac(i.mac)).filter(m => m.length === 12);
|
|
if (macs.length > 0) {
|
|
existing = candidates.find(r =>
|
|
r.metadata && (
|
|
(r.metadata.macAddress && macs.includes(normalizeMac(r.metadata.macAddress))) ||
|
|
(r.metadata.interfaces && r.metadata.interfaces.some(i => macs.includes(normalizeMac(i.mac))))
|
|
)
|
|
);
|
|
}
|
|
}
|
|
|
|
// 2. Fallback matching by IP address
|
|
let ipsToMatch = [];
|
|
if (res.metadata.interfaces) {
|
|
ipsToMatch = res.metadata.interfaces.map(i => i.ip).filter(i => !!i);
|
|
}
|
|
if (res.metadata.ip) ipsToMatch.push(res.metadata.ip);
|
|
if (res.metadata.address) {
|
|
res.metadata.address.split(',').forEach(a => ipsToMatch.push(a.trim()));
|
|
}
|
|
ipsToMatch = [...new Set(ipsToMatch.filter(Boolean))];
|
|
|
|
if (!existing && ipsToMatch.length > 0) {
|
|
existing = candidates.find(r => {
|
|
if (!r.metadata) return false;
|
|
if (r.metadata.ip && ipsToMatch.includes(r.metadata.ip)) return true;
|
|
if (r.metadata.address) {
|
|
const addrs = r.metadata.address.split(',').map(a => a.trim());
|
|
if (addrs.some(a => ipsToMatch.includes(a))) return true;
|
|
}
|
|
if (r.metadata.interfaces && r.metadata.interfaces.some(i => ipsToMatch.includes(i.ip))) return true;
|
|
return false;
|
|
});
|
|
}
|
|
|
|
// 3. Fallback matching by Slug, Name, or Base Hostname
|
|
if (!existing && (res.slug || res.name)) {
|
|
const inputName = normalizeHost(res.name || res.slug);
|
|
existing = candidates.find(r => {
|
|
if (res.slug && r.slug === res.slug) return true;
|
|
if (res.name && r.name && r.name.toLowerCase() === res.name.toLowerCase()) return true;
|
|
if (inputName && r.name && normalizeHost(r.name) === inputName) return true;
|
|
if (inputName && r.slug && normalizeHost(r.slug) === inputName) return true;
|
|
return false;
|
|
});
|
|
}
|
|
|
|
if (existing) {
|
|
// Merge metadata
|
|
const mergedMeta = { ...existing.metadata, ...res.metadata };
|
|
|
|
// Merge interfaces cleanly
|
|
if (res.metadata.interfaces) {
|
|
const existingIntfs = existing.metadata.interfaces || [];
|
|
const newIntfs = res.metadata.interfaces;
|
|
// Simple union based on mac or ip
|
|
for (const ni of newIntfs) {
|
|
const idx = existingIntfs.findIndex(ei =>
|
|
(ni.mac && ei.mac && ei.mac.toLowerCase() === ni.mac.toLowerCase()) ||
|
|
(ni.ip && ei.ip && ei.ip === ni.ip)
|
|
);
|
|
if (idx >= 0) existingIntfs[idx] = { ...existingIntfs[idx], ...ni };
|
|
else existingIntfs.push(ni);
|
|
}
|
|
mergedMeta.interfaces = existingIntfs;
|
|
}
|
|
|
|
// Add discovery source
|
|
const sources = new Set(mergedMeta.discovery_sources || []);
|
|
sources.add(sourceName);
|
|
mergedMeta.discovery_sources = [...sources];
|
|
|
|
mergedMeta.last_seen = Date.now();
|
|
|
|
// Pick the most human name across sources. Rank first, length only as
|
|
// a tie-break within a rank -- comparing lengths alone let a UniFi
|
|
// client named after its MAC ("ac:16:2d:b3:da:80", 17 chars) beat the
|
|
// hypervisor's real hostname from Proxmox ("dl380-0", 7), so the
|
|
// Directory listed MAC addresses where host names belong.
|
|
//
|
|
// NB: `\\.` inside a regex LITERAL matches a backslash, not a dot, so
|
|
// the old isIp returned false for every input and IP-shaped names were
|
|
// never replaced either. It is `\.` here.
|
|
const isIp = (str) => /^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$/.test(str || '');
|
|
const isMac = (str) => /^([0-9a-f]{2}[:-]){5}[0-9a-f]{2}$/i.test((str || '').trim());
|
|
// 2 = a real name, 1 = an IP (at least routable/recognizable), 0 = a
|
|
// MAC or nothing (pure machine identifier, the worst thing to show).
|
|
const nameRank = (str) => {
|
|
if (!str || !String(str).trim()) return 0;
|
|
if (isMac(str)) return 0;
|
|
if (isIp(str)) return 1;
|
|
return 2;
|
|
};
|
|
|
|
let bestName = existing.name;
|
|
if (res.name) {
|
|
const incoming = nameRank(res.name);
|
|
const current = nameRank(bestName);
|
|
if (incoming > current || (incoming === current && res.name.length > (bestName || '').length)) {
|
|
bestName = res.name;
|
|
}
|
|
}
|
|
|
|
await existing.update({
|
|
name: bestName,
|
|
description: res.description || existing.description,
|
|
metadata: mergedMeta,
|
|
updated_on: Math.floor(Date.now() / 1000)
|
|
});
|
|
res._actualId = existing.id;
|
|
} else {
|
|
// Create new
|
|
const sources = new Set([sourceName]);
|
|
res.metadata.discovery_sources = [...sources];
|
|
res.metadata.last_seen = Date.now();
|
|
|
|
const slug = res.slug || `${res.kind}-${crypto.randomBytes(4).toString('hex')}`;
|
|
|
|
const created = await Resource.create({
|
|
id: crypto.randomUUID(),
|
|
kind: res.kind || 'unmanaged_device',
|
|
name: res.name || slug,
|
|
slug: slug,
|
|
metadata: res.metadata,
|
|
created_on: Math.floor(Date.now() / 1000)
|
|
});
|
|
|
|
newDevices++;
|
|
res._actualId = created.id; // Map original slug to actual ID
|
|
// Make it visible to the rest of THIS payload: a Proxmox run reports
|
|
// the endpoint, then its nodes, then their guests, and two of them can
|
|
// legitimately share a MAC/IP. Without this the same device could be
|
|
// created twice in a single run.
|
|
allRes.push(created);
|
|
WebhookEmitter.emit('discovery.new_device', created.toJSON());
|
|
}
|
|
}
|
|
|
|
// Now process edges. `allRes` above is already current -- rows created in
|
|
// the loop were pushed onto it -- so no second full read is needed.
|
|
const existingEdges = await ResourceEdge.list();
|
|
|
|
for (const edge of edges) {
|
|
// Find parent ID. It might be in the current payload (mapped to _actualId) or in DB by slug
|
|
let parentId = null;
|
|
const parentResInPayload = resources.find(r => r._originalSlug === edge.parentSlug);
|
|
if (parentResInPayload && parentResInPayload._actualId) {
|
|
parentId = parentResInPayload._actualId;
|
|
} else {
|
|
const parentResInDb = allRes.find(r => r.slug === edge.parentSlug);
|
|
if (parentResInDb) parentId = parentResInDb.id;
|
|
}
|
|
|
|
// Find child ID
|
|
let childId = null;
|
|
const childResInPayload = resources.find(r => r._originalSlug === edge.childSlug);
|
|
if (childResInPayload && childResInPayload._actualId) {
|
|
childId = childResInPayload._actualId;
|
|
} else {
|
|
const childResInDb = allRes.find(r => r.slug === edge.childSlug);
|
|
if (childResInDb) childId = childResInDb.id;
|
|
}
|
|
|
|
// Two slugs in one payload can resolve to the SAME resource once the
|
|
// matcher has merged them -- a Proxmox endpoint reached at the address
|
|
// of the node that answers for it is the case that produced this. The
|
|
// edge would then make a resource its own parent, which renders as an
|
|
// infinitely nested tree and defeats every ancestor walk in the app
|
|
// (findAncestorSiteSlug, withResolvedAddress) that relies on a cycle
|
|
// guard to terminate rather than to be correct.
|
|
if (parentId && childId && parentId === childId) {
|
|
console.warn(`[DiscoveryReconciler] ${sourceName}: dropping self-edge on ${edge.parentSlug} -> ${edge.childSlug} (both resolved to the same resource)`);
|
|
continue;
|
|
}
|
|
|
|
// Likewise refuse an edge that closes a loop: if the proposed parent is
|
|
// already a descendant of the proposed child, adding this makes a cycle.
|
|
if (parentId && childId && isDescendant(parentId, childId, existingEdges)) {
|
|
console.warn(`[DiscoveryReconciler] ${sourceName}: dropping ${edge.parentSlug} -> ${edge.childSlug} (would create a cycle)`);
|
|
continue;
|
|
}
|
|
|
|
if (parentId && childId) {
|
|
const edgeExists = existingEdges.find(e => e.parentId === parentId && e.childId === childId && e.relation === edge.relation);
|
|
if (!edgeExists) {
|
|
const created = await ResourceEdge.create({
|
|
id: crypto.randomUUID(),
|
|
parentId,
|
|
childId,
|
|
relation: edge.relation
|
|
});
|
|
// Keep the in-memory edge list current so the cycle check above sees
|
|
// edges added earlier in this same payload.
|
|
existingEdges.push(created);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (targetSite) {
|
|
const childSlugs = new Set(edges.map(e => e.childSlug));
|
|
for (const res of resources) {
|
|
if (res._actualId && res._actualId !== targetSite.id && !childSlugs.has(res._originalSlug || res.slug)) {
|
|
const edgeExists = existingEdges.find(e => e.childId === res._actualId);
|
|
if (!edgeExists) {
|
|
const created = await ResourceEdge.create({
|
|
id: crypto.randomUUID(),
|
|
parentId: targetSite.id,
|
|
childId: res._actualId,
|
|
relation: 'hosts'
|
|
}).catch(() => null);
|
|
if (created) existingEdges.push(created);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (autoPromote) {
|
|
const { Group } = require('../models/group_ldap');
|
|
for (const res of resources) {
|
|
if (!res._actualId) continue;
|
|
const accessGroup = `${res.slug}_access`;
|
|
const adminGroup = `${res.slug}_admin`;
|
|
try {
|
|
await Group.get(accessGroup).catch(async (e) => {
|
|
if (e.status === 404) await Group.add({ name: accessGroup, description: `Access to ${res.name}`, owner: 'cn=admin' });
|
|
});
|
|
await Group.get(adminGroup).catch(async (e) => {
|
|
if (e.status === 404) await Group.add({ name: adminGroup, description: `Admin access to ${res.name}`, owner: 'cn=admin' });
|
|
});
|
|
await ResourceGroup.create({ id: crypto.randomUUID(), resourceId: res._actualId, groupCn: accessGroup, accessLevel: 'user' }).catch(() => {});
|
|
await ResourceGroup.create({ id: crypto.randomUUID(), resourceId: res._actualId, groupCn: adminGroup, accessLevel: 'admin' }).catch(() => {});
|
|
} catch (err) {
|
|
console.error(`[DiscoveryReconciler] autoPromote failed for ${res.slug}:`, err.message);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (newDevices > 0) {
|
|
console.log(`[DiscoveryReconciler] Source ${sourceName} discovered ${newDevices} new devices.`);
|
|
}
|
|
}
|
|
|
|
static async garbageCollect(staleMs = 7 * 24 * 60 * 60 * 1000) {
|
|
const allRes = await Resource.list();
|
|
const cutoff = Date.now() - staleMs;
|
|
let archived = 0;
|
|
|
|
for (const res of allRes) {
|
|
const meta = res.metadata || {};
|
|
const sources = meta.discovery_sources || [];
|
|
// Only garbage collect things that are exclusively auto-discovered
|
|
if (sources.length > 0 && !sources.includes('manual')) {
|
|
if (meta.last_seen && meta.last_seen < cutoff && meta.lifecycle_state !== 'archived') {
|
|
meta.lifecycle_state = 'archived';
|
|
await res.update({ metadata: meta, updated_on: Math.floor(Date.now() / 1000) });
|
|
archived++;
|
|
WebhookEmitter.emit('discovery.device_archived', res.toJSON());
|
|
}
|
|
}
|
|
}
|
|
if (archived > 0) console.log(`[DiscoveryReconciler] Garbage collected ${archived} stale devices.`);
|
|
}
|
|
}
|
|
|
|
module.exports = { DiscoveryReconciler };
|