HaverLab
/
sdl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored Cluster Status

This commit is contained in:
John Haverlack 2026-02-08 10:30:53 -09:00
parent 7c73b68981
commit 425821bce7
4 changed files with 148 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
docs/DESIGN.md
View File

@ -34,19 +34,19 @@ MVP - Minimum Viable Product Task List
- [x] Create Worker Telemetry Process - [x] Create Worker Telemetry Process
**Worker Monitoring & Telemetry** 🔄 (In Progress) **Worker Monitoring & Telemetry** 🔄 (In Progress)
- [ ] Worker Hardware Inventory Detection - [x] Worker Hardware Inventory Detection
- [ ] Physical vs Logical CPU cores (hyperthreading) - [x] Physical vs Logical CPU cores (hyperthreading)
- [ ] Physical vs VM detection (hypervisor flag) - [x] Physical vs VM detection (hypervisor flag)
- [ ] GPU VRAM capacity aggregation - [x] GPU VRAM capacity aggregation
- [ ] Memory capacity (bytes → GB conversion) - [x] Memory capacity (bytes → GB conversion)
- [ ] Worker Real-time Load Status (pure Node.js) - [x] Worker Real-time Load Status (pure Node.js)
- [ ] CPU usage percentage (`os.cpus()` times) - [x] CPU usage percentage (`os.cpus()` times)
- [ ] RAM usage (total/free/used via `os.totalmem()/freemem()`) - [x] RAM usage (total/free/used via `os.totalmem()/freemem()`)
- [ ] GPU usage (exec-based: nvidia-smi, rocm-smi) - [x] GPU usage (exec-based: nvidia-smi, rocm-smi)
- [ ] Disk IO (optional: exec-based) - [ ] Disk IO (optional: exec-based)
- [ ] Worker Uptime Tracking - [x] Worker Uptime Tracking
- [ ] SDL process uptime (`process.uptime()`) - [x] SDL process uptime (`process.uptime()`)
- [ ] OS uptime (`os.uptime()`) - [x] OS uptime (`os.uptime()`)
- [ ] Worker Display Improvements - [ ] Worker Display Improvements
- [ ] Show OS distro + version in Platform column (e.g., "Debian 12 / x64") - [ ] Show OS distro + version in Platform column (e.g., "Debian 12 / x64")
- [ ] Show system type (Physical 🖥️ / VM 💠) - [ ] Show system type (Physical 🖥️ / VM 💠)

304
sdl-mgr/app/modules/sdl-mgr/index.js
View File

@ -7,13 +7,12 @@ import dgram from 'dgram';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
const config = load_config(); const config = load_config();
log(`Loaded module: ${module}`); log(`Loaded module: ${module}`);
// log(`${module}: Cluster Conf: ${JSON.stringify(config.cluster, null, 2)}`, true);
// log(`${module}: Dirs: ${JSON.stringify(config.dirs, null, 2)}`, true); // ✅ In-memory workers state (built from telemetry only)
let workersState = {};
function loadClusterState(config) { function loadClusterState(config) {
const clusterDir = config.dirs.clstr; const clusterDir = config.dirs.clstr;
@ -30,31 +29,7 @@ function loadClusterState(config) {
meta: { meta: {
updated: new Date().toISOString(), updated: new Date().toISOString(),
started_at: new Date(getProcessStartTs()).toISOString(), started_at: new Date(getProcessStartTs()).toISOString(),
uptime: getUptimeDHMS(), uptime: getUptimeDHMS()
stats: {
workers: {
allocated: 0,
available: 0,
used: 0
},
resources: {
cpus: {
allocated: 0,
available: 0,
used: 0
},
memory: {
allocated: 0,
available: 0,
used: 0
},
gpus: {
allocated: 0,
available: 0,
used: 0
}
}
}
}, },
cluster: config.cluster, cluster: config.cluster,
"sdl-mgr": { "sdl-mgr": {
@ -65,8 +40,7 @@ function loadClusterState(config) {
distro: config.host.os.pretty_name, distro: config.host.os.pretty_name,
distro_name: config.host.os.name, distro_name: config.host.os.name,
distro_version: config.host.os.version distro_version: config.host.os.version
}, }
workers: {}
}; };
fs.writeFileSync(clusterFile, JSON.stringify(initialState, null, 2)); fs.writeFileSync(clusterFile, JSON.stringify(initialState, null, 2));
@ -89,55 +63,15 @@ function saveClusterState(config, state) {
// Write to disk // Write to disk
try { try {
fs.writeFileSync(clusterFile, JSON.stringify(state, null, 2)); fs.writeFileSync(clusterFile, JSON.stringify(state, null, 2));
log(`${module}: cluster state saved to ${clusterFile}`); // log(`${module}: cluster state saved to ${clusterFile}`);
} catch (err) { } catch (err) {
log(`${module}: failed to save cluster state: ${err}`); log(`${module}: failed to save cluster state: ${err}`);
throw err; throw err;
} }
} }
function addWorker(state, workerData) { // ✅ Compute stats from in-memory workers state
const sdl_id = workerData.sdl_id; function computeStats() {
// Check if worker already exists
const isNewWorker = !state.workers[sdl_id];
// Add or update worker
state.workers[sdl_id] = {
sdl_id: workerData.sdl_id,
hostname: workerData.hostname,
version: workerData.sdl_version,
platform: workerData.platform,
arch: workerData.arch,
distro: workerData.distro || "Unknown",
distro_name: workerData.distro_name || "Unknown",
distro_version: workerData.distro_version || "Unknown",
resources: {
cpus: {
allocated: workerData.cpus || 0,
available: workerData.cpus || 0,
used: 0
},
memory: {
allocated: workerData.totalmem || 0,
available: workerData.totalmem || 0,
used: 0
},
gpus: {
allocated: workerData.gpus || 0,
available: workerData.gpus || 0,
used: 0
}
},
status: 'active',
joined_at: isNewWorker ? new Date().toISOString() : state.workers[sdl_id].joined_at,
last_seen: new Date().toISOString()
};
return state;
}
function computeStats(state) {
const stats = { const stats = {
workers: { workers: {
allocated: 0, allocated: 0,
@ -164,29 +98,28 @@ function computeStats(state) {
}; };
// Count workers and aggregate resources // Count workers and aggregate resources
for (const worker of Object.values(state.workers)) { for (const worker of Object.values(workersState)) {
stats.workers.allocated++; stats.workers.allocated++;
if (worker.status === 'active') { if (worker.status === 'active') {
stats.workers.available++; stats.workers.available++;
stats.resources.cpus.available += worker.resources.cpus.available; stats.resources.cpus.available += worker.resources?.cpus?.available || 0;
stats.resources.memory.available += worker.resources.memory.available; stats.resources.memory.available += worker.resources?.memory?.available || 0;
stats.resources.gpus.available += worker.resources.gpus.available; stats.resources.gpus.available += worker.resources?.gpus?.available || 0;
} }
stats.resources.cpus.allocated += worker.resources.cpus.allocated; stats.resources.cpus.allocated += worker.resources?.cpus?.allocated || 0;
stats.resources.cpus.used += worker.resources.cpus.used; stats.resources.cpus.used += worker.usage?.cpu?.total || 0;
stats.resources.memory.allocated += worker.resources.memory.allocated; stats.resources.memory.allocated += worker.resources?.memory?.allocated || 0;
stats.resources.memory.used += worker.resources.memory.used; stats.resources.memory.used += worker.usage?.memory?.used || 0;
stats.resources.gpus.allocated += worker.resources.gpus.allocated; stats.resources.gpus.allocated += worker.resources?.gpus?.allocated || 0;
stats.resources.gpus.used += worker.resources.gpus.used; stats.resources.gpus.used += worker.usage?.gpu?.total_utilization || 0;
} }
state.meta.stats = stats; return stats;
return state;
} }
// Worker Join Handler // Worker Join Handler
@ -204,9 +137,6 @@ function startJoinHandler() {
return; return;
} }
// Load current cluster state
let clusterState = loadClusterState(config);
const joinReqTopic = mqttCfg.topics?.[module]?.sub?.['sdl_join-req']; const joinReqTopic = mqttCfg.topics?.[module]?.sub?.['sdl_join-req'];
const joinAuthzTopic = mqttCfg.topics?.[module]?.pub?.['sdl_join-authz']; const joinAuthzTopic = mqttCfg.topics?.[module]?.pub?.['sdl_join-authz'];
@ -238,10 +168,8 @@ function startJoinHandler() {
log(`${module}: received join request from ${joinReq.host} (${joinReq.sdl_id})`); log(`${module}: received join request from ${joinReq.host} (${joinReq.sdl_id})`);
// Validate version
const workerData = joinReq.msg['sdl-wkr']; const workerData = joinReq.msg['sdl-wkr'];
const workerVersion = workerData?.sdl_version; const workerVersion = workerData?.sdl_version;
// const workerVersion = joinReq.msg?.worker?.version;
const clusterVersion = config.package.version; const clusterVersion = config.package.version;
let authorized = false; let authorized = false;
@ -249,13 +177,7 @@ function startJoinHandler() {
if (workerVersion === clusterVersion) { if (workerVersion === clusterVersion) {
authorized = true; authorized = true;
log(`${module}: worker ${joinReq.host} authorized (version ${workerVersion})`);
// Add worker to cluster state
clusterState = addWorker(clusterState, workerData);
clusterState = computeStats(clusterState);
saveClusterState(config, clusterState);
log(`${module}: worker ${joinReq.host} authorized and added to cluster`);
} else { } else {
authorized = false; authorized = false;
reason = 'version_mismatch'; reason = 'version_mismatch';
@ -271,7 +193,8 @@ function startJoinHandler() {
type: 'join-authz', type: 'join-authz',
msg: { msg: {
sdl_id: joinReq.sdl_id, sdl_id: joinReq.sdl_id,
authorized: authorized authorized: authorized,
reason: reason
} }
}; };
@ -282,8 +205,6 @@ function startJoinHandler() {
err => { err => {
if (err) { if (err) {
log(`${module}: failed to publish join authz: ${err}`); log(`${module}: failed to publish join authz: ${err}`);
} else {
// log(`${module}: published join authz to ${joinAuthzTopic}`);
} }
} }
); );
@ -298,8 +219,6 @@ function startJoinHandler() {
}); });
} }
function startSDLStatusPub() { function startSDLStatusPub() {
const sdlCfg = config.modules[module]; const sdlCfg = config.modules[module];
const mqttCfg = config.modules.mqtt; const mqttCfg = config.modules.mqtt;
@ -322,60 +241,49 @@ function startSDLStatusPub() {
const statusTopic = mqttCfg.topics?.[module]?.pub?.['sdl_cluster-status']; const statusTopic = mqttCfg.topics?.[module]?.pub?.['sdl_cluster-status'];
if (!statusTopic) { if (!statusTopic) {
log(`${module}: ${statusTopic} not configured`); log(`${module}: cluster-status topic not configured`);
return; return;
} }
// This is ok because we are running on localhost
const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`; const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`;
const client = mqtt.connect(mqttUrl); const client = mqtt.connect(mqttUrl);
client.on('connect', () => { client.on('connect', () => {
log(`${module}: connected to MQTT at ${mqttUrl}`); log(`${module}: status publisher connected to MQTT at ${mqttUrl}`);
publishStatus(client, statusTopic); publishStatus(client, statusTopic);
setInterval(() => publishStatus(client, statusTopic), statusInterval); setInterval(() => publishStatus(client, statusTopic), statusInterval);
}); });
client.on('error', err => { client.on('error', err => {
log(`${module}: MQTT error: ${err}`); log(`${module}: status publisher MQTT error: ${err}`);
}); });
} }
function publishStatus(client, topic) { function publishStatus(client, topic) {
// Load current cluster state // Compute stats from in-memory workers
const clusterState = loadClusterState(config); const stats = computeStats();
// Recompute stats to get latest resource totals // Get IP for update command
const updatedState = computeStats(clusterState);
//get ip addr
const interfaces = os.networkInterfaces(); const interfaces = os.networkInterfaces();
let ip_addr = null; let ip_addr = null;
let web_port = config.modules.web.port;
for (const [iface, addrs] of Object.entries(interfaces)) { for (const [iface, addrs] of Object.entries(interfaces)) {
for (const addr of addrs) { for (const addr of addrs) {
// --- FILTERS ---
// IPv4 only
if (addr.family !== 'IPv4') continue; if (addr.family !== 'IPv4') continue;
// Skip loopback
if (addr.internal === true) continue; if (addr.internal === true) continue;
// Skip /32 networks (no broadcast: e.g. Tailscale)
const cidr = Number(addr.cidr?.split('/')[1]); const cidr = Number(addr.cidr?.split('/')[1]);
if (!Number.isInteger(cidr) || cidr >= 32) continue; if (!Number.isInteger(cidr) || cidr >= 32) continue;
// Compute broadcast address ip_addr = addr.address;
const broadcastAddr = computeBroadcast(addr.address, addr.netmask); break;
if (!broadcastAddr) continue;
ip_addr = addr.address
} }
if (ip_addr) break;
} }
const web_port = config.modules.web.port;
const status = { const status = {
ts: new Date().toISOString(), ts: new Date().toISOString(),
sdl_id: config.identity.sdl_id, sdl_id: config.identity.sdl_id,
@ -387,11 +295,13 @@ function publishStatus(client, topic) {
version: config.package.version, version: config.package.version,
uptime_secs: getUptimeSec(), uptime_secs: getUptimeSec(),
uptime: getUptimeDHMS(), uptime: getUptimeDHMS(),
update_cmd: `curl -s http://${ip_addr}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${ip_addr}:${web_port}`, update_cmd: ip_addr
? `curl -s http://${ip_addr}:${web_port}/dist/install-sdl-wkr.sh | bash -s ${ip_addr}:${web_port}`
: null
}, },
cluster: config.cluster, cluster: config.cluster,
resources: updatedState.meta.stats.resources, // ✅ Add cluster resources resources: stats.resources,
workers: updatedState.meta.stats.workers, // ✅ Add worker counts workers: stats.workers,
modules: Object.fromEntries( modules: Object.fromEntries(
Object.entries(config.modules).map(([name, mod]) => [ Object.entries(config.modules).map(([name, mod]) => [
name, name,
@ -408,15 +318,12 @@ function publishStatus(client, topic) {
err => { err => {
if (err) { if (err) {
log(`${module}: failed to publish status: ${err}`); log(`${module}: failed to publish status: ${err}`);
} else {
// log(`${module}: published status to ${topic}`);
} }
} }
); );
} }
// ✅ Publish workers from in-memory state
// ✅ NEW: Publish individual worker details
function startWorkersPub() { function startWorkersPub() {
const sdlCfg = config.modules[module]; const sdlCfg = config.modules[module];
const mqttCfg = config.modules.mqtt; const mqttCfg = config.modules.mqtt;
@ -459,9 +366,6 @@ function startWorkersPub() {
} }
function publishWorkers(client, topic) { function publishWorkers(client, topic) {
// Load current cluster state
const clusterState = loadClusterState(config);
const workersMsg = { const workersMsg = {
ts: new Date().toISOString(), ts: new Date().toISOString(),
sdl_id: config.identity.sdl_id, sdl_id: config.identity.sdl_id,
@ -469,7 +373,7 @@ function publishWorkers(client, topic) {
host: config.identity.hostname, host: config.identity.hostname,
type: 'cluster-workers', type: 'cluster-workers',
msg: { msg: {
workers: clusterState.workers workers: workersState // ✅ Use in-memory state from telemetry
} }
}; };
@ -480,18 +384,13 @@ function publishWorkers(client, topic) {
err => { err => {
if (err) { if (err) {
log(`${module}: failed to publish workers: ${err}`); log(`${module}: failed to publish workers: ${err}`);
} else {
// log(`${module}: published workers to ${topic}`);
} }
} }
); );
} }
function startUdpBeacon() { function startUdpBeacon() {
let sdlCfg = config.modules['sdl-mgr']; let sdlCfg = config.modules['sdl-mgr'];
// log(`${module}: DEBUG: SDL config: ${JSON.stringify(sdlCfg, null, 2)}`, true);
// log(`${module}: DEBUG: Cluster Conf: ${JSON.stringify(config.cluster, null, 2)}`, true);
sdlCfg.cluster = config.cluster; sdlCfg.cluster = config.cluster;
const mqttCfg = config.modules.mqtt; const mqttCfg = config.modules.mqtt;
@ -523,7 +422,6 @@ function startUdpBeacon() {
log(`${module}: cluster: ${JSON.stringify(cluster)}`); log(`${module}: cluster: ${JSON.stringify(cluster)}`);
if (!config.host?.network) { if (!config.host?.network) {
log(`${module}: no host network metadata available for UDP beacon`); log(`${module}: no host network metadata available for UDP beacon`);
return; return;
@ -543,26 +441,17 @@ function startUdpBeacon() {
setInterval(() => { setInterval(() => {
const interfaces = os.networkInterfaces(); const interfaces = os.networkInterfaces();
// for (const [iface, addrs] of Object.entries(config.host.network)) {
for (const [iface, addrs] of Object.entries(interfaces)) { for (const [iface, addrs] of Object.entries(interfaces)) {
for (const addr of addrs) { for (const addr of addrs) {
// --- FILTERS ---
// IPv4 only
if (addr.family !== 'IPv4') continue; if (addr.family !== 'IPv4') continue;
// Skip loopback
if (addr.internal === true) continue; if (addr.internal === true) continue;
// Skip /32 networks (no broadcast: e.g. Tailscale)
const cidr = Number(addr.cidr?.split('/')[1]); const cidr = Number(addr.cidr?.split('/')[1]);
if (!Number.isInteger(cidr) || cidr >= 32) continue; if (!Number.isInteger(cidr) || cidr >= 32) continue;
// Compute broadcast address
const broadcastAddr = computeBroadcast(addr.address, addr.netmask); const broadcastAddr = computeBroadcast(addr.address, addr.netmask);
if (!broadcastAddr) continue; if (!broadcastAddr) continue;
const beacon = { const beacon = {
ts: new Date().toISOString(), ts: new Date().toISOString(),
sdl_id: config.identity.sdl_id, sdl_id: config.identity.sdl_id,
@ -622,9 +511,8 @@ function startUdpBeacon() {
); );
} }
// ✅ Build workers state from telemetry
// Update startHeartbeatListener to use telemetry topic function startTelemetryListener() {
function startTelemetryListener() { // ✅ Renamed function
const sdlCfg = config.modules[module]; const sdlCfg = config.modules[module];
const mqttCfg = config.modules.mqtt; const mqttCfg = config.modules.mqtt;
@ -638,9 +526,7 @@ function startTelemetryListener() { // ✅ Renamed function
return; return;
} }
let clusterState = loadClusterState(config); const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry'];
const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry']; // ✅ Changed
if (!telemetryTopic) { if (!telemetryTopic) {
log(`${module}: cluster-telemetry topic not configured`); log(`${module}: cluster-telemetry topic not configured`);
return; return;
@ -661,55 +547,53 @@ function startTelemetryListener() { // ✅ Renamed function
}); });
}); });
client.on('message', (topic, message) => { client.on('message', (topic, message) => {
if (topic !== telemetryTopic) return; if (topic !== telemetryTopic) return;
try { try {
const telemetry = JSON.parse(message.toString()); const telemetry = JSON.parse(message.toString());
const sdl_id = telemetry.msg.sdl_id; const sdl_id = telemetry.sdl_id;
const hostname = telemetry.host;
const now = new Date();
// Update worker's last_seen timestamp // ✅ Build/update worker state from telemetry
if (clusterState.workers[sdl_id]) { workersState[sdl_id] = {
clusterState.workers[sdl_id].last_seen = new Date().toISOString(); sdl_id: sdl_id,
clusterState.workers[sdl_id].status = 'active'; hostname: hostname,
role: telemetry.role,
last_seen: now.toISOString(), // ✅ ISO string
last_seen_utime: Math.floor(now.getTime() / 1000), // ✅ Unix timestamp (seconds)
// ✅ MERGE resources instead of replacing // System info
if (telemetry.msg.resources) { platform: telemetry.msg?.system?.platform,
// Merge each resource type arch: telemetry.msg?.system?.arch,
if (telemetry.msg.resources.cpus) { distro: telemetry.msg?.system?.distro,
clusterState.workers[sdl_id].resources.cpus = { hardware: telemetry.msg?.system?.hardware,
...clusterState.workers[sdl_id].resources.cpus,
...telemetry.msg.resources.cpus
};
}
if (telemetry.msg.resources.memory) {
clusterState.workers[sdl_id].resources.memory = {
...clusterState.workers[sdl_id].resources.memory,
...telemetry.msg.resources.memory
};
}
if (telemetry.msg.resources.gpus) {
clusterState.workers[sdl_id].resources.gpus = {
...clusterState.workers[sdl_id].resources.gpus,
...telemetry.msg.resources.gpus
};
}
}
clusterState = computeStats(clusterState); // Uptime
saveClusterState(config, clusterState); uptime: telemetry.msg?.uptime,
// Resources
resources: telemetry.msg?.resources,
// Usage
usage: telemetry.msg?.usage,
// Load
load: telemetry.msg?.load
};
} catch (err) {
log(`${module}: failed to process telemetry: ${err}`);
} }
} catch (err) { });
log(`${module}: failed to process telemetry: ${err}`);
}
});
client.on('error', err => { client.on('error', err => {
log(`${module}: telemetry listener MQTT error: ${err}`); log(`${module}: telemetry listener MQTT error: ${err}`);
}); });
} }
// Add Stale Worker Detection // ✅ Mark stale workers as inactive
function startStaleWorkerDetection() { function startStaleWorkerDetection() {
const sdlCfg = config.modules[module]; const sdlCfg = config.modules[module];
@ -718,39 +602,32 @@ function startStaleWorkerDetection() {
return; return;
} }
const checkInterval = 10000; // Check every 10 seconds const checkInterval = 10000;
// Try multiple config locations for stale threshold
const staleThreshold = const staleThreshold =
Number.isInteger(sdlCfg.worker_stale_threshold) && sdlCfg.expiration_timeout?.cluster_telemetry ||
sdlCfg.worker_stale_threshold > 0 sdlCfg.worker_stale_threshold ||
? sdlCfg.worker_stale_threshold 30000;
: 30000; // Default 30 seconds
setInterval(() => { setInterval(() => {
let clusterState = loadClusterState(config);
const now = Date.now(); const now = Date.now();
let changed = false;
for (const [sdl_id, worker] of Object.entries(clusterState.workers)) { for (const [sdl_id, worker] of Object.entries(workersState)) {
const lastSeenMs = Date.parse(worker.last_seen); const lastSeenMs = Date.parse(worker.last_seen);
const ageMs = now - lastSeenMs; const ageMs = now - lastSeenMs;
if (ageMs > staleThreshold && worker.status === 'active') { if (ageMs > staleThreshold && worker.status === 'active') {
log(`${module}: worker ${worker.hostname} (${sdl_id}) marked as inactive (last seen ${Math.round(ageMs / 1000)}s ago)`); log(`${module}: worker ${worker.hostname} (${sdl_id}) marked inactive (${Math.round(ageMs / 1000)}s)`);
clusterState.workers[sdl_id].status = 'inactive'; workersState[sdl_id].status = 'inactive';
changed = true;
} }
} }
if (changed) {
clusterState = computeStats(clusterState);
saveClusterState(config, clusterState);
}
}, checkInterval); }, checkInterval);
log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check interval: ${checkInterval}ms)`); log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check: ${checkInterval}ms)`);
} }
// Update entry point // Entry point
startSDLStatusPub(); startSDLStatusPub();
startWorkersPub(); startWorkersPub();
startTelemetryListener(); startTelemetryListener();
@ -758,4 +635,3 @@ startUdpBeacon();
loadClusterState(config); loadClusterState(config);
startJoinHandler(); startJoinHandler();
startStaleWorkerDetection(); startStaleWorkerDetection();

51
sdl-mgr/html/js/dash.js
View File

@ -502,10 +502,11 @@ function renderWorkersTable(msg) {
html += '<tr>'; html += '<tr>';
html += '<th>Hostname</th>'; html += '<th>Hostname</th>';
html += '<th>Status</th>'; html += '<th>Status</th>';
html += '<th>Hardware</th>';
html += '<th><i class="fa fa-microchip"></i> CPU</th>'; html += '<th><i class="fa fa-microchip"></i> CPU</th>';
html += '<th><i class="fa fa-memory"></i> RAM</th>'; html += '<th><i class="fa fa-memory"></i> RAM</th>';
html += '<th><i class="fa fa-dice-d20"></i> GPU</th>'; html += '<th><i class="fa fa-dice-d20"></i> GPU</th>';
html += '<th>Platform</th>'; html += '<th>Uptime</th>';
html += '<th>Last Seen</th>'; html += '<th>Last Seen</th>';
html += '</tr>'; html += '</tr>';
html += '</thead>'; html += '</thead>';
@ -516,36 +517,62 @@ function renderWorkersTable(msg) {
? '<span class="badge bg-success">Active</span>' ? '<span class="badge bg-success">Active</span>'
: '<span class="badge bg-secondary">Inactive</span>'; : '<span class="badge bg-secondary">Inactive</span>';
// Hardware info
const hwType = worker.hardware?.type || 'unknown';
const hwIcon = hwType === 'hw' ? '🖥️' : hwType === 'vm' ? '💠' : '❓';
const hwManuf = worker.hardware?.manufacturer || 'Unknown';
const hwModel = worker.hardware?.model || 'Unknown';
const hardware = `<span title="${hwManuf} - ${hwModel}">${hwIcon} ${hwType === 'vm' ? hwModel : hwManuf}</span>`;
// CPU
const cpuAvail = worker.resources?.cpus?.available || 0; const cpuAvail = worker.resources?.cpus?.available || 0;
const cpuTotal = worker.resources?.cpus?.allocated || 0; const cpuTotal = worker.resources?.cpus?.allocated || 0;
const cpuUsed = worker.resources?.cpus?.used || 0; const cpuUsed = worker.usage?.cpu?.total || 0;
// Memory
const memAvail = worker.resources?.memory?.available || 0; const memAvail = worker.resources?.memory?.available || 0;
const memTotal = worker.resources?.memory?.allocated || 0; const memTotal = worker.resources?.memory?.allocated || 0;
const memUsed = worker.resources?.memory?.used || 0; const memUsed = worker.usage?.memory?.used || 0;
// Convert bytes to GB
const memAvailGB = Math.round(memAvail / (1024 * 1024 * 1024)); const memAvailGB = Math.round(memAvail / (1024 * 1024 * 1024));
const memTotalGB = Math.round(memTotal / (1024 * 1024 * 1024)); const memTotalGB = Math.round(memTotal / (1024 * 1024 * 1024));
const memUsedGB = memUsed > 0 ? Math.round(memUsed / (1024 * 1024 * 1024)) : 0; const memUsedGB = Math.round(memUsed / (1024 * 1024 * 1024));
const memPercent = worker.usage?.memory?.percent_used || 0;
// GPU
const gpuAvail = worker.resources?.gpus?.available || 0; const gpuAvail = worker.resources?.gpus?.available || 0;
const gpuTotal = worker.resources?.gpus?.allocated || 0; const gpuTotal = worker.resources?.gpus?.allocated || 0;
const gpuUsed = worker.resources?.gpus?.used || 0; const gpuData = worker.usage?.gpu;
const gpuUsed = gpuData?.total_utilization || 0;
const gpuMemMB = gpuData?.total_memory_total_mb || 0;
const gpuMemGB = Math.round(gpuMemMB / 1024);
const platform = `${worker.platform || 'unknown'} / ${worker.arch || 'unknown'}`; // Uptime
const uptime = worker.uptime?.proc_dhms || 'N/A';
// Last seen
const lastSeen = worker.last_seen const lastSeen = worker.last_seen
? new Date(worker.last_seen).toLocaleString() ? new Date(worker.last_seen).toLocaleString()
: 'N/A'; : 'N/A';
html += '<tr>'; html += '<tr>';
html += `<td><strong>${worker.hostname}</strong><br><small class="text-muted">${worker.sdl_id}</small></td>`; html += `<td><strong>${worker.hostname}</strong><br><small class="text-muted" style="font-size:0.7em">${worker.sdl_id.substring(0, 8)}...</small></td>`;
html += `<td>${status}</td>`; html += `<td>${status}</td>`;
html += `<td><span class="dash-val">${cpuAvail}</span> / ${cpuTotal}${cpuUsed > 0 ? `<br><small>(${cpuUsed} used)</small>` : ''}</td>`; html += `<td><small>${hardware}</small></td>`;
html += `<td><span class="dash-val">${memAvailGB}</span> / ${memTotalGB} GB${memUsedGB > 0 ? `<br><small>(${memUsedGB} GB used)</small>` : ''}</td>`; html += `<td>
html += `<td><span class="dash-val">${gpuAvail}</span> / ${gpuTotal}${gpuUsed > 0 ? `<br><small>(${gpuUsed} used)</small>` : ''}</td>`; <span class="dash-val">${cpuAvail}</span> / ${cpuTotal}
html += `<td><small>${platform}</small></td>`; ${cpuUsed > 0 ? `<br><small class="text-muted">${cpuUsed}% load</small>` : ''}
</td>`;
html += `<td>
<span class="dash-val">${memAvailGB}</span> / ${memTotalGB} GB
${memPercent > 0 ? `<br><small class="text-muted">${memPercent}% used</small>` : ''}
</td>`;
html += `<td>
<span class="dash-val">${gpuAvail}</span> / ${gpuTotal}
${gpuMemGB > 0 ? `<br><small class="text-muted">(${gpuMemGB} GB)</small>` : ''}
${gpuUsed > 0 ? `<br><small class="text-muted">${gpuUsed}% load</small>` : ''}
</td>`;
html += `<td><small>${uptime}</small></td>`;
html += `<td><small>${lastSeen}</small></td>`; html += `<td><small>${lastSeen}</small></td>`;
html += '</tr>'; html += '</tr>';
} }

2
sdl-mgr/html/md/dash.md
View File

@ -7,7 +7,7 @@
<div id="dash-modules-list"></div> <div id="dash-modules-list"></div>
<div id="dash-modules-age"></div> <div id="dash-modules-age"></div>
</div> </div>
<div class="col-8" id="dash-sdl-wkrs"> <div class="col-10" id="dash-sdl-wkrs">
<h5>Workers</h5> <h5>Workers</h5>
</div> </div>
</div> </div>