diff --git a/README.md b/README.md index 56f9fe7..b563d1e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.5 | +| **Version** | 0.3.6 | | **Date** | 2026-02-01 | ## Overview diff --git a/changelog.json b/changelog.json index bf1e67e..1428b1b 100644 --- a/changelog.json +++ b/changelog.json @@ -1,5 +1,12 @@ { "releases": [ + { + "version": "0.3.6", + "date": "2026-02-01", + "maturity": "ALPHA", + "summary": "Light/Dark Mode, Wkr Install, Cluster Stats", + "notes": "" + }, { "version": "0.3.5", "date": "2026-02-01", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 17bdf31..8c7bb7d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,14 +5,26 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.5 | +| **Version** | 0.3.6 | | **Date** | 2026-02-01 | -## v0.3.5 - 2026-02-01 (ALPHA) +## v0.3.6 - 2026-02-01 (ALPHA) **Summary** Light/Dark Mode, Wkr Install, Cluster Stats +- Adding Cluster Telemetry +- WIP: Worker Display +- Tweaks +- Start v0.3.6 +- CP Updates +- Dash: Total Cluster Stats +- Publishing to cluster/workers +- Light/Dark Mode Toggle +- Added light/dark theme CSS + +## v0.3.5 - 2026-02-01 (ALPHA) + - v0.3.5 - Fixing sdl-wkr install script - Start v0.3.5 diff --git a/docs/DESIGN.md b/docs/DESIGN.md index dccda85..6179362 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -5,7 +5,7 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.5 | +| **Version** | 0.3.6 | | **Date** | 2026-02-01 | ## Overview diff --git a/metadata.json b/metadata.json index 9903564..3ad3fde 100644 --- a/metadata.json +++ b/metadata.json @@ -3,7 +3,7 @@ "name": "sdl", "description": "Software Defined Laboratory", "abbr": "SDL", - "version": "0.3.5", + "version": "0.3.6", "version_date": "2026-02-01", "maturity": "ALPHA", "author": "John Haverlack", diff --git a/sdl-mgr/app/modules/sdl-mgr/index.js b/sdl-mgr/app/modules/sdl-mgr/index.js index 715a07b..fb4f7f7 100644 --- a/sdl-mgr/app/modules/sdl-mgr/index.js +++ b/sdl-mgr/app/modules/sdl-mgr/index.js @@ -621,12 +621,121 @@ function startUdpBeacon() { } +// Update startHeartbeatListener to use telemetry topic +function startTelemetryListener() { // ✅ Renamed function + const sdlCfg = config.modules[module]; + const mqttCfg = config.modules.mqtt; -// ------------------------------- -// Entry point (ESM-safe) -// ------------------------------- + if (!sdlCfg?.enabled) { + log(`${module}: disabled, not listening for telemetry`); + return; + } + + if (!mqttCfg?.enabled) { + log(`${module}: MQTT disabled, cannot listen for telemetry`); + return; + } + + let clusterState = loadClusterState(config); + + const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry']; // ✅ Changed + if (!telemetryTopic) { + log(`${module}: cluster-telemetry topic not configured`); + return; + } + + const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`; + const client = mqtt.connect(mqttUrl); + + client.on('connect', () => { + log(`${module}: telemetry listener connected to MQTT at ${mqttUrl}`); + + client.subscribe(telemetryTopic, { qos: 1 }, err => { + if (err) { + log(`${module}: failed to subscribe to ${telemetryTopic}: ${err}`); + } else { + log(`${module}: subscribed to ${telemetryTopic}`); + } + }); + }); + + client.on('message', (topic, message) => { + if (topic !== telemetryTopic) return; + + try { + const telemetry = JSON.parse(message.toString()); + const sdl_id = telemetry.msg.sdl_id; + + // Update worker's last_seen timestamp + if (clusterState.workers[sdl_id]) { + clusterState.workers[sdl_id].last_seen = new Date().toISOString(); + clusterState.workers[sdl_id].status = 'active'; + + // Update resources if provided + if (telemetry.msg.resources) { + clusterState.workers[sdl_id].resources = telemetry.msg.resources; + } + + clusterState = computeStats(clusterState); + saveClusterState(config, clusterState); + } + } catch (err) { + log(`${module}: failed to process telemetry: ${err}`); + } + }); + + client.on('error', err => { + log(`${module}: telemetry listener MQTT error: ${err}`); + }); +} + +// Add Stale Worker Detection +function startStaleWorkerDetection() { + const sdlCfg = config.modules[module]; + + if (!sdlCfg?.enabled) { + log(`${module}: disabled, not starting stale worker detection`); + return; + } + + const checkInterval = 10000; // Check every 10 seconds + const staleThreshold = + Number.isInteger(sdlCfg.worker_stale_threshold) && + sdlCfg.worker_stale_threshold > 0 + ? sdlCfg.worker_stale_threshold + : 30000; // Default 30 seconds + + setInterval(() => { + let clusterState = loadClusterState(config); + const now = Date.now(); + let changed = false; + + for (const [sdl_id, worker] of Object.entries(clusterState.workers)) { + const lastSeenMs = Date.parse(worker.last_seen); + const ageMs = now - lastSeenMs; + + if (ageMs > staleThreshold && worker.status === 'active') { + log(`${module}: worker ${worker.hostname} (${sdl_id}) marked as inactive (last seen ${Math.round(ageMs / 1000)}s ago)`); + clusterState.workers[sdl_id].status = 'inactive'; + changed = true; + } + } + + if (changed) { + clusterState = computeStats(clusterState); + saveClusterState(config, clusterState); + } + }, checkInterval); + + log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check interval: ${checkInterval}ms)`); +} + +// Update entry point startSDLStatusPub(); -startWorkersPub(); // ✅ Start workers publisher +startWorkersPub(); +startTelemetryListener(); startUdpBeacon(); loadClusterState(config); -startJoinHandler(); \ No newline at end of file +startJoinHandler(); +startStaleWorkerDetection(); + diff --git a/sdl-mgr/app/package.json b/sdl-mgr/app/package.json index 504c250..417c095 100644 --- a/sdl-mgr/app/package.json +++ b/sdl-mgr/app/package.json @@ -1,6 +1,6 @@ { "name": "sdl-mgr", - "version": "0.3.5", + "version": "0.3.6", "version_date": "2026-02-01", "description": "Software Defined Laboratory: Manager (sdl-mgr)", "main": "index.js", diff --git a/sdl-mgr/conf/modules/sdl-mgr.json b/sdl-mgr/conf/modules/sdl-mgr.json index 049e810..d4f7d6d 100644 --- a/sdl-mgr/conf/modules/sdl-mgr.json +++ b/sdl-mgr/conf/modules/sdl-mgr.json @@ -1,6 +1,7 @@ { "enabled": true, "beacon_udp_port":10101, + "worker_stale_threshold": 15000, "update_interval": { "udp_beacon": 2000, "cluster_status": 10000, diff --git a/sdl-mgr/html/conf/nav-sidepanel.json b/sdl-mgr/html/conf/nav-sidepanel.json index e8fa3c0..cd19180 100644 --- a/sdl-mgr/html/conf/nav-sidepanel.json +++ b/sdl-mgr/html/conf/nav-sidepanel.json @@ -20,7 +20,7 @@ "desc": "Workers", "icon": "fa-solid fa-server", "page": "wkrs", - "enabled": true, + "enabled": false, "items": [] }, "proj": { diff --git a/sdl-mgr/html/js/dash.js b/sdl-mgr/html/js/dash.js index e6cc551..caa76a2 100644 --- a/sdl-mgr/html/js/dash.js +++ b/sdl-mgr/html/js/dash.js @@ -13,7 +13,7 @@ let lastSdlStatus = null; let lastSdlTimestamp = null; let statusCheckTimer = null; const STALE_GRACE_MS = 2000; - +let lastWorkers = null; // ------------------------------- // GPU Summary (for dashboard) @@ -44,7 +44,7 @@ if (Array.isArray(config.host.gpu) && config.host.gpu.length > 0) { return ` ${displayName} ${memLabel} @@ -55,7 +55,7 @@ if (Array.isArray(config.host.gpu) && config.host.gpu.length > 0) { // Hostinfo: hostname, cpu, cores, ram, os const hostinfo = { "hostname": '' + config.host.hostname + '', - "sdl_id": '' + config.identity.sdl_id + '', + "sdl_id": '' + config.identity.sdl_id + '', "cpu cores": '' + config.host.cpu.cores_logical + ' ' + config.host.cpu.model + '', // "cores": config.host.cpu.cores_logical, "ram": '' + config.host.memory.total_gb + ' GB ', @@ -70,7 +70,7 @@ for (let intf in config.host.network) { if (intf != 'lo') { for (let addr in config.host.network[intf]) { if (config.host.network[intf][addr].family == 'IPv4') { - hostinfo["net"] += '' + config.host.network[intf][addr].cidr + '
'; + hostinfo["net"] += '' + config.host.network[intf][addr].cidr + '
'; } } } @@ -201,22 +201,32 @@ async function initMqtt() { }); // 5) Register handlers - mqttClient.on('connect', () => { + mqttClient.on('connect', () => { console.log('MQTT connected (dashboard)'); - const topic = mqttConfig.topics['sdl-web'].sub['sdl_cluster-status']; - console.log('Subscribing to:', topic); + const statusTopic = mqttConfig.topics['sdl-web'].sub['sdl_cluster-status']; + const workersTopic = mqttConfig.topics['sdl-web'].sub['sdl_cluster-workers']; + + console.log('Subscribing to:', statusTopic); + console.log('Subscribing to:', workersTopic); - mqttClient.subscribe(topic, { qos: 1 }, err => { + mqttClient.subscribe(statusTopic, { qos: 1 }, err => { if (err) { - console.error('MQTT subscribe error:', err); + console.error('MQTT subscribe error (status):', err); } else { - console.log('Subscribed to', topic); + console.log('Subscribed to', statusTopic); + } + }); + + mqttClient.subscribe(workersTopic, { qos: 1 }, err => { + if (err) { + console.error('MQTT subscribe error (workers):', err); + } else { + console.log('Subscribed to', workersTopic); } }); startStatusAgeMonitor(); - }); mqttClient.on('message', (topic, payload, packet) => { @@ -235,6 +245,9 @@ async function initMqtt() { lastSdlStatus = msg; lastSdlTimestamp = Date.parse(msg.ts); renderModulesTable(msg); + } else if (msg.type === 'cluster-workers') { + lastWorkers = msg; + renderWorkersTable(msg); } } catch (e) { @@ -278,7 +291,7 @@ function renderModulesTable(msg) { sdl_html += ` Uptime (d:h:m:s) - ${msg.msg.sdl.uptime} + ${msg.msg.sdl.uptime} `; @@ -472,3 +485,73 @@ function markAllModulesDown() { } +function renderWorkersTable(msg) { + const workers = msg.msg?.workers || {}; + const workerCount = Object.keys(workers).length; + + let html = `
Workers (${workerCount})
`; + + if (workerCount === 0) { + html += '

No workers connected

'; + document.getElementById('dash-sdl-wkrs').innerHTML = html; + return; + } + + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + html += ''; + + for (const [sdl_id, worker] of Object.entries(workers)) { + const status = worker.status === 'active' + ? 'Active' + : 'Inactive'; + + const cpuAvail = worker.resources?.cpus?.available || 0; + const cpuTotal = worker.resources?.cpus?.allocated || 0; + const cpuUsed = worker.resources?.cpus?.used || 0; + + const memAvail = worker.resources?.memory?.available || 0; + const memTotal = worker.resources?.memory?.allocated || 0; + const memUsed = worker.resources?.memory?.used || 0; + + // Convert bytes to GB + const memAvailGB = Math.round(memAvail / (1024 * 1024 * 1024)); + const memTotalGB = Math.round(memTotal / (1024 * 1024 * 1024)); + const memUsedGB = memUsed > 0 ? Math.round(memUsed / (1024 * 1024 * 1024)) : 0; + + const gpuAvail = worker.resources?.gpus?.available || 0; + const gpuTotal = worker.resources?.gpus?.allocated || 0; + const gpuUsed = worker.resources?.gpus?.used || 0; + + const platform = `${worker.platform || 'unknown'} / ${worker.arch || 'unknown'}`; + + const lastSeen = worker.last_seen + ? new Date(worker.last_seen).toLocaleString() + : 'N/A'; + + html += ''; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ``; + html += ''; + } + + html += ''; + html += '
HostnameStatus CPU RAM GPUPlatformLast Seen
${worker.hostname}
${worker.sdl_id}		${status}${cpuAvail} / ${cpuTotal}${cpuUsed > 0 ? `
(${cpuUsed} used)` : ''}		${memAvailGB} / ${memTotalGB} GB${memUsedGB > 0 ? `
(${memUsedGB} GB used)` : ''}		${gpuAvail} / ${gpuTotal}${gpuUsed > 0 ? `
(${gpuUsed} used)` : ''}		${platform}${lastSeen}
'; + + document.getElementById('dash-sdl-wkrs').innerHTML = html; +} diff --git a/sdl-mgr/html/md/changelog.md b/sdl-mgr/html/md/changelog.md index 17bdf31..8c7bb7d 100644 --- a/sdl-mgr/html/md/changelog.md +++ b/sdl-mgr/html/md/changelog.md @@ -5,14 +5,26 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.5 | +| **Version** | 0.3.6 | | **Date** | 2026-02-01 | -## v0.3.5 - 2026-02-01 (ALPHA) +## v0.3.6 - 2026-02-01 (ALPHA) **Summary** Light/Dark Mode, Wkr Install, Cluster Stats +- Adding Cluster Telemetry +- WIP: Worker Display +- Tweaks +- Start v0.3.6 +- CP Updates +- Dash: Total Cluster Stats +- Publishing to cluster/workers +- Light/Dark Mode Toggle +- Added light/dark theme CSS + +## v0.3.5 - 2026-02-01 (ALPHA) + - v0.3.5 - Fixing sdl-wkr install script - Start v0.3.5 diff --git a/sdl-wkr/app/modules/sdl-wkr/index.js b/sdl-wkr/app/modules/sdl-wkr/index.js index dab2045..94d47bd 100644 --- a/sdl-wkr/app/modules/sdl-wkr/index.js +++ b/sdl-wkr/app/modules/sdl-wkr/index.js @@ -18,6 +18,7 @@ let authorized = false; let mqttClient = null; let clusterConfig = null; let updating = false; +let telemetryTimer = null; // ------------------------------- // Start UDP discovery @@ -144,7 +145,8 @@ function joinCluster(mqttInfo, cluster) { distro_name: config.host.os.name, distro_version: config.host.os.version, cpus: os.cpus().length, - totalmem: os.totalmem() + totalmem: os.totalmem(), + gpus: Array.isArray(config.host.gpu) ? config.host.gpu.length : 0 // ✅ Fixed } } }; @@ -176,22 +178,21 @@ function joinCluster(mqttInfo, cluster) { // Handle MQTT messages // ------------------------------- function handleMqttMessage(topic, message) { - try { const payload = JSON.parse(message.toString()); const mqttTopics = clusterConfig.modules.mqtt.topics[module]; - if (topic === mqttTopics.sub['sdl_join-authz']) { - handleJoinAuthz(payload); - return; - } + if (topic === mqttTopics.sub['sdl_join-authz']) { + handleJoinAuthz(payload); + return; + } - if ( - authorized && - topic === mqttTopics.sub['sdl_cluster-status'] - ) { - handleClusterStatus(payload); - } + if ( + authorized && + topic === mqttTopics.sub['sdl_cluster-status'] + ) { + handleClusterStatus(payload); + } } catch (err) { log(`${module}: failed to parse MQTT message: ${err}`); @@ -221,11 +222,17 @@ function handleJoinAuthz(payload) { log(`${module}: subscribed to ${statusTopic}`); } }); + + // ✅ Start telemetry after authorization + startTelemetry(); } else { log(`${module}: join denied: ${payload.msg?.reason || 'unknown'}`); } } +// ------------------------------- +// Handle cluster status +// ------------------------------- function handleClusterStatus(payload) { const clusterVersion = payload.msg?.sdl?.version; const updateCmd = payload.msg?.sdl?.update_cmd; @@ -241,7 +248,9 @@ function handleClusterStatus(payload) { } } - +// ------------------------------- +// Trigger worker update +// ------------------------------- function triggerWorkerUpdate(targetVersion, updateCmd) { if (updating) return; updating = true; @@ -252,6 +261,9 @@ function triggerWorkerUpdate(targetVersion, updateCmd) { // Stop reacting to further control messages authorized = false; + + // ✅ Stop telemetry during update + stopTelemetry(); try { exec(updateCmd, { stdio: 'inherit' }); @@ -261,7 +273,93 @@ function triggerWorkerUpdate(targetVersion, updateCmd) { } } +// ------------------------------- +// ✅ Worker Telemetry +// ------------------------------- +function startTelemetry() { + if (telemetryTimer) { + clearInterval(telemetryTimer); + } + const telemetryInterval = + Number.isInteger(config.modules[module].update_interval?.worker_telemetry) && + config.modules[module].update_interval.worker_telemetry > 0 + ? config.modules[module].update_interval.worker_telemetry + : 5000; // Default 5 seconds + + const mqttTopics = clusterConfig.modules.mqtt.topics[module]; + const telemetryTopic = mqttTopics.pub['sdl_cluster-telemetry']; + + if (!telemetryTopic) { + log(`${module}: cluster-telemetry topic not configured`); + return; + } + + log(`${module}: starting telemetry (interval: ${telemetryInterval}ms)`); + + // Publish immediately + publishTelemetry(telemetryTopic); + + // Then publish periodically + telemetryTimer = setInterval(() => { + publishTelemetry(telemetryTopic); + }, telemetryInterval); +} + +function publishTelemetry(topic) { + if (!mqttClient || !authorized) return; + + const gpuCount = Array.isArray(config.host.gpu) ? config.host.gpu.length : 0; // ✅ Fixed + + const telemetry = { + ts: new Date().toISOString(), + sdl_id: config.identity.sdl_id, + role: 'sdl-wkr', + host: config.identity.hostname, + type: 'worker-telemetry', + msg: { + sdl_id: config.identity.sdl_id, + hostname: config.identity.hostname, + status: 'active', + resources: { + cpus: { + allocated: config.host.cpu.cores_logical, + available: config.host.cpu.cores_logical, + used: 0 + }, + memory: { + allocated: config.host.memory.total_bytes, + available: config.host.memory.total_bytes, + used: 0 + }, + gpus: { + allocated: gpuCount, + available: gpuCount, + used: 0 + } + } + } + }; + + mqttClient.publish( + topic, + JSON.stringify(telemetry), + { qos: 1 }, + err => { + if (err) { + log(`${module}: failed to publish telemetry: ${err}`); + } + } + ); +} + +function stopTelemetry() { + if (telemetryTimer) { + clearInterval(telemetryTimer); + telemetryTimer = null; + log(`${module}: telemetry stopped`); + } +} // ------------------------------- // Entry point diff --git a/sdl-wkr/app/package.json b/sdl-wkr/app/package.json index d2cf6ea..5679dbf 100644 --- a/sdl-wkr/app/package.json +++ b/sdl-wkr/app/package.json @@ -1,7 +1,7 @@ { "name": "sdl-wkr", "description": "Software Defined Laboratory: Worker (sdl-wkr)", - "version": "0.3.5", + "version": "0.3.6", "version_date": "2026-02-01", "main": "index.js", "type": "module",