Release v0.3.6

README.md

@@ -5,7 +5,7 @@
 | **Author** | John Haverlack |
 | **Copyright** | 2026 John Haverlack |
 | **License** | MIT |
-| **Version** | 0.3.5 |
+| **Version** | 0.3.6 |
 | **Date** | 2026-02-01 |
 
 ## Overview

changelog.json

@@ -1,5 +1,12 @@
 {
   "releases": [
+    {
+      "version": "0.3.6",
+      "date": "2026-02-01",
+      "maturity": "ALPHA",
+      "summary": "Light/Dark Mode, Wkr Install, Cluster Stats",
+      "notes": ""
+    },
     {
       "version": "0.3.5",
       "date": "2026-02-01",

docs/CHANGELOG.md

@@ -5,14 +5,26 @@
 | **Author** | John Haverlack |
 | **Copyright** | 2026 John Haverlack |
 | **License** | MIT |
-| **Version** | 0.3.5 |
+| **Version** | 0.3.6 |
 | **Date** | 2026-02-01 |
 
-## v0.3.5 - 2026-02-01 (ALPHA)
+## v0.3.6 - 2026-02-01 (ALPHA)
 
 **Summary**
 Light/Dark Mode, Wkr Install, Cluster Stats
 
+- Adding Cluster Telemetry
+- WIP: Worker Display
+- Tweaks
+- Start v0.3.6
+- CP Updates
+- Dash: Total Cluster Stats
+- Publishing to cluster/workers
+- Light/Dark Mode Toggle
+- Added light/dark theme CSS
+
+## v0.3.5 - 2026-02-01 (ALPHA)
+
 - v0.3.5
 - Fixing sdl-wkr install script
 - Start v0.3.5

docs/DESIGN.md

@@ -5,7 +5,7 @@
 | **Author** | John Haverlack |
 | **Copyright** | 2026 John Haverlack |
 | **License** | MIT |
-| **Version** | 0.3.5 |
+| **Version** | 0.3.6 |
 | **Date** | 2026-02-01 |
 
 ## Overview

metadata.json

@@ -3,7 +3,7 @@
     "name": "sdl",
     "description": "Software Defined Laboratory",
     "abbr": "SDL",
-    "version": "0.3.5",
+    "version": "0.3.6",
     "version_date": "2026-02-01",
     "maturity": "ALPHA",
     "author": "John Haverlack",

sdl-mgr/app/modules/sdl-mgr/index.js

@@ -621,12 +621,121 @@ function startUdpBeacon() {
 }
 
 
+// Update startHeartbeatListener to use telemetry topic
+function startTelemetryListener() {  // ✅ Renamed function
+  const sdlCfg = config.modules[module];
+  const mqttCfg = config.modules.mqtt;
 
-// -------------------------------
+  if (!sdlCfg?.enabled) {
-// Entry point (ESM-safe)
+    log(`${module}: disabled, not listening for telemetry`);
-// -------------------------------
+    return;
+  }
+
+  if (!mqttCfg?.enabled) {
+    log(`${module}: MQTT disabled, cannot listen for telemetry`);
+    return;
+  }
+
+  let clusterState = loadClusterState(config);
+
+  const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry'];  // ✅ Changed
+  if (!telemetryTopic) {
+    log(`${module}: cluster-telemetry topic not configured`);
+    return;
+  }
+
+  const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`;
+  const client = mqtt.connect(mqttUrl);
+
+  client.on('connect', () => {
+    log(`${module}: telemetry listener connected to MQTT at ${mqttUrl}`);
+
+    client.subscribe(telemetryTopic, { qos: 1 }, err => {
+      if (err) {
+        log(`${module}: failed to subscribe to ${telemetryTopic}: ${err}`);
+      } else {
+        log(`${module}: subscribed to ${telemetryTopic}`);
+      }
+    });
+  });
+
+  client.on('message', (topic, message) => {
+    if (topic !== telemetryTopic) return;
+
+    try {
+      const telemetry = JSON.parse(message.toString());
+      const sdl_id = telemetry.msg.sdl_id;
+
+      // Update worker's last_seen timestamp
+      if (clusterState.workers[sdl_id]) {
+        clusterState.workers[sdl_id].last_seen = new Date().toISOString();
+        clusterState.workers[sdl_id].status = 'active';
+        
+        // Update resources if provided
+        if (telemetry.msg.resources) {
+          clusterState.workers[sdl_id].resources = telemetry.msg.resources;
+        }
+
+        clusterState = computeStats(clusterState);
+        saveClusterState(config, clusterState);
+      }
+    } catch (err) {
+      log(`${module}: failed to process telemetry: ${err}`);
+    }
+  });
+
+  client.on('error', err => {
+    log(`${module}: telemetry listener MQTT error: ${err}`);
+  });
+}
+
+// Add Stale Worker Detection
+function startStaleWorkerDetection() {
+  const sdlCfg = config.modules[module];
+
+  if (!sdlCfg?.enabled) {
+    log(`${module}: disabled, not starting stale worker detection`);
+    return;
+  }
+
+  const checkInterval = 10000;  // Check every 10 seconds
+  const staleThreshold =
+    Number.isInteger(sdlCfg.worker_stale_threshold) &&
+    sdlCfg.worker_stale_threshold > 0
+      ? sdlCfg.worker_stale_threshold
+      : 30000;  // Default 30 seconds
+
+  setInterval(() => {
+    let clusterState = loadClusterState(config);
+    const now = Date.now();
+    let changed = false;
+
+    for (const [sdl_id, worker] of Object.entries(clusterState.workers)) {
+      const lastSeenMs = Date.parse(worker.last_seen);
+      const ageMs = now - lastSeenMs;
+
+      if (ageMs > staleThreshold && worker.status === 'active') {
+        log(`${module}: worker ${worker.hostname} (${sdl_id}) marked as inactive (last seen ${Math.round(ageMs / 1000)}s ago)`);
+        clusterState.workers[sdl_id].status = 'inactive';
+        changed = true;
+      }
+    }
+
+    if (changed) {
+      clusterState = computeStats(clusterState);
+      saveClusterState(config, clusterState);
+    }
+  }, checkInterval);
+
+  log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check interval: ${checkInterval}ms)`);
+}
+
+// Update entry point
 startSDLStatusPub();
-startWorkersPub();  // ✅ Start workers publisher
+startWorkersPub();
+startTelemetryListener(); 
 startUdpBeacon();
 loadClusterState(config);
 startJoinHandler();
+startStaleWorkerDetection();
+

sdl-mgr/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sdl-mgr",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "version_date": "2026-02-01",
   "description": "Software Defined Laboratory: Manager (sdl-mgr)",
   "main": "index.js",

sdl-mgr/conf/modules/sdl-mgr.json

@@ -1,6 +1,7 @@
 {
     "enabled": true,
     "beacon_udp_port":10101,
+    "worker_stale_threshold": 15000,
     "update_interval": {
         "udp_beacon": 2000,
         "cluster_status": 10000,

sdl-mgr/html/conf/nav-sidepanel.json

@@ -20,7 +20,7 @@
     "desc": "Workers",
     "icon": "fa-solid fa-server",
     "page": "wkrs",
-    "enabled": true,
+    "enabled": false,
     "items": []
   },
   "proj": {

sdl-mgr/html/js/dash.js

@@ -13,7 +13,7 @@ let lastSdlStatus = null;
 let lastSdlTimestamp = null;
 let statusCheckTimer = null;
 const STALE_GRACE_MS = 2000;
-
+let lastWorkers = null;
 
 // -------------------------------
 // GPU Summary (for dashboard)
@@ -44,7 +44,7 @@ if (Array.isArray(config.host.gpu) && config.host.gpu.length > 0) {
 
     return `
       <span
-        style="font-size:0.75em; cursor:help"
+        style="font-size:0.8em; cursor:help"
         title="${fullName}">
         ${displayName} ${memLabel}
       </span>
@@ -55,7 +55,7 @@ if (Array.isArray(config.host.gpu) && config.host.gpu.length > 0) {
 // Hostinfo: hostname, cpu, cores, ram, os
 const hostinfo = {
     "hostname": '<span class="dash-val">' + config.host.hostname + '</span>',
-    "sdl_id": '<span class="dash-val" style=" font-size:0.7em">' + config.identity.sdl_id + '</span>',
+    "sdl_id": '<span class="dash-val" style=" font-size:0.8em">' + config.identity.sdl_id + '</span>',
     "cpu cores": '<span class="dash-val">' + config.host.cpu.cores_logical + '</span> <span style="font-size:0.7em">' + config.host.cpu.model + '</span>',
     // "cores": config.host.cpu.cores_logical,
     "ram": '<span class="dash-val">' + config.host.memory.total_gb + '</span> GB ',
@@ -70,7 +70,7 @@ for (let intf in config.host.network) {
     if (intf != 'lo') {
       for (let addr in config.host.network[intf]) {
         if (config.host.network[intf][addr].family == 'IPv4') {
-          hostinfo["net"] += '<span class="dash-val" style=" font-size: 0.8em">' + config.host.network[intf][addr].cidr + '</span><br> ';
+          hostinfo["net"] += '<span class="dash-val" style=" font-size: 0.9em">' + config.host.network[intf][addr].cidr + '</span><br> ';
         }
       }
     }    
@@ -201,22 +201,32 @@ async function initMqtt() {
   });
 
   // 5) Register handlers
-  mqttClient.on('connect', () => {
+    mqttClient.on('connect', () => {
     console.log('MQTT connected (dashboard)');
 
-    const topic = mqttConfig.topics['sdl-web'].sub['sdl_cluster-status'];
+    const statusTopic = mqttConfig.topics['sdl-web'].sub['sdl_cluster-status'];
-    console.log('Subscribing to:', topic);
+    const workersTopic = mqttConfig.topics['sdl-web'].sub['sdl_cluster-workers'];
     
-    mqttClient.subscribe(topic, { qos: 1 }, err => {
+    console.log('Subscribing to:', statusTopic);
+    console.log('Subscribing to:', workersTopic);
+
+    mqttClient.subscribe(statusTopic, { qos: 1 }, err => {
       if (err) {
-        console.error('MQTT subscribe error:', err);
+        console.error('MQTT subscribe error (status):', err);
       } else {
-        console.log('Subscribed to', topic);
+        console.log('Subscribed to', statusTopic);
+      }
+    });
+
+    mqttClient.subscribe(workersTopic, { qos: 1 }, err => {
+      if (err) {
+        console.error('MQTT subscribe error (workers):', err);
+      } else {
+        console.log('Subscribed to', workersTopic);
       }
     });
 
     startStatusAgeMonitor();
-
   });
 
   mqttClient.on('message', (topic, payload, packet) => {
@@ -235,6 +245,9 @@ async function initMqtt() {
         lastSdlStatus = msg;
         lastSdlTimestamp = Date.parse(msg.ts);
         renderModulesTable(msg);
+      } else if (msg.type === 'cluster-workers') {
+        lastWorkers = msg;
+        renderWorkersTable(msg);
       }
 
     } catch (e) {
@@ -278,7 +291,7 @@ function renderModulesTable(msg) {
   sdl_html += `
       <tr>
         <th>Uptime (d:h:m:s)</th>
-        <td class="dash-val" style=" font-size:0.8em">${msg.msg.sdl.uptime}</td>
+        <td class="dash-val" style=" font-size:1em">${msg.msg.sdl.uptime}</td>
       </tr>
     `;
 
@@ -472,3 +485,73 @@ function markAllModulesDown() {
 }
 
 
+function renderWorkersTable(msg) {
+  const workers = msg.msg?.workers || {};
+  const workerCount = Object.keys(workers).length;
+
+  let html = `<h5>Workers (${workerCount})</h5>`;
+  
+  if (workerCount === 0) {
+    html += '<p class="text-muted">No workers connected</p>';
+    document.getElementById('dash-sdl-wkrs').innerHTML = html;
+    return;
+  }
+
+  html += '<table class="table table-sm table-striped table-hover">';
+  html += '<thead>';
+  html += '<tr>';
+  html += '<th>Hostname</th>';
+  html += '<th>Status</th>';
+  html += '<th><i class="fa fa-microchip"></i> CPU</th>';
+  html += '<th><i class="fa fa-memory"></i> RAM</th>';
+  html += '<th><i class="fa fa-dice-d20"></i> GPU</th>';
+  html += '<th>Platform</th>';
+  html += '<th>Last Seen</th>';
+  html += '</tr>';
+  html += '</thead>';
+  html += '<tbody>';
+
+  for (const [sdl_id, worker] of Object.entries(workers)) {
+    const status = worker.status === 'active'
+      ? '<span class="badge bg-success">Active</span>'
+      : '<span class="badge bg-secondary">Inactive</span>';
+
+    const cpuAvail = worker.resources?.cpus?.available || 0;
+    const cpuTotal = worker.resources?.cpus?.allocated || 0;
+    const cpuUsed = worker.resources?.cpus?.used || 0;
+
+    const memAvail = worker.resources?.memory?.available || 0;
+    const memTotal = worker.resources?.memory?.allocated || 0;
+    const memUsed = worker.resources?.memory?.used || 0;
+    
+    // Convert bytes to GB
+    const memAvailGB = Math.round(memAvail / (1024 * 1024 * 1024));
+    const memTotalGB = Math.round(memTotal / (1024 * 1024 * 1024));
+    const memUsedGB = memUsed > 0 ? Math.round(memUsed / (1024 * 1024 * 1024)) : 0;
+
+    const gpuAvail = worker.resources?.gpus?.available || 0;
+    const gpuTotal = worker.resources?.gpus?.allocated || 0;
+    const gpuUsed = worker.resources?.gpus?.used || 0;
+
+    const platform = `${worker.platform || 'unknown'} / ${worker.arch || 'unknown'}`;
+    
+    const lastSeen = worker.last_seen 
+      ? new Date(worker.last_seen).toLocaleString()
+      : 'N/A';
+
+    html += '<tr>';
+    html += `<td><strong>${worker.hostname}</strong><br><small class="text-muted">${worker.sdl_id}</small></td>`;
+    html += `<td>${status}</td>`;
+    html += `<td><span class="dash-val">${cpuAvail}</span> / ${cpuTotal}${cpuUsed > 0 ? `<br><small>(${cpuUsed} used)</small>` : ''}</td>`;
+    html += `<td><span class="dash-val">${memAvailGB}</span> / ${memTotalGB} GB${memUsedGB > 0 ? `<br><small>(${memUsedGB} GB used)</small>` : ''}</td>`;
+    html += `<td><span class="dash-val">${gpuAvail}</span> / ${gpuTotal}${gpuUsed > 0 ? `<br><small>(${gpuUsed} used)</small>` : ''}</td>`;
+    html += `<td><small>${platform}</small></td>`;
+    html += `<td><small>${lastSeen}</small></td>`;
+    html += '</tr>';
+  }
+
+  html += '</tbody>';
+  html += '</table>';
+
+  document.getElementById('dash-sdl-wkrs').innerHTML = html;
+}

sdl-mgr/html/md/changelog.md

@@ -5,14 +5,26 @@
 | **Author** | John Haverlack |
 | **Copyright** | 2026 John Haverlack |
 | **License** | MIT |
-| **Version** | 0.3.5 |
+| **Version** | 0.3.6 |
 | **Date** | 2026-02-01 |
 
-## v0.3.5 - 2026-02-01 (ALPHA)
+## v0.3.6 - 2026-02-01 (ALPHA)
 
 **Summary**
 Light/Dark Mode, Wkr Install, Cluster Stats
 
+- Adding Cluster Telemetry
+- WIP: Worker Display
+- Tweaks
+- Start v0.3.6
+- CP Updates
+- Dash: Total Cluster Stats
+- Publishing to cluster/workers
+- Light/Dark Mode Toggle
+- Added light/dark theme CSS
+
+## v0.3.5 - 2026-02-01 (ALPHA)
+
 - v0.3.5
 - Fixing sdl-wkr install script
 - Start v0.3.5

sdl-wkr/app/modules/sdl-wkr/index.js

@@ -18,6 +18,7 @@ let authorized = false;
 let mqttClient = null;
 let clusterConfig = null;
 let updating = false;
+let telemetryTimer = null; 
 
 // -------------------------------
 // Start UDP discovery
@@ -144,7 +145,8 @@ function joinCluster(mqttInfo, cluster) {
           distro_name: config.host.os.name,
           distro_version: config.host.os.version,
           cpus: os.cpus().length,
-          totalmem: os.totalmem()
+          totalmem: os.totalmem(),
+          gpus: Array.isArray(config.host.gpu) ? config.host.gpu.length : 0  // ✅ Fixed
         }
       }
     };
@@ -176,22 +178,21 @@ function joinCluster(mqttInfo, cluster) {
 // Handle MQTT messages
 // -------------------------------
 function handleMqttMessage(topic, message) {
-
   try {
     const payload = JSON.parse(message.toString());
     const mqttTopics = clusterConfig.modules.mqtt.topics[module];
     
-  if (topic === mqttTopics.sub['sdl_join-authz']) {
+    if (topic === mqttTopics.sub['sdl_join-authz']) {
-    handleJoinAuthz(payload);
+      handleJoinAuthz(payload);
-    return;
+      return;
-  }
+    }
 
-  if (
+    if (
-    authorized &&
+      authorized &&
-    topic === mqttTopics.sub['sdl_cluster-status']
+      topic === mqttTopics.sub['sdl_cluster-status']
-  ) {
+    ) {
-    handleClusterStatus(payload);
+      handleClusterStatus(payload);
-  }
+    }
 
   } catch (err) {
     log(`${module}: failed to parse MQTT message: ${err}`);
@@ -221,11 +222,17 @@ function handleJoinAuthz(payload) {
         log(`${module}: subscribed to ${statusTopic}`);
       }
     });
+
+    // ✅ Start telemetry after authorization
+    startTelemetry();
   } else {
     log(`${module}: join denied: ${payload.msg?.reason || 'unknown'}`);
   }
 }
 
+// -------------------------------
+// Handle cluster status
+// -------------------------------
 function handleClusterStatus(payload) {
   const clusterVersion = payload.msg?.sdl?.version;
   const updateCmd = payload.msg?.sdl?.update_cmd;
@@ -241,7 +248,9 @@ function handleClusterStatus(payload) {
   }
 }
 
-
+// -------------------------------
+// Trigger worker update
+// -------------------------------
 function triggerWorkerUpdate(targetVersion, updateCmd) {
   if (updating) return;
   updating = true;
@@ -253,6 +262,9 @@ function triggerWorkerUpdate(targetVersion, updateCmd) {
   // Stop reacting to further control messages
   authorized = false;
   
+  // ✅ Stop telemetry during update
+  stopTelemetry();
+
   try {
     exec(updateCmd, { stdio: 'inherit' });
   } catch (err) {
@@ -261,7 +273,93 @@ function triggerWorkerUpdate(targetVersion, updateCmd) {
   }
 }
 
+// -------------------------------
+// ✅ Worker Telemetry
+// -------------------------------
+function startTelemetry() {
+  if (telemetryTimer) {
+    clearInterval(telemetryTimer);
+  }
 
+  const telemetryInterval =
+    Number.isInteger(config.modules[module].update_interval?.worker_telemetry) &&
+    config.modules[module].update_interval.worker_telemetry > 0
+      ? config.modules[module].update_interval.worker_telemetry
+      : 5000;  // Default 5 seconds
+
+  const mqttTopics = clusterConfig.modules.mqtt.topics[module];
+  const telemetryTopic = mqttTopics.pub['sdl_cluster-telemetry'];
+
+  if (!telemetryTopic) {
+    log(`${module}: cluster-telemetry topic not configured`);
+    return;
+  }
+
+  log(`${module}: starting telemetry (interval: ${telemetryInterval}ms)`);
+
+  // Publish immediately
+  publishTelemetry(telemetryTopic);
+
+  // Then publish periodically
+  telemetryTimer = setInterval(() => {
+    publishTelemetry(telemetryTopic);
+  }, telemetryInterval);
+}
+
+function publishTelemetry(topic) {
+  if (!mqttClient || !authorized) return;
+
+  const gpuCount = Array.isArray(config.host.gpu) ? config.host.gpu.length : 0;  // ✅ Fixed
+
+  const telemetry = {
+    ts: new Date().toISOString(),
+    sdl_id: config.identity.sdl_id,
+    role: 'sdl-wkr',
+    host: config.identity.hostname,
+    type: 'worker-telemetry',
+    msg: {
+      sdl_id: config.identity.sdl_id,
+      hostname: config.identity.hostname,
+      status: 'active',
+      resources: {
+        cpus: {
+          allocated: config.host.cpu.cores_logical,
+          available: config.host.cpu.cores_logical,
+          used: 0
+        },
+        memory: {
+          allocated: config.host.memory.total_bytes,
+          available: config.host.memory.total_bytes,
+          used: 0
+        },
+        gpus: {
+          allocated: gpuCount,
+          available: gpuCount,
+          used: 0
+        }
+      }
+    }
+  };
+
+  mqttClient.publish(
+    topic,
+    JSON.stringify(telemetry),
+    { qos: 1 },
+    err => {
+      if (err) {
+        log(`${module}: failed to publish telemetry: ${err}`);
+      }
+    }
+  );
+}
+
+function stopTelemetry() {
+  if (telemetryTimer) {
+    clearInterval(telemetryTimer);
+    telemetryTimer = null;
+    log(`${module}: telemetry stopped`);
+  }
+}
 
 // -------------------------------
 // Entry point

sdl-wkr/app/package.json

@@ -1,7 +1,7 @@
 {
   "name": "sdl-wkr",
   "description": "Software Defined Laboratory: Worker (sdl-wkr)",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "version_date": "2026-02-01",
   "main": "index.js",
   "type": "module",