diff --git a/.gitignore b/.gitignore index e69de29..7a05565 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +docs/.obsidian \ No newline at end of file diff --git a/README.md b/README.md index f3d0a42..2385611 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.7 | -| **Date** | 2026-02-01 | +| **Version** | 0.3.8 | +| **Date** | 2026-02-08 | ## Overview @@ -14,6 +14,9 @@ The Software Defined Laboratory (SDL) project provides a generalized distributed ## Design +![SDL Dashboard](docs/lib/img/sdl-dash.png) +![SDL About](docs/lib/img/sdl-about.png) + For more information, see the [Design](docs/DESIGN.md) document. ## Getting Started @@ -47,6 +50,12 @@ Listen to UDP 10101 for beacon messages broadcast by your SDL Manager. nc -u -l -k 10101 |jq ``` +Or to get the install command from the beacon message: + +``` +nc -u -l -k 10101 | jq -r '.msg.sdl_wkr_install_cmd[]' +``` + > NOTE: CTRL-C to exit `nc`. You cannot run `nc` on port 10101 in parallel with the sdl-wkr. So run it once to get the install command. Tnen install sdl-wkr. Run the **curl** or **wget** command provided by SDL Manager UDP beacon to install SDL Worker locally. diff --git a/changelog.json b/changelog.json index 9d59a56..caf96f4 100644 --- a/changelog.json +++ b/changelog.json @@ -1,5 +1,12 @@ { "releases": [ + { + "version": "0.3.8", + "date": "2026-02-01", + "maturity": "ALPHA", + "summary": "", + "notes": "" + }, { "version": "0.3.7", "date": "2026-02-01", diff --git a/config.json b/config.json index 135cbe5..7ca7dbf 100644 --- a/config.json +++ b/config.json @@ -2,11 +2,13 @@ "DIRS": { "SDL_HOME": "~/.sdl", "SDL_CONF": "SDL_HOME/conf", + "SDL_DOCS": "SDL_HOME/docs", "SDL_LOGS": "SDL_HOME/logs", "SDL_DIST": "SDL_HOME/dist", "SDL_DATA": "SDL_HOME/data", "SDL_PROJ": "SDL_DATA/proj", "SDL_JOBS": "SDL_DATA/jobs", + "SDL_SCRIPTS": "SDL_HOME/scripts", "SDL_MGR": "SDL_HOME/sdl-mgr", "SDL_WKR": "SDL_HOME/sdl-wkr", "SDL_NODEJS": "SDL_HOME/nodejs" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index dd8144e..e950d88 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,14 +5,19 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.7 | +| **Version** | 0.3.8 | | **Date** | 2026-02-01 | +## v0.3.8 - 2026-02-01 (ALPHA) + +- Start v0.3.8 + ## v0.3.7 - 2026-02-01 (ALPHA) **Summary** sdl-wkr Telemetry +- v0.3.7 - Fixing sdl-wkr RAM detection - Start v0.3.7 diff --git a/docs/DESIGN.md b/docs/DESIGN.md index af32591..df5a246 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -5,8 +5,8 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.7 | -| **Date** | 2026-02-01 | +| **Version** | 0.3.8 | +| **Date** | 2026-02-08 | ## Overview @@ -14,31 +14,118 @@ The Software Defined Laboratory (SDL) project provides a generalized distributed # Roadmap -## Status - -- [x] sdl-mgr MQTT Broker -- [x] sdl-mgr UDP Beacon -- [x] sdl-mgr Web UI MQTT WebSocket Pub/Sub -- [x] sdl-wkr UDP Listener - - ## Prioritized Tasks MVP - Minimum Viable Product Task List +**Core Infrastructure** ✅ (Complete) +- [x] sdl-mgr MQTT Broker +- [x] sdl-mgr UDP Beacon +- [x] sdl-mgr Web UI MQTT WebSocket Pub/Sub +- [x] sdl-wkr UDP Listener - [x] SDL_ID Node ID - [x] Version Updater Script -- [x] Deploy sld-mgr to $SDL_HOME/sdl-mgr +- [x] Deploy sdl-mgr to $SDL_HOME/sdl-mgr - [x] Create SDL Worker Dist Build Process - [x] Create SDL Worker Install to $SDL_HOME - [x] SDL Install Script -- [ ] Create SDL Worker Auto Update Process +- [x] Create SDL Worker Auto Update Process - [x] Create Worker Join Process -- [ ] Create Worker Telementry Process +- [x] Create Worker Telemetry Process + +**Worker Monitoring & Telemetry** 🔄 (In Progress) +- [x] Worker Hardware Inventory Detection + - [x] Physical vs Logical CPU cores (hyperthreading) + - [x] Physical vs VM detection (hypervisor flag) + - [x] GPU VRAM capacity aggregation + - [x] Memory capacity (bytes → GB conversion) +- [x] Worker Real-time Load Status (pure Node.js) + - [x] CPU usage percentage (`os.cpus()` times) + - [x] RAM usage (total/free/used via `os.totalmem()/freemem()`) + - [x] GPU usage (exec-based: nvidia-smi, rocm-smi) + - [ ] Disk IO (optional: exec-based) +- [x] Worker Uptime Tracking + - [x] SDL process uptime (`process.uptime()`) + - [x] OS uptime (`os.uptime()`) +- [ ] Worker Display Improvements + - [x] Show OS distro + version in Platform column (e.g., "Debian 12 / x64") + - [x] Show system type (Physical 🖥️ / VM 💠) + - [ ] Show GPU VRAM in GPU column (e.g., "2 GPUs (48 GB)") + - [x] Show worker uptime in dashboard + - [x] Display last seen as elapsed time in dashboard, not date + - [ ] Track Username for sdl process +- [ ] Web UI + - [ ] Progresive Web App + - [ ] Editable Dashboard + - [x] Panel Icon + + +**Resource Management** +- [ ] CPU Reserve Capacity Allocation + - [ ] Config option: `reserve_cpus: N` (keep N cores for system) + - [ ] Track: `total`, `reserved`, `allocated`, `available`, `used` + - [ ] Display reserved capacity in dashboard +- [ ] Memory Reserve Capacity (optional, future) +- [ ] GPU Reserve Capacity (optional, future) + +**Service Management** +- [x] Start/Stop shell scripts (for systems without systemd user services) + - [x] `start-sdl-mgr.sh` / `stop-sdl-mgr.sh` + - [x] `start-sdl-wkr.sh` / `stop-sdl-wkr.sh` +- [ ] Combined systemd service + - [ ] `sdl.service` to manage both sdl-mgr and sdl-wkr together + - [ ] Support for system-level (sudo) and user-level services +- [ ] SDL-MGR auto-installs SDL-WKR + - [ ] Manager node becomes worker by default + - [ ] Config flag to disable: `install_worker_on_manager: false + - [x] Pass HTTP Port to SDL-WKR Curl installer + - [ ] Update Install Summary + - [ ] Add Quite Mode to sdl-wkr install + + + +**Data & Storage** - [ ] Data Storage Organizational Structure -- [ ] MinIO S3 Storage Server -- [ ] Worker Telemetry Hardware Inventory (CPU, RAM, GPU, etc) -- [ ] Worker Telementry Load Status (CPU, RAM, GPU, Disk IO, etc) + - [ ] Define directory hierarchy for tasks, results, logs + - [ ] Implement file naming conventions + - [ ] Add cleanup/retention policies +- [ ] MinIO S3 Storage Server Integration + - [ ] Install and configure MinIO + - [ ] S3 bucket structure for SDL data + - [ ] Worker access to S3 storage + - [ ] Web UI S3 browser integration + +**Projects** +- [ ] Project Management Org Structure + - [ ] Define project metadata schema + - [ ] Project lifecycle states (draft, active, archived) + - [ ] Permission/access control model +- [ ] Experiment Organization under Org Structure + - [ ] Experiment templates + - [ ] Parameter tracking + - [ ] Result linking to experiments +- [ ] HPC Experiment Binary Development + - [ ] sdl-wkr Capablity detection (phyton, etc) + - [ ] Binary packaging format + - [ ] Dependency management + - [ ] Version control for binaries + +**Dashboard Enhancements** +- [x] Dark/Light theme polish (CSS refinements) +- [ ] Real-time resource graphs (Chart.js) +- [ ] Worker status history timeline +- [ ] Cluster health indicators +- [ ] Alert/notification system for worker failures +- [ ] Add a Clock to the Dashboard (ISO 8601) + +**Testing & Documentation** +- [ ] Cross-platform testing (Debian, Ubuntu, Arch, Fedora, Win, OSx, BSD) +- [ ] Installation documentation +- [ ] Configuration examples +- [ ] Troubleshooting guide +- [ ] Validate IP change Stablity + +--- # Design diff --git a/docs/lib/img/sdl-about.png b/docs/lib/img/sdl-about.png new file mode 100644 index 0000000..44469b1 Binary files /dev/null and b/docs/lib/img/sdl-about.png differ diff --git a/docs/lib/img/sdl-dash.png b/docs/lib/img/sdl-dash.png new file mode 100644 index 0000000..ba11404 Binary files /dev/null and b/docs/lib/img/sdl-dash.png differ diff --git a/install-sdl.sh b/install-sdl.sh index 6aadcff..cd0e6ce 100755 --- a/install-sdl.sh +++ b/install-sdl.sh @@ -53,6 +53,13 @@ resolve_dirs() { done } +use_systemd() { + # Check if systemd user mode is available and service exists + if systemctl --user status >/dev/null 2>&1; then + return 0 # Use systemd + fi + return 1 # Use manual scripts +} # ------------------------------------------------------------ # Main @@ -69,7 +76,7 @@ fi # ------------------------------------------------------------ # Requirements # ------------------------------------------------------------ -for cmd in awk curl grep jq sed sha256sum tar unzip; do +for cmd in awk curl grep jq nc sed sha256sum tar unzip; do command -v "$cmd" >/dev/null || { echo "ERROR: $cmd is required" exit 1 @@ -160,13 +167,8 @@ fi # ------------------------------------------------------------ -# systemd user service check +# Setting SDL variables # ------------------------------------------------------------ -if ! systemctl --user status >/dev/null 2>&1; then - echo "ERROR: systemd user services are not available" - echo " SDL requires systemd --user support" - exit 1 -fi SDL_VERSION="$(jq -r '.METADATA.version' "$META_FILE")" SDL_VERSION_DATE="$(jq -r '.METADATA.version_date' "$META_FILE")" @@ -384,32 +386,52 @@ cd "$SDL_MGR/current/app" export PATH="$SDL_NODEJS/current/bin:$PATH" $NPM_BIN install + +# ------------------------------------------------------------ +# Copy Start / Stop Scripts +# ------------------------------------------------------------ +mkdir -p "$SDL_HOME/scripts/" +cp "$SDL_MGR/current/scripts/start-sdl-mgr.sh" "$SDL_HOME/scripts/" +cp "$SDL_MGR/current/scripts/stop-sdl-mgr.sh" "$SDL_HOME/scripts/" +cp "$SDL_MGR/current/scripts/status-sdl-mgr.sh" "$SDL_HOME/scripts/" +chmod u+x "$SDL_HOME/scripts/"* + + # ------------------------------------------------------------ # Install Systemd Unit Files # Copy SDL_MGR/current/scripts/sdl-mgr.service to ~/.config/systemd/user # ------------------------------------------------------------ -echo "" -echo "Installing SystemD Unit Files" +if use_systemd; then + echo "" + echo "Installing SystemD Unit Files" -mkdir -p ~/.config/systemd/user -cp "$SDL_MGR/current/scripts/sdl-mgr.service" ~/.config/systemd/user/ + mkdir -p ~/.config/systemd/user + cp "$SDL_MGR/current/scripts/sdl-mgr.service" ~/.config/systemd/user/ + + "${SDL_HOME}/scripts/stop-sdl-mgr.sh" + sleep 1 + systemctl --user daemon-reload --no-pager +fi + + +# ------------------------------------------------------------ +# Start SDL Manager +# ------------------------------------------------------------ +"${SDL_HOME}/scripts/start-sdl-mgr.sh" +sleep 2 +"${SDL_HOME}/scripts/status-sdl-mgr.sh" + +if [ $? -ne 0 ]; then + echo "" + echo "ERROR: SDL Manager failed to start" + exit 1 +fi + + +# -systemctl --user daemon-reload --no-pager -systemctl --user restart sdl-mgr --no-pager -sleep 3 -systemctl --user status sdl-mgr --no-pager echo "" echo "SDL Manager deployed:" echo " to: $SDL_HOME" -echo "SDL Controls:" -echo " systemctl --user status sdl-mgr" -echo " systemctl --user start sdl-mgr" -echo " systemctl --user stop sdl-mgr" - -echo "" -echo "Removal Commands:" -echo " WARNING: Backup data directory first!!!!" -echo " systemctl --user disable sdl-mgr" -echo " rm -rf $SDL_HOME" \ No newline at end of file diff --git a/metadata.json b/metadata.json index f911de1..e60810b 100644 --- a/metadata.json +++ b/metadata.json @@ -3,8 +3,8 @@ "name": "sdl", "description": "Software Defined Laboratory", "abbr": "SDL", - "version": "0.3.7", - "version_date": "2026-02-01", + "version": "0.3.8", + "version_date": "2026-02-08", "maturity": "ALPHA", "author": "John Haverlack", "copyright": "2026 John Haverlack", diff --git a/sdl-mgr/app/modules/nwa-lib/index.js b/sdl-mgr/app/modules/nwa-lib/index.js index 270823e..4a76bf2 100644 --- a/sdl-mgr/app/modules/nwa-lib/index.js +++ b/sdl-mgr/app/modules/nwa-lib/index.js @@ -605,7 +605,7 @@ function getUptimeDHMS() { const h = Math.floor((uptimeSec % 86400) / 3600); const m = Math.floor((uptimeSec % 3600) / 60); const s = Math.floor(uptimeSec % 60); - return `${d}:${h}:${m}:${s}`; + return `${d}d ${h}h ${m}m ${s}s`; } diff --git a/sdl-mgr/app/modules/sdl-mgr/index.js b/sdl-mgr/app/modules/sdl-mgr/index.js index de10346..e152706 100644 --- a/sdl-mgr/app/modules/sdl-mgr/index.js +++ b/sdl-mgr/app/modules/sdl-mgr/index.js @@ -1,19 +1,18 @@ const module = 'sdl-mgr'; // Module Name -import { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeDHMS } from '../nwa-lib/index.js'; +import { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeDHMS, getUptimeSec } from '../nwa-lib/index.js'; import mqtt from 'mqtt'; import os from 'os'; import dgram from 'dgram'; import fs from 'fs'; import path from 'path'; - - const config = load_config(); log(`Loaded module: ${module}`); -// log(`${module}: Cluster Conf: ${JSON.stringify(config.cluster, null, 2)}`, true); -// log(`${module}: Dirs: ${JSON.stringify(config.dirs, null, 2)}`, true); + +// ✅ In-memory workers state (built from telemetry only) +let workersState = {}; function loadClusterState(config) { const clusterDir = config.dirs.clstr; @@ -30,31 +29,7 @@ function loadClusterState(config) { meta: { updated: new Date().toISOString(), started_at: new Date(getProcessStartTs()).toISOString(), - uptime: getUptimeDHMS(), - stats: { - workers: { - allocated: 0, - available: 0, - used: 0 - }, - resources: { - cpus: { - allocated: 0, - available: 0, - used: 0 - }, - memory: { - allocated: 0, - available: 0, - used: 0 - }, - gpus: { - allocated: 0, - available: 0, - used: 0 - } - } - } + uptime: getUptimeDHMS() }, cluster: config.cluster, "sdl-mgr": { @@ -65,8 +40,7 @@ function loadClusterState(config) { distro: config.host.os.pretty_name, distro_name: config.host.os.name, distro_version: config.host.os.version - }, - workers: {} + } }; fs.writeFileSync(clusterFile, JSON.stringify(initialState, null, 2)); @@ -89,55 +63,15 @@ function saveClusterState(config, state) { // Write to disk try { fs.writeFileSync(clusterFile, JSON.stringify(state, null, 2)); - log(`${module}: cluster state saved to ${clusterFile}`); + // log(`${module}: cluster state saved to ${clusterFile}`); } catch (err) { log(`${module}: failed to save cluster state: ${err}`); throw err; } } -function addWorker(state, workerData) { - const sdl_id = workerData.sdl_id; - - // Check if worker already exists - const isNewWorker = !state.workers[sdl_id]; - - // Add or update worker - state.workers[sdl_id] = { - sdl_id: workerData.sdl_id, - hostname: workerData.hostname, - version: workerData.sdl_version, - platform: workerData.platform, - arch: workerData.arch, - distro: workerData.distro || "Unknown", - distro_name: workerData.distro_name || "Unknown", - distro_version: workerData.distro_version || "Unknown", - resources: { - cpus: { - allocated: workerData.cpus || 0, - available: workerData.cpus || 0, - used: 0 - }, - memory: { - allocated: workerData.totalmem || 0, - available: workerData.totalmem || 0, - used: 0 - }, - gpus: { - allocated: workerData.gpus || 0, - available: workerData.gpus || 0, - used: 0 - } - }, - status: 'active', - joined_at: isNewWorker ? new Date().toISOString() : state.workers[sdl_id].joined_at, - last_seen: new Date().toISOString() - }; - - return state; -} - -function computeStats(state) { +// ✅ Compute stats from in-memory workers state +function computeStats() { const stats = { workers: { allocated: 0, @@ -164,29 +98,28 @@ function computeStats(state) { }; // Count workers and aggregate resources - for (const worker of Object.values(state.workers)) { + for (const worker of Object.values(workersState)) { stats.workers.allocated++; if (worker.status === 'active') { stats.workers.available++; - stats.resources.cpus.available += worker.resources.cpus.available; - stats.resources.memory.available += worker.resources.memory.available; - stats.resources.gpus.available += worker.resources.gpus.available; + stats.resources.cpus.available += worker.resources?.cpus?.available || 0; + stats.resources.memory.available += worker.resources?.memory?.available || 0; + stats.resources.gpus.available += worker.resources?.gpus?.available || 0; } - stats.resources.cpus.allocated += worker.resources.cpus.allocated; - stats.resources.cpus.used += worker.resources.cpus.used; + stats.resources.cpus.allocated += worker.resources?.cpus?.allocated || 0; + stats.resources.cpus.used += worker.usage?.cpu?.total || 0; - stats.resources.memory.allocated += worker.resources.memory.allocated; - stats.resources.memory.used += worker.resources.memory.used; + stats.resources.memory.allocated += worker.resources?.memory?.allocated || 0; + stats.resources.memory.used += worker.usage?.memory?.used || 0; - stats.resources.gpus.allocated += worker.resources.gpus.allocated; - stats.resources.gpus.used += worker.resources.gpus.used; + stats.resources.gpus.allocated += worker.resources?.gpus?.allocated || 0; + stats.resources.gpus.used += worker.usage?.gpu?.total_utilization || 0; } - state.meta.stats = stats; - return state; + return stats; } // Worker Join Handler @@ -204,9 +137,6 @@ function startJoinHandler() { return; } - // Load current cluster state - let clusterState = loadClusterState(config); - const joinReqTopic = mqttCfg.topics?.[module]?.sub?.['sdl_join-req']; const joinAuthzTopic = mqttCfg.topics?.[module]?.pub?.['sdl_join-authz']; @@ -238,10 +168,8 @@ function startJoinHandler() { log(`${module}: received join request from ${joinReq.host} (${joinReq.sdl_id})`); - // Validate version const workerData = joinReq.msg['sdl-wkr']; const workerVersion = workerData?.sdl_version; - // const workerVersion = joinReq.msg?.worker?.version; const clusterVersion = config.package.version; let authorized = false; @@ -249,13 +177,7 @@ function startJoinHandler() { if (workerVersion === clusterVersion) { authorized = true; - - // Add worker to cluster state - clusterState = addWorker(clusterState, workerData); - clusterState = computeStats(clusterState); - saveClusterState(config, clusterState); - - log(`${module}: worker ${joinReq.host} authorized and added to cluster`); + log(`${module}: worker ${joinReq.host} authorized (version ${workerVersion})`); } else { authorized = false; reason = 'version_mismatch'; @@ -271,7 +193,8 @@ function startJoinHandler() { type: 'join-authz', msg: { sdl_id: joinReq.sdl_id, - authorized: authorized + authorized: authorized, + reason: reason } }; @@ -282,8 +205,6 @@ function startJoinHandler() { err => { if (err) { log(`${module}: failed to publish join authz: ${err}`); - } else { - // log(`${module}: published join authz to ${joinAuthzTopic}`); } } ); @@ -298,8 +219,6 @@ function startJoinHandler() { }); } - - function startSDLStatusPub() { const sdlCfg = config.modules[module]; const mqttCfg = config.modules.mqtt; @@ -322,59 +241,49 @@ function startSDLStatusPub() { const statusTopic = mqttCfg.topics?.[module]?.pub?.['sdl_cluster-status']; if (!statusTopic) { - log(`${module}: ${statusTopic} not configured`); + log(`${module}: cluster-status topic not configured`); return; } - // This is ok because we are running on localhost const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`; const client = mqtt.connect(mqttUrl); client.on('connect', () => { - log(`${module}: connected to MQTT at ${mqttUrl}`); + log(`${module}: status publisher connected to MQTT at ${mqttUrl}`); publishStatus(client, statusTopic); setInterval(() => publishStatus(client, statusTopic), statusInterval); }); client.on('error', err => { - log(`${module}: MQTT error: ${err}`); + log(`${module}: status publisher MQTT error: ${err}`); }); } function publishStatus(client, topic) { - // Load current cluster state - const clusterState = loadClusterState(config); - - // Recompute stats to get latest resource totals - const updatedState = computeStats(clusterState); + // Compute stats from in-memory workers + const stats = computeStats(); - //get ip addr + // Get IP for update command const interfaces = os.networkInterfaces(); let ip_addr = null; for (const [iface, addrs] of Object.entries(interfaces)) { for (const addr of addrs) { - // --- FILTERS --- - - // IPv4 only if (addr.family !== 'IPv4') continue; - - // Skip loopback if (addr.internal === true) continue; - - // Skip /32 networks (no broadcast: e.g. Tailscale) + const cidr = Number(addr.cidr?.split('/')[1]); if (!Number.isInteger(cidr) || cidr >= 32) continue; - - // Compute broadcast address - const broadcastAddr = computeBroadcast(addr.address, addr.netmask); - if (!broadcastAddr) continue; - - ip_addr = addr.address + + ip_addr = addr.address; + break; } + if (ip_addr) break; } + const web_port = config.modules.web.port; + const status = { ts: new Date().toISOString(), sdl_id: config.identity.sdl_id, @@ -384,12 +293,15 @@ function publishStatus(client, topic) { msg: { sdl: { version: config.package.version, + uptime_secs: getUptimeSec(), uptime: getUptimeDHMS(), - update_cmd: `curl -s http://${ip_addr}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${ip_addr}` + update_cmd: ip_addr + ? `curl -s http://${ip_addr}:${web_port}/dist/install-sdl-wkr.sh | bash -s ${ip_addr}:${web_port}` + : null }, cluster: config.cluster, - resources: updatedState.meta.stats.resources, // ✅ Add cluster resources - workers: updatedState.meta.stats.workers, // ✅ Add worker counts + resources: stats.resources, + workers: stats.workers, modules: Object.fromEntries( Object.entries(config.modules).map(([name, mod]) => [ name, @@ -406,15 +318,12 @@ function publishStatus(client, topic) { err => { if (err) { log(`${module}: failed to publish status: ${err}`); - } else { - // log(`${module}: published status to ${topic}`); } } ); } - -// ✅ NEW: Publish individual worker details +// ✅ Publish workers from in-memory state function startWorkersPub() { const sdlCfg = config.modules[module]; const mqttCfg = config.modules.mqtt; @@ -457,9 +366,6 @@ function startWorkersPub() { } function publishWorkers(client, topic) { - // Load current cluster state - const clusterState = loadClusterState(config); - const workersMsg = { ts: new Date().toISOString(), sdl_id: config.identity.sdl_id, @@ -467,7 +373,7 @@ function publishWorkers(client, topic) { host: config.identity.hostname, type: 'cluster-workers', msg: { - workers: clusterState.workers + workers: workersState // ✅ Use in-memory state from telemetry } }; @@ -478,18 +384,13 @@ function publishWorkers(client, topic) { err => { if (err) { log(`${module}: failed to publish workers: ${err}`); - } else { - // log(`${module}: published workers to ${topic}`); } } ); } - function startUdpBeacon() { let sdlCfg = config.modules['sdl-mgr']; - // log(`${module}: DEBUG: SDL config: ${JSON.stringify(sdlCfg, null, 2)}`, true); - // log(`${module}: DEBUG: Cluster Conf: ${JSON.stringify(config.cluster, null, 2)}`, true); sdlCfg.cluster = config.cluster; const mqttCfg = config.modules.mqtt; @@ -520,7 +421,6 @@ function startUdpBeacon() { }; log(`${module}: cluster: ${JSON.stringify(cluster)}`); - if (!config.host?.network) { log(`${module}: no host network metadata available for UDP beacon`); @@ -541,26 +441,17 @@ function startUdpBeacon() { setInterval(() => { const interfaces = os.networkInterfaces(); - // for (const [iface, addrs] of Object.entries(config.host.network)) { for (const [iface, addrs] of Object.entries(interfaces)) { for (const addr of addrs) { - // --- FILTERS --- - - // IPv4 only if (addr.family !== 'IPv4') continue; - - // Skip loopback if (addr.internal === true) continue; - // Skip /32 networks (no broadcast: e.g. Tailscale) const cidr = Number(addr.cidr?.split('/')[1]); if (!Number.isInteger(cidr) || cidr >= 32) continue; - // Compute broadcast address const broadcastAddr = computeBroadcast(addr.address, addr.netmask); if (!broadcastAddr) continue; - const beacon = { ts: new Date().toISOString(), sdl_id: config.identity.sdl_id, @@ -591,8 +482,8 @@ function startUdpBeacon() { web_ui_url: `http://${addr.address}:${config.modules.web.port}`, }, sdl_wkr_install_cmd: { - curl: `curl -s http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}`, - wget: `wget -O - http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}` + curl: `curl -s http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}:${config.modules.web.port}`, + wget: `wget -O - http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}:${config.modules.web.port}`, } } }; @@ -620,9 +511,8 @@ function startUdpBeacon() { ); } - -// Update startHeartbeatListener to use telemetry topic -function startTelemetryListener() { // ✅ Renamed function +// ✅ Build workers state from telemetry +function startTelemetryListener() { const sdlCfg = config.modules[module]; const mqttCfg = config.modules.mqtt; @@ -636,9 +526,7 @@ function startTelemetryListener() { // ✅ Renamed function return; } - let clusterState = loadClusterState(config); - - const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry']; // ✅ Changed + const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry']; if (!telemetryTopic) { log(`${module}: cluster-telemetry topic not configured`); return; @@ -659,55 +547,53 @@ function startTelemetryListener() { // ✅ Renamed function }); }); -client.on('message', (topic, message) => { - if (topic !== telemetryTopic) return; + client.on('message', (topic, message) => { + if (topic !== telemetryTopic) return; - try { - const telemetry = JSON.parse(message.toString()); - const sdl_id = telemetry.msg.sdl_id; - - // Update worker's last_seen timestamp - if (clusterState.workers[sdl_id]) { - clusterState.workers[sdl_id].last_seen = new Date().toISOString(); - clusterState.workers[sdl_id].status = 'active'; + try { + const telemetry = JSON.parse(message.toString()); + const sdl_id = telemetry.sdl_id; + const hostname = telemetry.host; + const now = new Date(); - // ✅ MERGE resources instead of replacing - if (telemetry.msg.resources) { - // Merge each resource type - if (telemetry.msg.resources.cpus) { - clusterState.workers[sdl_id].resources.cpus = { - ...clusterState.workers[sdl_id].resources.cpus, - ...telemetry.msg.resources.cpus - }; - } - if (telemetry.msg.resources.memory) { - clusterState.workers[sdl_id].resources.memory = { - ...clusterState.workers[sdl_id].resources.memory, - ...telemetry.msg.resources.memory - }; - } - if (telemetry.msg.resources.gpus) { - clusterState.workers[sdl_id].resources.gpus = { - ...clusterState.workers[sdl_id].resources.gpus, - ...telemetry.msg.resources.gpus - }; - } - } + // ✅ Build/update worker state from telemetry + workersState[sdl_id] = { + sdl_id: sdl_id, + hostname: hostname, + role: telemetry.role, + last_seen: now.toISOString(), // ✅ ISO string + last_seen_utime: Math.floor(now.getTime() / 1000), // ✅ Unix timestamp (seconds) + + // System info + platform: telemetry.msg?.system?.platform, + arch: telemetry.msg?.system?.arch, + distro: telemetry.msg?.system?.distro, + hardware: telemetry.msg?.system?.hardware, + + // Uptime + uptime: telemetry.msg?.uptime, + + // Resources + resources: telemetry.msg?.resources, + + // Usage + usage: telemetry.msg?.usage, + + // Load + load: telemetry.msg?.load + }; - clusterState = computeStats(clusterState); - saveClusterState(config, clusterState); + } catch (err) { + log(`${module}: failed to process telemetry: ${err}`); } - } catch (err) { - log(`${module}: failed to process telemetry: ${err}`); - } -}); + }); client.on('error', err => { log(`${module}: telemetry listener MQTT error: ${err}`); }); } -// Add Stale Worker Detection +// ✅ Mark stale workers as inactive function startStaleWorkerDetection() { const sdlCfg = config.modules[module]; @@ -716,44 +602,36 @@ function startStaleWorkerDetection() { return; } - const checkInterval = 10000; // Check every 10 seconds - const staleThreshold = - Number.isInteger(sdlCfg.worker_stale_threshold) && - sdlCfg.worker_stale_threshold > 0 - ? sdlCfg.worker_stale_threshold - : 30000; // Default 30 seconds + const checkInterval = 10000; + + // Try multiple config locations for stale threshold + const staleThreshold = + sdlCfg.expiration_timeout?.cluster_telemetry || + sdlCfg.worker_stale_threshold || + 30000; setInterval(() => { - let clusterState = loadClusterState(config); const now = Date.now(); - let changed = false; - for (const [sdl_id, worker] of Object.entries(clusterState.workers)) { + for (const [sdl_id, worker] of Object.entries(workersState)) { const lastSeenMs = Date.parse(worker.last_seen); const ageMs = now - lastSeenMs; if (ageMs > staleThreshold && worker.status === 'active') { - log(`${module}: worker ${worker.hostname} (${sdl_id}) marked as inactive (last seen ${Math.round(ageMs / 1000)}s ago)`); - clusterState.workers[sdl_id].status = 'inactive'; - changed = true; + log(`${module}: worker ${worker.hostname} (${sdl_id}) marked inactive (${Math.round(ageMs / 1000)}s)`); + workersState[sdl_id].status = 'inactive'; } } - - if (changed) { - clusterState = computeStats(clusterState); - saveClusterState(config, clusterState); - } }, checkInterval); - log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check interval: ${checkInterval}ms)`); + log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check: ${checkInterval}ms)`); } -// Update entry point +// Entry point startSDLStatusPub(); startWorkersPub(); startTelemetryListener(); startUdpBeacon(); loadClusterState(config); startJoinHandler(); -startStaleWorkerDetection(); - +startStaleWorkerDetection(); \ No newline at end of file diff --git a/sdl-mgr/app/package.json b/sdl-mgr/app/package.json index d7237e1..94c0a84 100644 --- a/sdl-mgr/app/package.json +++ b/sdl-mgr/app/package.json @@ -1,7 +1,7 @@ { "name": "sdl-mgr", - "version": "0.3.7", - "version_date": "2026-02-01", + "version": "0.3.8", + "version_date": "2026-02-08", "description": "Software Defined Laboratory: Manager (sdl-mgr)", "main": "index.js", "type": "module", @@ -25,6 +25,7 @@ "@fortawesome/fontawesome-free": "^6.7.2", "aedes": "^0.51.3", "bootstrap": "^5.3.7", + "bootstrap-icons": "^1.13.1", "chart.js": "^4.5.1", "csv-parse": "^5.6.0", "d3": "^7.9.0", diff --git a/sdl-mgr/conf/modules/sdl-mgr.json b/sdl-mgr/conf/modules/sdl-mgr.json index d4f7d6d..f8b93bf 100644 --- a/sdl-mgr/conf/modules/sdl-mgr.json +++ b/sdl-mgr/conf/modules/sdl-mgr.json @@ -5,8 +5,8 @@ "update_interval": { "udp_beacon": 2000, "cluster_status": 10000, - "cluster_workers": 10000, - "cluster_telemetry": 3000 + "cluster_workers": 2000, + "cluster_telemetry": 1000 }, "expiration_timeout": { "cluster_status": 60000, diff --git a/sdl-mgr/conf/modules/web.json b/sdl-mgr/conf/modules/web.json index f3710bb..992a2bf 100644 --- a/sdl-mgr/conf/modules/web.json +++ b/sdl-mgr/conf/modules/web.json @@ -5,6 +5,7 @@ "open_browser": false, "weblibs": { "bootstrap":"bootstrap/dist", + "bootstrap-icons":"bootstrap-icons", "chartjs":"chart.js/dist", "d3":"d3/dist", "fontawesome":"@fortawesome/fontawesome-free", diff --git a/sdl-mgr/html/index.html b/sdl-mgr/html/index.html index cdb154d..7c66dfe 100644 --- a/sdl-mgr/html/index.html +++ b/sdl-mgr/html/index.html @@ -9,6 +9,9 @@ + + + diff --git a/sdl-mgr/html/js/dash.js b/sdl-mgr/html/js/dash.js index caa76a2..7689997 100644 --- a/sdl-mgr/html/js/dash.js +++ b/sdl-mgr/html/js/dash.js @@ -500,53 +500,118 @@ function renderWorkersTable(msg) { html += ''; html += ''; html += ''; - html += ''; + html += ''; html += ''; html += ''; html += ''; html += ''; - html += ''; - html += ''; html += ''; html += ''; html += ''; - for (const [sdl_id, worker] of Object.entries(workers)) { - const status = worker.status === 'active' - ? 'Active' - : 'Inactive'; + const nowSec = Math.floor(Date.now() / 1000); + const staleThresholdSec = 60; + for (const [sdl_id, worker] of Object.entries(workers)) { + const ageSec = nowSec - worker.last_seen_utime; + const isActive = ageSec <= staleThresholdSec; + + // Status badge + const statusBadge = isActive + ? 'Active' + : 'Inactive'; // ✅ Red instead of grey + + // Hardware info + const hwType = worker.hardware?.type || 'unknown'; + const hwIcon = hwType === 'hw' ? '🖥️' : hwType === 'vm' ? '💠' : '❓'; + const hwManuf = worker.hardware?.manufacturer || 'Unknown'; + const hwModel = worker.hardware?.model || 'Unknown'; + + // ✅ Full hardware display with model + const hardwareDisplay = hwType === 'vm' + ? `${hwIcon} ${hwModel}` + : `${hwIcon} ${hwManuf} ${hwModel}`; + + // ✅ OS info + const osDistro = worker.distro || 'Unknown'; + const osArch = worker.arch || 'unknown'; + const osPlatform = worker.platform || 'unknown'; + + // CPU const cpuAvail = worker.resources?.cpus?.available || 0; const cpuTotal = worker.resources?.cpus?.allocated || 0; - const cpuUsed = worker.resources?.cpus?.used || 0; + const cpuUsed = worker.usage?.cpu?.total || 0; + // Memory const memAvail = worker.resources?.memory?.available || 0; const memTotal = worker.resources?.memory?.allocated || 0; - const memUsed = worker.resources?.memory?.used || 0; + const memUsed = worker.usage?.memory?.used || 0; - // Convert bytes to GB const memAvailGB = Math.round(memAvail / (1024 * 1024 * 1024)); const memTotalGB = Math.round(memTotal / (1024 * 1024 * 1024)); - const memUsedGB = memUsed > 0 ? Math.round(memUsed / (1024 * 1024 * 1024)) : 0; + const memPercent = worker.usage?.memory?.percent_used || 0; + // GPU const gpuAvail = worker.resources?.gpus?.available || 0; const gpuTotal = worker.resources?.gpus?.allocated || 0; - const gpuUsed = worker.resources?.gpus?.used || 0; + const gpuData = worker.usage?.gpu; + const gpuUsed = gpuData?.total_utilization || 0; + const gpuMemMB = gpuData?.total_memory_total_mb || 0; + const gpuMemGB = Math.round(gpuMemMB / 1024); - const platform = `${worker.platform || 'unknown'} / ${worker.arch || 'unknown'}`; + // Uptime + const procUptime = worker.uptime?.proc_dhms || 'N/A'; + const sysUptime = worker.uptime?.sys_dhms || 'N/A'; - const lastSeen = worker.last_seen - ? new Date(worker.last_seen).toLocaleString() - : 'N/A'; + // ✅ Last seen age + const days = Math.floor(ageSec / 86400); + const hours = Math.floor((ageSec % 86400) / 3600); + const minutes = Math.floor((ageSec % 3600) / 60); + const seconds = Math.floor(ageSec % 60); + + let lastSeenAge = ''; + if (days > 0) lastSeenAge += `${days}d `; + if (hours > 0 || days > 0) lastSeenAge += `${hours}h `; + if (minutes > 0 || hours > 0 || days > 0) lastSeenAge += `${minutes}m `; + lastSeenAge += `${seconds}s ago`; html += ''; - html += ``; - html += ``; - html += ``; - html += ``; - html += ``; - html += ``; - html += ``; + + // ✅ Hostname column with hardware underneath + html += ``; + + // ✅ Status column with uptime and last seen + html += ``; + + // CPU + html += ``; + + // Memory + html += ``; + + // GPU + html += ``; + html += ''; } @@ -554,4 +619,4 @@ function renderWorkersTable(msg) { html += '
HostnameSystemStatus CPU RAM GPUPlatformLast Seen
${worker.hostname}
${worker.sdl_id}		${status}${cpuAvail} / ${cpuTotal}${cpuUsed > 0 ? `
(${cpuUsed} used)` : ''}		${memAvailGB} / ${memTotalGB} GB${memUsedGB > 0 ? `
(${memUsedGB} GB used)` : ''}		${gpuAvail} / ${gpuTotal}${gpuUsed > 0 ? `
(${gpuUsed} used)` : ''}		${platform}${lastSeen} + ${worker.hostname}
+ ${worker.sdl_id}
+ ${hardwareDisplay}
+ ${osDistro} ${osArch} ${osPlatform} + 		+ ${statusBadge}
+ SDL: ${procUptime}
+ OS: ${sysUptime}
+ ${lastSeenAge} + 		+ ${cpuAvail} / ${cpuTotal} + ${cpuUsed > 0 ? `
${cpuUsed}% load` : ''} + 		+ ${memAvailGB} / ${memTotalGB} GB + ${memPercent > 0 ? `
${memPercent}% used` : ''} + 		+ ${gpuAvail} / ${gpuTotal} + ${gpuMemGB > 0 ? `
(${gpuMemGB} GB)` : ''} + ${gpuUsed > 0 ? `
${gpuUsed}% load` : ''} +
'; document.getElementById('dash-sdl-wkrs').innerHTML = html; -} +} \ No newline at end of file diff --git a/sdl-mgr/html/js/weblib.js b/sdl-mgr/html/js/weblib.js index dc6b7c4..46125dc 100644 --- a/sdl-mgr/html/js/weblib.js +++ b/sdl-mgr/html/js/weblib.js @@ -371,54 +371,99 @@ function loadNavTabs() { // =============================== // Side Panel (layout-collapsing) // =============================== -async function loadSidePanel() { - // console.log('DEBUG: loadSidePanel() called'); +// async function loadSidePanel() { +// // console.log('DEBUG: loadSidePanel() called'); +// const panel = document.getElementById('sidepanel'); +// const inner = document.getElementById('sidepanel-inner'); +// const toggleBtn = document.getElementById('sidepanel-toggle'); + +// if (!panel || !inner || !toggleBtn) return; + +// const currentPage = +// new URLSearchParams(window.location.search).get('page') || 'index'; + +// const isCollapsed = +// localStorage.getItem('sidepanel_collapsed') === 'true'; + +// if (isCollapsed) { +// panel.classList.add('collapsed'); +// // toggleBtn.innerHTML = ``; +// toggleBtn.innerHTML = ''; +// } else { +// // toggleBtn.innerHTML = ``; +// toggleBtn.innerHTML = ''; +// } + +// try { +// const res = await fetch('/api/nav/sidepanel'); +// if (!res.ok) throw new Error(`HTTP ${res.status}`); + +// const sideNav = await res.json(); + +// renderAllSidePanelSections(inner, sideNav, currentPage); +// } catch (err) { +// console.error('Failed to load side panel:', err); +// inner.innerHTML = ''; +// } + +// // Collapse / expand behavior +// toggleBtn.onclick = () => { +// const collapsed = panel.classList.toggle('collapsed'); +// localStorage.setItem('sidepanel_collapsed', collapsed); + +// // toggleBtn.innerHTML = ` +// // +// // `; + +// toggleBtn.innerHTML = ` +// +// `; +// }; +// } +async function loadSidePanel() { const panel = document.getElementById('sidepanel'); const inner = document.getElementById('sidepanel-inner'); const toggleBtn = document.getElementById('sidepanel-toggle'); - + if (!panel || !inner || !toggleBtn) return; - + const currentPage = new URLSearchParams(window.location.search).get('page') || 'index'; - const isCollapsed = localStorage.getItem('sidepanel_collapsed') === 'true'; - + if (isCollapsed) { panel.classList.add('collapsed'); - toggleBtn.innerHTML = ``; + toggleBtn.innerHTML = ''; } else { - toggleBtn.innerHTML = ``; + toggleBtn.innerHTML = ''; } - + try { const res = await fetch('/api/nav/sidepanel'); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - + if (!res.ok) throw new Error(`HTTP ${res.status}`); // ✅ Fixed template literal const sideNav = await res.json(); - renderAllSidePanelSections(inner, sideNav, currentPage); } catch (err) { console.error('Failed to load side panel:', err); inner.innerHTML = ''; } - + // Collapse / expand behavior toggleBtn.onclick = () => { const collapsed = panel.classList.toggle('collapsed'); localStorage.setItem('sidepanel_collapsed', collapsed); - toggleBtn.innerHTML = ` - + `; }; } - // ------------------------------- // Find matching section by page // ------------------------------- diff --git a/sdl-mgr/html/md/changelog.md b/sdl-mgr/html/md/changelog.md index dd8144e..e950d88 100644 --- a/sdl-mgr/html/md/changelog.md +++ b/sdl-mgr/html/md/changelog.md @@ -5,14 +5,19 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.7 | +| **Version** | 0.3.8 | | **Date** | 2026-02-01 | +## v0.3.8 - 2026-02-01 (ALPHA) + +- Start v0.3.8 + ## v0.3.7 - 2026-02-01 (ALPHA) **Summary** sdl-wkr Telemetry +- v0.3.7 - Fixing sdl-wkr RAM detection - Start v0.3.7 diff --git a/sdl-mgr/html/md/dash.md b/sdl-mgr/html/md/dash.md index afd8c68..2a61aed 100644 --- a/sdl-mgr/html/md/dash.md +++ b/sdl-mgr/html/md/dash.md @@ -7,7 +7,7 @@
-
+
Workers
\ No newline at end of file diff --git a/sdl-mgr/scripts/sdl-uninstall.sh b/sdl-mgr/scripts/sdl-uninstall.sh new file mode 100755 index 0000000..e69de29 diff --git a/sdl-mgr/scripts/start-sdl-mgr.sh b/sdl-mgr/scripts/start-sdl-mgr.sh new file mode 100755 index 0000000..5903030 --- /dev/null +++ b/sdl-mgr/scripts/start-sdl-mgr.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# start-sdl-mgr.sh - Script to start SDL Manager +set -euo pipefail + +use_systemd() { + # Check if systemd user mode is available and service exists + if systemctl --user status >/dev/null 2>&1; then + # Check if our service file exists + if systemctl --user list-unit-files | grep -q "^sdl-mgr.service"; then + return 0 # Use systemd + fi + fi + return 1 # Use manual scripts +} + +# Get $SDL_HOME directory from SCRIPT PATH +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SDL_HOME="$(cd "$SCRIPT_DIR/.." && pwd)" + +if use_systemd; then + # Systemd mode - let systemd handle it + echo "Starting SDL Manager (systemd)..." + systemctl --user start sdl-mgr + exit 0 +fi + +# Manual mode - continue with checks +NODE_BIN_DIR="${SDL_HOME}/nodejs/current/bin" +NODE_BIN="${NODE_BIN_DIR}/node" +NPM_BIN="${NODE_BIN_DIR}/npm" + +SDL_MGR_APP_DIR="${SDL_HOME}/sdl-mgr/current/app" + +# Check that sdl-mgr is not already running +SDL_MGR_PID="" +for pid in $(pgrep -f 'node' 2>/dev/null || true); do + cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true) + if [[ "$cwd" == *"sdl-mgr"* ]]; then + SDL_MGR_PID="$pid" + break + fi +done + +if [[ -n "$SDL_MGR_PID" ]]; then + echo "ERROR: SDL Manager is already running with PID ${SDL_MGR_PID}" + echo " Stop it first: ${SDL_HOME}/scripts/stop-sdl-mgr.sh" + exit 1 +fi + +# Verify paths exist +if [[ ! -f "$NODE_BIN" ]]; then + echo "ERROR: Node.js not found at: $NODE_BIN" + echo " Run install-sdl.sh first" + exit 1 +fi + +if [[ ! -d "$SDL_MGR_APP_DIR" ]]; then + echo "ERROR: SDL Manager app not found at: $SDL_MGR_APP_DIR" + echo " Run install-sdl.sh first" + exit 1 +fi + +echo "Starting SDL Manager..." +echo " App Dir: ${SDL_MGR_APP_DIR}" +echo " Node: \ No newline at end of file diff --git a/sdl-mgr/scripts/status-sdl-mgr.sh b/sdl-mgr/scripts/status-sdl-mgr.sh new file mode 100755 index 0000000..a69a4cf --- /dev/null +++ b/sdl-mgr/scripts/status-sdl-mgr.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# status-sdl-mgr.sh - Script to show status of SDL Manager +set -euo pipefail + +use_systemd() { + # Check if systemd user mode is available and service exists + if systemctl --user status >/dev/null 2>&1; then + # Check if our service file exists + if systemctl --user list-unit-files | grep -q "^sdl-mgr.service"; then + return 0 # Use systemd + fi + fi + return 1 # Use manual scripts +} + +if use_systemd; then + systemctl --user status sdl-mgr +else + # Get sdl-mgr PID + SDL_MGR_PID="" + for pid in $(pgrep -f 'node' 2>/dev/null || true); do + cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true) + if [[ "$cwd" == *"sdl-mgr"* ]]; then + SDL_MGR_PID="$pid" + break + fi + done + + if [[ -z "$SDL_MGR_PID" ]]; then + echo "SDL Manager is not running" + exit 0 + fi + + # Get process uptime + PROCESS_START=$(ps -p "$SDL_MGR_PID" -o lstart= 2>/dev/null || echo "unknown") + + echo "SDL Manager is running" + echo " PID: ${SDL_MGR_PID}" + echo " Started: ${PROCESS_START}" + exit 0 +fi \ No newline at end of file diff --git a/sdl-mgr/scripts/stop-sdl-mgr.sh b/sdl-mgr/scripts/stop-sdl-mgr.sh new file mode 100755 index 0000000..454f4d8 --- /dev/null +++ b/sdl-mgr/scripts/stop-sdl-mgr.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# stop-sdl-mgr.sh - Script to stop SDL Manager +set -euo pipefail + +use_systemd() { + # Check if systemd user mode is available and service exists + if systemctl --user status >/dev/null 2>&1; then + # Check if our service file exists + if systemctl --user list-unit-files | grep -q "^sdl-mgr.service"; then + return 0 # Use systemd + fi + fi + return 1 # Use manual scripts +} + +if use_systemd; then + # Systemd mode - let systemd handle it + echo "Stopping SDL Manager (systemd)..." + systemctl --user stop sdl-mgr --quiet 2>/dev/null || true + echo "SDL Manager stopped" + exit 0 +fi + +# Manual mode - find and kill process +SDL_MGR_PID="" +for pid in $(pgrep -f 'node' 2>/dev/null || true); do + cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true) + if [[ "$cwd" == *"sdl-mgr"* ]]; then + SDL_MGR_PID="$pid" + break + fi +done + +if [[ -z "$SDL_MGR_PID" ]]; then + echo "SDL Manager is not running" + exit 0 +fi + +echo "Stopping SDL Manager (PID: ${SDL_MGR_PID})..." + +# Send SIGTERM for graceful shutdown +kill "$SDL_MGR_PID" + +# Wait up to 10 seconds for process to exit +for i in {1..10}; do + if ! kill -0 "$SDL_MGR_PID" 2>/dev/null; then + echo "SDL Manager stopped successfully" + exit 0 + fi + sleep 1 +done + +# If still running after 10 seconds, force kill +if kill -0 "$SDL_MGR_PID" 2>/dev/null; then + echo "WARNING: Graceful shutdown failed, forcing kill..." + kill -9 "$SDL_MGR_PID" + sleep 1 + + if kill -0 "$SDL_MGR_PID" 2>/dev/null; then + echo "ERROR: Failed to stop SDL Manager" + exit 1 + fi +fi + +echo "SDL Manager stopped" \ No newline at end of file diff --git a/sdl-wkr/app/index.js b/sdl-wkr/app/index.js index 6ee98d7..e57d332 100644 --- a/sdl-wkr/app/index.js +++ b/sdl-wkr/app/index.js @@ -12,7 +12,7 @@ log('=========================================================================== log(config.package.name + ': STARTING: ' + config.package.description + ' v' + config.package.version); // log(JSON.stringify(config, null, 2)); // log(JSON.stringify(config.dirs, null, 2), false); -log(JSON.stringify(config.host, null, 2), false); +// log(JSON.stringify(config.host, null, 2), false); for (let m in config.modules) { // log(`Loading module: ${m}`); diff --git a/sdl-wkr/app/modules/nwa-lib/index.js b/sdl-wkr/app/modules/nwa-lib/index.js index a0de696..ef07cd4 100644 --- a/sdl-wkr/app/modules/nwa-lib/index.js +++ b/sdl-wkr/app/modules/nwa-lib/index.js @@ -602,8 +602,16 @@ function getUptimeDHMS() { const h = Math.floor((uptimeSec % 86400) / 3600); const m = Math.floor((uptimeSec % 3600) / 60); const s = Math.floor(uptimeSec % 60); - return `${d}.${h}.${m}.${s}`; + return `${d}d ${h}h ${m}m ${s}s`; } +function getOSUptimeDHMS() { + const uptimeSec = os.uptime(); + const d = Math.floor(uptimeSec / 86400); + const h = Math.floor((uptimeSec % 86400) / 3600); + const m = Math.floor((uptimeSec % 3600) / 60); + const s = Math.floor(uptimeSec % 60); + return `${d}d ${h}h ${m}m ${s}s`; +} -export { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeMs, getUptimeSec, getUptimeDHMS }; +export { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeMs, getUptimeSec, getUptimeDHMS, getOSUptimeDHMS }; diff --git a/sdl-wkr/app/modules/sdl-wkr/index.js b/sdl-wkr/app/modules/sdl-wkr/index.js index baa67cc..79d684c 100644 --- a/sdl-wkr/app/modules/sdl-wkr/index.js +++ b/sdl-wkr/app/modules/sdl-wkr/index.js @@ -1,10 +1,11 @@ const module = 'sdl-wkr'; // Module Name -import { load_config, log } from '../nwa-lib/index.js'; +import { load_config, log, getUptimeDHMS, getOSUptimeDHMS } from '../nwa-lib/index.js'; import dgram from 'dgram'; import mqtt from 'mqtt'; -import os from 'os'; -import { exec } from 'child_process'; - +import fs from 'fs'; +import os, { platform } from 'os'; +import { exec, execSync } from 'child_process'; +import { get } from 'http'; const config = load_config(); @@ -19,6 +20,7 @@ let mqttClient = null; let clusterConfig = null; let updating = false; let telemetryTimer = null; +let lastCpuInfo = null; // ------------------------------- // Start UDP discovery @@ -237,6 +239,8 @@ function handleClusterStatus(payload) { const clusterVersion = payload.msg?.sdl?.version; const updateCmd = payload.msg?.sdl?.update_cmd; const localVersion = config.package.version; + const uptime = payload.msg?.sdl?.uptime_secs; + if (!clusterVersion) return; @@ -245,6 +249,11 @@ function handleClusterStatus(payload) { `${module}: version mismatch detected (cluster=${clusterVersion}, local=${localVersion})` ); triggerWorkerUpdate(clusterVersion, updateCmd); + } else if (uptime < 5) { // sld-mgr restarts for Development Testing + log( + `${module}: version mismatch detected (cluster=${clusterVersion}, local=${localVersion})` + ); + triggerWorkerUpdate(clusterVersion, updateCmd); } } @@ -306,14 +315,80 @@ function startTelemetry() { }, telemetryInterval); } +// Add this function for per-CPU usage +function getPerCPUUsage() { + const cpus = os.cpus(); + + if (!lastCpuInfo || !lastCpuInfo.perCpu) { + // First call - initialize + const perCpu = cpus.map(cpu => { + let total = 0; + for (let type in cpu.times) { + total += cpu.times[type]; + } + return { + idle: cpu.times.idle, + total: total + }; + }); + + lastCpuInfo = lastCpuInfo || {}; + lastCpuInfo.perCpu = perCpu; + + return cpus.map(() => 0); // Return 0% for all CPUs on first call + } + + // Calculate per-CPU usage + const usage = cpus.map((cpu, i) => { + let total = 0; + for (let type in cpu.times) { + total += cpu.times[type]; + } + + const idleDelta = cpu.times.idle - lastCpuInfo.perCpu[i].idle; + const totalDelta = total - lastCpuInfo.perCpu[i].total; + + const cpuUsage = 100 - (100 * idleDelta / totalDelta); + + // Update stored values + lastCpuInfo.perCpu[i] = { + idle: cpu.times.idle, + total: total + }; + + return Math.max(0, Math.min(100, Math.round(cpuUsage))); + }); + + return usage; +} + +// Update getCPUUsagePercent to calculate average from per-CPU +function getCPUUsagePercent() { + const perCpuUsage = getPerCPUUsage(); + if (perCpuUsage.length === 0) return 0; + + const total = perCpuUsage.reduce((sum, usage) => sum + usage, 0); + return Math.round(total / perCpuUsage.length); +} + +// ------------------------------- +// Publish telemetry +// ------------------------------- function publishTelemetry(topic) { if (!mqttClient || !authorized) return; const gpuCount = Array.isArray(config.host.gpu) ? config.host.gpu.length : 0; - // ✅ Convert GB to bytes - const memoryBytes = config.host.memory.total_gb - ? config.host.memory.total_gb * 1024 * 1024 * 1024 + // Get real-time memory usage + const totalMem = os.totalmem(); + const freeMem = os.freemem(); + const usedMem = totalMem - freeMem; + + // Get CPU usage + const perCpuUsage = getPerCPUUsage(); + // const totalCpuUsage = getCPUUsagePercent(); + const totalCpuUsage = perCpuUsage.length > 0 + ? Math.round(perCpuUsage.reduce((sum, usage) => sum + usage, 0) / perCpuUsage.length) : 0; const telemetry = { @@ -323,25 +398,50 @@ function publishTelemetry(topic) { host: config.identity.hostname, type: 'worker-telemetry', msg: { - sdl_id: config.identity.sdl_id, - hostname: config.identity.hostname, - status: 'active', + system: { + platform: config.host.os.platform, + arch: config.host.cpu.arch, + distro: config.host.os.pretty_name, + distro_name: config.host.os.name, + distro_version: config.host.os.version, + hardware: getSystemHardware() + }, + uptime: { + process: Math.floor(process.uptime()), + proc_dhms: getUptimeDHMS(), + system: Math.floor(os.uptime()), + sys_dhms: getOSUptimeDHMS() + }, resources: { cpus: { allocated: config.host.cpu.cores_logical, - available: config.host.cpu.cores_logical, - used: 0 + available: config.host.cpu.cores_logical }, memory: { - allocated: memoryBytes, // ✅ Fixed - available: memoryBytes, // ✅ Fixed - used: 0 + allocated: totalMem }, gpus: { allocated: gpuCount, - available: gpuCount, - used: 0 + available: gpuCount } + }, + usage: { + cpu: { + total: totalCpuUsage, + per_core: perCpuUsage + }, + memory: { + total: totalMem, + used: usedMem, + free: freeMem, + percent_used: Math.round((usedMem / totalMem) * 100) + }, + gpu: getGPUUsage() + }, + load: { + 1: os.loadavg()[0], + 5: os.loadavg()[1], + 15: os.loadavg()[2] } } }; @@ -358,6 +458,142 @@ function publishTelemetry(topic) { ); } + +// Add GPU telemetry function (exec-based) +function getGPUUsage() { + const gpuCount = Array.isArray(config.host.gpu) ? config.host.gpu.length : 0; + + if (gpuCount === 0) { + return {}; + } + + try { + // Try nvidia-smi for NVIDIA GPUs + const output = execSync( + 'nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits', + { encoding: 'utf8', timeout: 5000 } + ).toString(); + + const lines = output.trim().split('\n'); + const gpus = lines.map((line, index) => { + const [util, memUsed, memTotal] = line.split(',').map(v => parseInt(v.trim())); + return { + id: index, + utilization: util, + memory_used_mb: memUsed, + memory_total_mb: memTotal, + memory_percent: Math.round((memUsed / memTotal) * 100) + }; + }); + + // Calculate totals + const totalUtil = Math.round(gpus.reduce((sum, gpu) => sum + gpu.utilization, 0) / gpus.length); + const totalMemUsed = gpus.reduce((sum, gpu) => sum + gpu.memory_used_mb, 0); + const totalMemTotal = gpus.reduce((sum, gpu) => sum + gpu.memory_total_mb, 0); + + return { + count: gpus.length, + total_utilization: totalUtil, + total_memory_used_mb: totalMemUsed, + total_memory_total_mb: totalMemTotal, + total_memory_percent: Math.round((totalMemUsed / totalMemTotal) * 100), + gpus: gpus + }; + } catch (err) { + // nvidia-smi not available or failed + return { error: 'unavailable' }; + } +} + + + +function getSystemHardware() { + const hardware = { + type: 'unknown', + manufacturer: 'Unknown', + model: 'Unknown' + }; + + // Only run Linux-specific detection on Linux + if (config.host.os.platform !== 'linux') { + return hardware; + } + + // === Linux-specific detection below === + + // Check for Raspberry Pi FIRST + try { + const piModel = fs.readFileSync('/proc/device-tree/model', 'utf8') + .replace(/\0/g, '') + .replace(/Raspberry Pi/, '') + .trim(); + + if (piModel) { + hardware.type = 'hw'; // ✅ Changed from 'physical' + hardware.manufacturer = 'Raspberry Pi'; + hardware.model = piModel; + return hardware; + } + } catch (err) { + // Not a Pi, continue + } + + // Check for hypervisor flag + try { + const cpuinfo = fs.readFileSync('/proc/cpuinfo', 'utf8'); + if (cpuinfo.includes('hypervisor')) { + hardware.type = 'vm'; + + // Detect hypervisor type from DMI + try { + const manufacturer = fs.readFileSync('/sys/class/dmi/id/sys_vendor', 'utf8').trim(); + const product = fs.readFileSync('/sys/class/dmi/id/product_name', 'utf8').trim(); + + hardware.manufacturer = manufacturer; + + // Set model to hypervisor type + if (manufacturer.includes('QEMU') || product.includes('KVM')) { + hardware.model = 'KVM'; + } else if (manufacturer.includes('VMware')) { + hardware.model = 'VMware'; + } else if (manufacturer.includes('innotek')) { + hardware.model = 'VirtualBox'; + } else if (manufacturer.includes('Microsoft')) { + hardware.model = 'Hyper-V'; + } else if (manufacturer.includes('Xen')) { + hardware.model = 'Xen'; + } else { + hardware.model = 'Unknown VM'; + } + + return hardware; + } catch (err) { + hardware.manufacturer = 'Unknown'; + hardware.model = 'Unknown VM'; + return hardware; + } + } + } catch (err) { + // Can't read cpuinfo + } + + // No hypervisor flag - likely physical, get DMI info + try { + const manufacturer = fs.readFileSync('/sys/class/dmi/id/sys_vendor', 'utf8').trim(); + const product = fs.readFileSync('/sys/class/dmi/id/product_name', 'utf8').trim(); + + hardware.type = 'hw'; // ✅ Changed from 'physical' + hardware.manufacturer = manufacturer; + hardware.model = product; + + return hardware; + } catch (err) { + // DMI not available + } + + return hardware; +} + function stopTelemetry() { if (telemetryTimer) { clearInterval(telemetryTimer); diff --git a/sdl-wkr/app/package.json b/sdl-wkr/app/package.json index c82294c..25821ae 100644 --- a/sdl-wkr/app/package.json +++ b/sdl-wkr/app/package.json @@ -1,8 +1,8 @@ { "name": "sdl-wkr", "description": "Software Defined Laboratory: Worker (sdl-wkr)", - "version": "0.3.7", - "version_date": "2026-02-01", + "version": "0.3.8", + "version_date": "2026-02-08", "main": "index.js", "type": "module", "scripts": { diff --git a/sdl-wkr/scripts/install-sdl-wkr.sh b/sdl-wkr/scripts/install-sdl-wkr.sh index 65f0383..f31bdf6 100644 --- a/sdl-wkr/scripts/install-sdl-wkr.sh +++ b/sdl-wkr/scripts/install-sdl-wkr.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # install-sdl-wkr.sh - Script to install SDL Worker -# Usage: curl -s http://:8081/dist/install-sdl-wkr.sh | bash -s +# Usage: curl -s http://:/dist/install-sdl-wkr.sh | bash -s : set -euo pipefail # ------------------------------------------------------------ @@ -79,11 +79,11 @@ install_nodejs() { esac - NODE_VER=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.nodejs.version') + NODE_VER=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.nodejs.version') NODE_DIR="${SDL_HOME}/nodejs" NODE_VER_BASE="node-${NODE_VER}-${OS_PLATFORM}-${OS_ARCH}" NODE_FILE="node-${NODE_VER}-${OS_PLATFORM}-${OS_ARCH}.${EXT}" - NODE_URL="http://${SDL_MGR_IP}:8081/dist/${NODE_FILE}" + NODE_URL="http://${SDL_MGR_IP}:${SDL_MGR_PORT}/dist/${NODE_FILE}" #echo "NodeJS Version: ${NODE_VER}" @@ -138,19 +138,21 @@ if [ -z "$1" ]; then exit 1 fi -SDL_MGR_IP=$1 +ARG1=$1 # SDL_MGR_IP:PORT +SDL_MGR_IP=$(echo "$ARG1" | cut -d ':' -f 1) +SDL_MGR_PORT=$(echo "$ARG1" | cut -d ':' -f 2) -# SDL_HOME=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.dirs.sdlhome') +# SDL_HOME=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.dirs.sdlhome') SDL_HOME=$HOME/.sdl -SDL_VER=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.version') -SDL_DESC=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.description') -SDL_ABBR=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.abbr') -SDL_COPYRT=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.copyright') +SDL_VER=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.version') +SDL_DESC=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.description') +# SDL_ABBR=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.abbr') +SDL_COPYRT=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.copyright') echo "###############################################" -echo "${SDL_DESC} (${SDL_ABBR})" +# echo "${SDL_DESC} (${SDL_ABBR})" echo "Version: ${SDL_VER}" echo "Copyright: (C) ${SDL_COPYRT}" echo "Installing SDL Worker..." @@ -160,7 +162,7 @@ echo "" TMP_DIR="${SDL_HOME}/tmp" SDL_WKR_DIR="${SDL_HOME}/sdl-wkr" -SDL_WKR_VERSION=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.version') +SDL_WKR_VERSION=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.version') SDL_WKR_TGZ="sdl-wkr_${SDL_WKR_VERSION}.tgz" SDL_CONF_DIR="${SDL_HOME}/conf" SLD_LOGS_DIR="${SDL_HOME}/logs" @@ -177,8 +179,8 @@ mkdir -p "${TMP_DIR}" # Download the latest sdl-wkr version to TMP_DIR -WKR_TGZ_URL="http://${SDL_MGR_IP}:8081/dist/${SDL_WKR_TGZ}" -WKR_TGZ_SHA256_URL="http://${SDL_MGR_IP}:8081/dist/${SDL_WKR_TGZ}.sha256" +WKR_TGZ_URL="http://${SDL_MGR_IP}:${SDL_MGR_PORT}/dist/${SDL_WKR_TGZ}" +WKR_TGZ_SHA256_URL="http://${SDL_MGR_IP}:${SDL_MGR_PORT}/dist/${SDL_WKR_TGZ}.sha256" curl -sSL --fail "${WKR_TGZ_URL}" -o "${TMP_DIR}/${SDL_WKR_TGZ}" || { echo "Failed to download sdl-wkr"; exit 1; } curl -sSL --fail "${WKR_TGZ_SHA256_URL}" -o "${TMP_DIR}/${SDL_WKR_TGZ}.sha256" || { echo "Failed to download sdl-wkr SHA256"; exit 1; } @@ -214,7 +216,7 @@ cp "$SDL_WKR_DIR/current/scripts/sdl-wkr.service" ~/.config/systemd/user/ # Install Node.js idempotently -NODE_VER=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.nodejs.version') +NODE_VER=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.nodejs.version') NODE_DIR="${SDL_HOME}/nodejs" NODE_BIN="${NODE_DIR}/current/bin/node" NPM_BIN="${NODE_DIR}/current/bin/npm" diff --git a/sdl-wkr/scripts/start-sdl-wkr.sh b/sdl-wkr/scripts/start-sdl-wkr.sh new file mode 100755 index 0000000..8074a88 --- /dev/null +++ b/sdl-wkr/scripts/start-sdl-wkr.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# start-sdl-wkr.sh - Script to start SDL Worker +set -euo pipefail + +use_systemd() { + # Check if systemd user mode is available and service exists + if systemctl --user status >/dev/null 2>&1; then + # Check if our service file exists + if systemctl --user list-unit-files | grep -q "^sdl-wkr.service"; then + return 0 # Use systemd + fi + fi + return 1 # Use manual scripts +} + +# Get $SDL_HOME directory from SCRIPT PATH +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SDL_HOME="$(cd "$SCRIPT_DIR/.." && pwd)" + +NODE_BIN_DIR="${SDL_HOME}/nodejs/current/bin" +NODE_BIN="${NODE_BIN_DIR}/node" +NPM_BIN="${NODE_BIN_DIR}/npm" + +SDL_WKR_APP_DIR="${SDL_HOME}/sdl-wkr/current/app" + +# Check that sdl-wkr is not already running +SDL_WKR_PID="" +for pid in $(pgrep -f 'node' 2>/dev/null || true); do + cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true) + if [[ "$cwd" == *"sdl-wkr"* ]]; then + SDL_WKR_PID="$pid" + break + fi +done + +if [[ -n "$SDL_WKR_PID" ]]; then + echo "ERROR: SDL Worker is already running with PID ${SDL_WKR_PID}" + echo " Stop it first: kill ${SDL_WKR_PID}" + exit 1 +fi + +# Verify paths exist +if [[ ! -f "$NODE_BIN" ]]; then + echo "ERROR: Node.js not found at: $NODE_BIN" + echo " Run install-sdl.sh first" + exit 1 +fi + +if [[ ! -d "$SDL_WKR_APP_DIR" ]]; then + echo "ERROR: SDL Worker app not found at: $SDL_WKR_APP_DIR" + exit 1 +fi + +echo "Starting SDL Worker..." +echo " App Dir: ${SDL_WKR_APP_DIR}" +echo " Node: ${NODE_BIN}" + +export PATH="${NODE_BIN_DIR}:${PATH}" +cd "${SDL_WKR_APP_DIR}" + +if use_systemd; then + systemctl --user start sdl-wkr +else + # Manual background start logic + "${NPM_BIN}" start & +fi \ No newline at end of file diff --git a/sdl-wkr/scripts/status-sdl-wkr.sh b/sdl-wkr/scripts/status-sdl-wkr.sh new file mode 100755 index 0000000..1f86a4a --- /dev/null +++ b/sdl-wkr/scripts/status-sdl-wkr.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# status-sdl-wkr.sh - Script to show status of SDL Worker +set -euo pipefail + +use_systemd() { + # Check if systemd user mode is available and service exists + if systemctl --user status >/dev/null 2>&1; then + # Check if our service file exists + if systemctl --user list-unit-files | grep -q "^sdl-wkr.service"; then + return 0 # Use systemd + fi + fi + return 1 # Use manual scripts +} + +if use_systemd; then + systemctl --user status sdl-wkr +else + # Get sdl-wkr PID + SDL_MGR_PID="" + for pid in $(pgrep -f 'node' 2>/dev/null || true); do + cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true) + if [[ "$cwd" == *"sdl-wkr"* ]]; then + SDL_MGR_PID="$pid" + break + fi + done + + if [[ -z "$SDL_MGR_PID" ]]; then + echo "SDL Worker is not running" + exit 0 + fi + + echo "SDL Worker is running with PID: ${SDL_MGR_PID}" + exit 0 +fi \ No newline at end of file diff --git a/sdl-wkr/scripts/stop-sdl-wkr.sh b/sdl-wkr/scripts/stop-sdl-wkr.sh new file mode 100755 index 0000000..7cba2e2 --- /dev/null +++ b/sdl-wkr/scripts/stop-sdl-wkr.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# stop-sdl-wkr.sh - Script to stop SDL Worker +set -euo pipefail + +use_systemd() { + # Check if systemd user mode is available and service exists + if systemctl --user status >/dev/null 2>&1; then + # Check if our service file exists + if systemctl --user list-unit-files | grep -q "^sdl-wkr.service"; then + return 0 # Use systemd + fi + fi + return 1 # Use manual scripts +} + +# Get sdl-wkr PID +SDL_WKR_PID="" +for pid in $(pgrep -f 'node' 2>/dev/null || true); do + cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true) + if [[ "$cwd" == *"sdl-wkr"* ]]; then + SDL_WKR_PID="$pid" + break + fi +done + +if [[ -z "$SDL_WKR_PID" ]]; then + echo "SDL Worker is not running" + exit 0 +fi + +echo "Stopping SDL Worker (PID: ${SDL_WKR_PID})..." + +if use_systemd; then + systemctl --user stop sdl-wkr +else + # Manual PID-based stop logic + # Send SIGTERM for graceful shutdown + kill "$SDL_WKR_PID" + + # Wait up to 10 seconds for process to exit + for i in {1..10}; do + if ! kill -0 "$SDL_WKR_PID" 2>/dev/null; then + echo "SDL Worker stopped successfully" + exit 0 + fi + sleep 1 + done + + # If still running after 10 seconds, force kill + if kill -0 "$SDL_WKR_PID" 2>/dev/null; then + echo "WARNING: Graceful shutdown failed, forcing kill..." + kill -9 "$SDL_WKR_PID" + sleep 1 + + if kill -0 "$SDL_WKR_PID" 2>/dev/null; then + echo "ERROR: Failed to stop SDL Worker" + exit 1 + fi + fi +fi + +echo "SDL Worker stopped" \ No newline at end of file