Compare commits
28 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
ed188e9cb0 | |
|
|
266adae45a | |
|
|
822e7c46e3 | |
|
|
6fa62c6d47 | |
|
|
b69513019b | |
|
|
b63e038f10 | |
|
|
25a69586db | |
|
|
dae2fd6000 | |
|
|
74ced96f26 | |
|
|
0edd5841d6 | |
|
|
425821bce7 | |
|
|
7c73b68981 | |
|
|
574c28c81d | |
|
|
211cdcc53c | |
|
|
0cb6fb8aa3 | |
|
|
78b3ae2763 | |
|
|
4d62549a55 | |
|
|
456ac1761b | |
|
|
8b00086e9f | |
|
|
d4d15effd5 | |
|
|
0b5960f61e | |
|
|
ee3b315a98 | |
|
|
b84f4475e0 | |
|
|
2b7fc38ca2 | |
|
|
9eee364202 | |
|
|
a4c3e04684 | |
|
|
a2835873fc | |
|
|
f23aa1c60f |
|
|
@ -0,0 +1 @@
|
||||||
|
docs/.obsidian
|
||||||
22
README.md
22
README.md
|
|
@ -5,8 +5,8 @@
|
||||||
| **Author** | John Haverlack |
|
| **Author** | John Haverlack |
|
||||||
| **Copyright** | 2026 John Haverlack |
|
| **Copyright** | 2026 John Haverlack |
|
||||||
| **License** | MIT |
|
| **License** | MIT |
|
||||||
| **Version** | 0.3.7 |
|
| **Version** | 0.3.9 |
|
||||||
| **Date** | 2026-02-01 |
|
| **Date** | 2026-02-08 |
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
|
|
@ -14,6 +14,9 @@ The Software Defined Laboratory (SDL) project provides a generalized distributed
|
||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
|

|
||||||
|

|
||||||
|
|
||||||
For more information, see the [Design](docs/DESIGN.md) document.
|
For more information, see the [Design](docs/DESIGN.md) document.
|
||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
@ -47,7 +50,20 @@ Listen to UDP 10101 for beacon messages broadcast by your SDL Manager.
|
||||||
nc -u -l -k 10101 |jq
|
nc -u -l -k 10101 |jq
|
||||||
```
|
```
|
||||||
|
|
||||||
> NOTE: CTRL-C to exit `nc`. You cannot run `nc` on port 10101 in parallel with the sdl-wkr. So run it once to get the install command. Tnen install sdl-wkr.
|
Or to get the install command from the beacon message:
|
||||||
|
|
||||||
|
```
|
||||||
|
nc -u -l -k 10101 | jq -r '.msg.sdl_wkr_install_cmd[]'
|
||||||
|
```
|
||||||
|
|
||||||
|
**NOTE**: If you use UFW or a local firewall, you must allow SDL Worker to listen on UDP port 10101.
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo ufw allow in proto udp from <NETWORK CIDR> to any port 10101
|
||||||
|
```
|
||||||
|
|
||||||
|
> **NOTE**: CTRL-C to exit `nc`. You cannot run `nc` on port 10101 in parallel with the sdl-wkr. So run it once to get the install command. Tnen install sdl-wkr.
|
||||||
|
|
||||||
|
|
||||||
Run the **curl** or **wget** command provided by SDL Manager UDP beacon to install SDL Worker locally.
|
Run the **curl** or **wget** command provided by SDL Manager UDP beacon to install SDL Worker locally.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,19 @@
|
||||||
{
|
{
|
||||||
"releases": [
|
"releases": [
|
||||||
|
{
|
||||||
|
"version": "0.3.9",
|
||||||
|
"date": "2026-02-08",
|
||||||
|
"maturity": "ALPHA",
|
||||||
|
"summary": "",
|
||||||
|
"notes": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "0.3.8",
|
||||||
|
"date": "2026-02-01",
|
||||||
|
"maturity": "ALPHA",
|
||||||
|
"summary": "",
|
||||||
|
"notes": ""
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"version": "0.3.7",
|
"version": "0.3.7",
|
||||||
"date": "2026-02-01",
|
"date": "2026-02-01",
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,13 @@
|
||||||
"DIRS": {
|
"DIRS": {
|
||||||
"SDL_HOME": "~/.sdl",
|
"SDL_HOME": "~/.sdl",
|
||||||
"SDL_CONF": "SDL_HOME/conf",
|
"SDL_CONF": "SDL_HOME/conf",
|
||||||
|
"SDL_DOCS": "SDL_HOME/docs",
|
||||||
"SDL_LOGS": "SDL_HOME/logs",
|
"SDL_LOGS": "SDL_HOME/logs",
|
||||||
"SDL_DIST": "SDL_HOME/dist",
|
"SDL_DIST": "SDL_HOME/dist",
|
||||||
"SDL_DATA": "SDL_HOME/data",
|
"SDL_DATA": "SDL_HOME/data",
|
||||||
"SDL_PROJ": "SDL_DATA/proj",
|
"SDL_PROJ": "SDL_DATA/proj",
|
||||||
"SDL_JOBS": "SDL_DATA/jobs",
|
"SDL_JOBS": "SDL_DATA/jobs",
|
||||||
|
"SDL_SCRIPTS": "SDL_HOME/scripts",
|
||||||
"SDL_MGR": "SDL_HOME/sdl-mgr",
|
"SDL_MGR": "SDL_HOME/sdl-mgr",
|
||||||
"SDL_WKR": "SDL_HOME/sdl-wkr",
|
"SDL_WKR": "SDL_HOME/sdl-wkr",
|
||||||
"SDL_NODEJS": "SDL_HOME/nodejs"
|
"SDL_NODEJS": "SDL_HOME/nodejs"
|
||||||
|
|
|
||||||
|
|
@ -5,14 +5,46 @@
|
||||||
| **Author** | John Haverlack |
|
| **Author** | John Haverlack |
|
||||||
| **Copyright** | 2026 John Haverlack |
|
| **Copyright** | 2026 John Haverlack |
|
||||||
| **License** | MIT |
|
| **License** | MIT |
|
||||||
| **Version** | 0.3.7 |
|
| **Version** | 0.3.9 |
|
||||||
| **Date** | 2026-02-01 |
|
| **Date** | 2026-02-08 |
|
||||||
|
|
||||||
|
## v0.3.9 - 2026-02-08 (ALPHA)
|
||||||
|
|
||||||
|
- Refactor Cluster Status Dash
|
||||||
|
- Refactor Cluster Status Dash
|
||||||
|
- System Columnn Display
|
||||||
|
- Start v0.3.9
|
||||||
|
|
||||||
|
## v0.3.8 - 2026-02-01 (ALPHA)
|
||||||
|
|
||||||
|
- v0.3.8
|
||||||
|
- Refactoing Worker Dashboard
|
||||||
|
- Refactoing Worker Dashboard
|
||||||
|
- Refactored Cluster Status
|
||||||
|
- Refactor of telemetry format + hw detection
|
||||||
|
- Switched Panel Icon to Bootstrap Icon
|
||||||
|
- Updating Worker Telemetry
|
||||||
|
- CP
|
||||||
|
- Fixing Wget/Curl Command Installer Port
|
||||||
|
- Fixing Wget/Curl Command Installer Port
|
||||||
|
- Fixing Wkr Start / Stop
|
||||||
|
- Update stop-sdl-wkr.sh
|
||||||
|
- WIP Install / Start / Stop
|
||||||
|
- WIP Install / Start /Stop
|
||||||
|
- Systemd Detectiion in Start/Stop/Status
|
||||||
|
- WIP: Start / Stop Scripts
|
||||||
|
- Non Systemd start stop scripts
|
||||||
|
- WIP: start stop scripts
|
||||||
|
- Updating Task List
|
||||||
|
- Adding Screeshot to README
|
||||||
|
- Start v0.3.8
|
||||||
|
|
||||||
## v0.3.7 - 2026-02-01 (ALPHA)
|
## v0.3.7 - 2026-02-01 (ALPHA)
|
||||||
|
|
||||||
**Summary**
|
**Summary**
|
||||||
sdl-wkr Telemetry
|
sdl-wkr Telemetry
|
||||||
|
|
||||||
|
- v0.3.7
|
||||||
- Fixing sdl-wkr RAM detection
|
- Fixing sdl-wkr RAM detection
|
||||||
- Start v0.3.7
|
- Start v0.3.7
|
||||||
|
|
||||||
|
|
|
||||||
119
docs/DESIGN.md
119
docs/DESIGN.md
|
|
@ -5,8 +5,8 @@
|
||||||
| **Author** | John Haverlack |
|
| **Author** | John Haverlack |
|
||||||
| **Copyright** | 2026 John Haverlack |
|
| **Copyright** | 2026 John Haverlack |
|
||||||
| **License** | MIT |
|
| **License** | MIT |
|
||||||
| **Version** | 0.3.7 |
|
| **Version** | 0.3.9 |
|
||||||
| **Date** | 2026-02-01 |
|
| **Date** | 2026-02-08 |
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
|
|
@ -14,31 +14,118 @@ The Software Defined Laboratory (SDL) project provides a generalized distributed
|
||||||
|
|
||||||
# Roadmap
|
# Roadmap
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
- [x] sdl-mgr MQTT Broker
|
|
||||||
- [x] sdl-mgr UDP Beacon
|
|
||||||
- [x] sdl-mgr Web UI MQTT WebSocket Pub/Sub
|
|
||||||
- [x] sdl-wkr UDP Listener
|
|
||||||
|
|
||||||
|
|
||||||
## Prioritized Tasks
|
## Prioritized Tasks
|
||||||
|
|
||||||
MVP - Minimum Viable Product Task List
|
MVP - Minimum Viable Product Task List
|
||||||
|
|
||||||
|
**Core Infrastructure** ✅ (Complete)
|
||||||
|
- [x] sdl-mgr MQTT Broker
|
||||||
|
- [x] sdl-mgr UDP Beacon
|
||||||
|
- [x] sdl-mgr Web UI MQTT WebSocket Pub/Sub
|
||||||
|
- [x] sdl-wkr UDP Listener
|
||||||
- [x] SDL_ID Node ID
|
- [x] SDL_ID Node ID
|
||||||
- [x] Version Updater Script
|
- [x] Version Updater Script
|
||||||
- [x] Deploy sld-mgr to $SDL_HOME/sdl-mgr
|
- [x] Deploy sdl-mgr to $SDL_HOME/sdl-mgr
|
||||||
- [x] Create SDL Worker Dist Build Process
|
- [x] Create SDL Worker Dist Build Process
|
||||||
- [x] Create SDL Worker Install to $SDL_HOME
|
- [x] Create SDL Worker Install to $SDL_HOME
|
||||||
- [x] SDL Install Script
|
- [x] SDL Install Script
|
||||||
- [ ] Create SDL Worker Auto Update Process
|
- [x] Create SDL Worker Auto Update Process
|
||||||
- [x] Create Worker Join Process
|
- [x] Create Worker Join Process
|
||||||
- [ ] Create Worker Telementry Process
|
- [x] Create Worker Telemetry Process
|
||||||
|
|
||||||
|
**Worker Monitoring & Telemetry** 🔄 (In Progress)
|
||||||
|
- [x] Worker Hardware Inventory Detection
|
||||||
|
- [x] Physical vs Logical CPU cores (hyperthreading)
|
||||||
|
- [x] Physical vs VM detection (hypervisor flag)
|
||||||
|
- [x] GPU VRAM capacity aggregation
|
||||||
|
- [x] Memory capacity (bytes → GB conversion)
|
||||||
|
- [x] Worker Real-time Load Status (pure Node.js)
|
||||||
|
- [x] CPU usage percentage (`os.cpus()` times)
|
||||||
|
- [x] RAM usage (total/free/used via `os.totalmem()/freemem()`)
|
||||||
|
- [x] GPU usage (exec-based: nvidia-smi, rocm-smi)
|
||||||
|
- [ ] Disk IO (optional: exec-based)
|
||||||
|
- [x] Worker Uptime Tracking
|
||||||
|
- [x] SDL process uptime (`process.uptime()`)
|
||||||
|
- [x] OS uptime (`os.uptime()`)
|
||||||
|
- [ ] Worker Display Improvements
|
||||||
|
- [x] Show OS distro + version in Platform column (e.g., "Debian 12 / x64")
|
||||||
|
- [x] Show system type (Physical 🖥️ / VM 💠)
|
||||||
|
- [ ] Show GPU VRAM in GPU column (e.g., "2 GPUs (48 GB)")
|
||||||
|
- [x] Show worker uptime in dashboard
|
||||||
|
- [x] Display last seen as elapsed time in dashboard, not date
|
||||||
|
- [ ] Track Username for sdl process
|
||||||
|
- [ ] Web UI
|
||||||
|
- [ ] Progresive Web App
|
||||||
|
- [ ] Editable Dashboard
|
||||||
|
- [x] Panel Icon
|
||||||
|
|
||||||
|
|
||||||
|
**Resource Management**
|
||||||
|
- [ ] CPU Reserve Capacity Allocation
|
||||||
|
- [ ] Config option: `reserve_cpus: N` (keep N cores for system)
|
||||||
|
- [ ] Track: `total`, `reserved`, `allocated`, `available`, `used`
|
||||||
|
- [ ] Display reserved capacity in dashboard
|
||||||
|
- [ ] Memory Reserve Capacity (optional, future)
|
||||||
|
- [ ] GPU Reserve Capacity (optional, future)
|
||||||
|
|
||||||
|
**Service Management**
|
||||||
|
- [x] Start/Stop shell scripts (for systems without systemd user services)
|
||||||
|
- [x] `start-sdl-mgr.sh` / `stop-sdl-mgr.sh`
|
||||||
|
- [x] `start-sdl-wkr.sh` / `stop-sdl-wkr.sh`
|
||||||
|
- [ ] Combined systemd service
|
||||||
|
- [ ] `sdl.service` to manage both sdl-mgr and sdl-wkr together
|
||||||
|
- [ ] Support for system-level (sudo) and user-level services
|
||||||
|
- [ ] SDL-MGR auto-installs SDL-WKR
|
||||||
|
- [ ] Manager node becomes worker by default
|
||||||
|
- [ ] Config flag to disable: `install_worker_on_manager: false
|
||||||
|
- [x] Pass HTTP Port to SDL-WKR Curl installer
|
||||||
|
- [ ] Update Install Summary
|
||||||
|
- [ ] Add Quite Mode to sdl-wkr install
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
**Data & Storage**
|
||||||
- [ ] Data Storage Organizational Structure
|
- [ ] Data Storage Organizational Structure
|
||||||
- [ ] MinIO S3 Storage Server
|
- [ ] Define directory hierarchy for tasks, results, logs
|
||||||
- [ ] Worker Telemetry Hardware Inventory (CPU, RAM, GPU, etc)
|
- [ ] Implement file naming conventions
|
||||||
- [ ] Worker Telementry Load Status (CPU, RAM, GPU, Disk IO, etc)
|
- [ ] Add cleanup/retention policies
|
||||||
|
- [ ] MinIO S3 Storage Server Integration
|
||||||
|
- [ ] Install and configure MinIO
|
||||||
|
- [ ] S3 bucket structure for SDL data
|
||||||
|
- [ ] Worker access to S3 storage
|
||||||
|
- [ ] Web UI S3 browser integration
|
||||||
|
|
||||||
|
**Projects**
|
||||||
|
- [ ] Project Management Org Structure
|
||||||
|
- [ ] Define project metadata schema
|
||||||
|
- [ ] Project lifecycle states (draft, active, archived)
|
||||||
|
- [ ] Permission/access control model
|
||||||
|
- [ ] Experiment Organization under Org Structure
|
||||||
|
- [ ] Experiment templates
|
||||||
|
- [ ] Parameter tracking
|
||||||
|
- [ ] Result linking to experiments
|
||||||
|
- [ ] HPC Experiment Binary Development
|
||||||
|
- [ ] sdl-wkr Capablity detection (phyton, etc)
|
||||||
|
- [ ] Binary packaging format
|
||||||
|
- [ ] Dependency management
|
||||||
|
- [ ] Version control for binaries
|
||||||
|
|
||||||
|
**Dashboard Enhancements**
|
||||||
|
- [x] Dark/Light theme polish (CSS refinements)
|
||||||
|
- [ ] Real-time resource graphs (Chart.js)
|
||||||
|
- [ ] Worker status history timeline
|
||||||
|
- [ ] Cluster health indicators
|
||||||
|
- [ ] Alert/notification system for worker failures
|
||||||
|
- [ ] Add a Clock to the Dashboard (ISO 8601)
|
||||||
|
|
||||||
|
**Testing & Documentation**
|
||||||
|
- [ ] Cross-platform testing (Debian, Ubuntu, Arch, Fedora, Win, OSx, BSD)
|
||||||
|
- [ ] Installation documentation
|
||||||
|
- [ ] Configuration examples
|
||||||
|
- [ ] Troubleshooting guide
|
||||||
|
- [ ] Validate IP change Stablity
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
# Design
|
# Design
|
||||||
|
|
||||||
|
|
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 109 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 140 KiB |
|
|
@ -53,6 +53,13 @@ resolve_dirs() {
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
use_systemd() {
|
||||||
|
# Check if systemd user mode is available and service exists
|
||||||
|
if systemctl --user status >/dev/null 2>&1; then
|
||||||
|
return 0 # Use systemd
|
||||||
|
fi
|
||||||
|
return 1 # Use manual scripts
|
||||||
|
}
|
||||||
|
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
# Main
|
# Main
|
||||||
|
|
@ -69,7 +76,7 @@ fi
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
# Requirements
|
# Requirements
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
for cmd in awk curl grep jq sed sha256sum tar unzip; do
|
for cmd in awk curl grep jq nc sed sha256sum tar unzip; do
|
||||||
command -v "$cmd" >/dev/null || {
|
command -v "$cmd" >/dev/null || {
|
||||||
echo "ERROR: $cmd is required"
|
echo "ERROR: $cmd is required"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
@ -160,13 +167,8 @@ fi
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
# systemd user service check
|
# Setting SDL variables
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
if ! systemctl --user status >/dev/null 2>&1; then
|
|
||||||
echo "ERROR: systemd user services are not available"
|
|
||||||
echo " SDL requires systemd --user support"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SDL_VERSION="$(jq -r '.METADATA.version' "$META_FILE")"
|
SDL_VERSION="$(jq -r '.METADATA.version' "$META_FILE")"
|
||||||
SDL_VERSION_DATE="$(jq -r '.METADATA.version_date' "$META_FILE")"
|
SDL_VERSION_DATE="$(jq -r '.METADATA.version_date' "$META_FILE")"
|
||||||
|
|
@ -384,32 +386,52 @@ cd "$SDL_MGR/current/app"
|
||||||
export PATH="$SDL_NODEJS/current/bin:$PATH"
|
export PATH="$SDL_NODEJS/current/bin:$PATH"
|
||||||
$NPM_BIN install
|
$NPM_BIN install
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Copy Start / Stop Scripts
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
mkdir -p "$SDL_HOME/scripts/"
|
||||||
|
cp "$SDL_MGR/current/scripts/start-sdl-mgr.sh" "$SDL_HOME/scripts/"
|
||||||
|
cp "$SDL_MGR/current/scripts/stop-sdl-mgr.sh" "$SDL_HOME/scripts/"
|
||||||
|
cp "$SDL_MGR/current/scripts/status-sdl-mgr.sh" "$SDL_HOME/scripts/"
|
||||||
|
chmod u+x "$SDL_HOME/scripts/"*
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
# Install Systemd Unit Files
|
# Install Systemd Unit Files
|
||||||
# Copy SDL_MGR/current/scripts/sdl-mgr.service to ~/.config/systemd/user
|
# Copy SDL_MGR/current/scripts/sdl-mgr.service to ~/.config/systemd/user
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
echo ""
|
if use_systemd; then
|
||||||
echo "Installing SystemD Unit Files"
|
echo ""
|
||||||
|
echo "Installing SystemD Unit Files"
|
||||||
|
|
||||||
mkdir -p ~/.config/systemd/user
|
mkdir -p ~/.config/systemd/user
|
||||||
cp "$SDL_MGR/current/scripts/sdl-mgr.service" ~/.config/systemd/user/
|
cp "$SDL_MGR/current/scripts/sdl-mgr.service" ~/.config/systemd/user/
|
||||||
|
|
||||||
|
"${SDL_HOME}/scripts/stop-sdl-mgr.sh"
|
||||||
|
sleep 1
|
||||||
|
systemctl --user daemon-reload --no-pager
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Start SDL Manager
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
"${SDL_HOME}/scripts/start-sdl-mgr.sh"
|
||||||
|
sleep 2
|
||||||
|
"${SDL_HOME}/scripts/status-sdl-mgr.sh"
|
||||||
|
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "ERROR: SDL Manager failed to start"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
#
|
||||||
|
|
||||||
systemctl --user daemon-reload --no-pager
|
|
||||||
systemctl --user restart sdl-mgr --no-pager
|
|
||||||
sleep 3
|
|
||||||
systemctl --user status sdl-mgr --no-pager
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "SDL Manager deployed:"
|
echo "SDL Manager deployed:"
|
||||||
echo " to: $SDL_HOME"
|
echo " to: $SDL_HOME"
|
||||||
|
|
||||||
echo "SDL Controls:"
|
|
||||||
echo " systemctl --user status sdl-mgr"
|
|
||||||
echo " systemctl --user start sdl-mgr"
|
|
||||||
echo " systemctl --user stop sdl-mgr"
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "Removal Commands:"
|
|
||||||
echo " WARNING: Backup data directory first!!!!"
|
|
||||||
echo " systemctl --user disable sdl-mgr"
|
|
||||||
echo " rm -rf $SDL_HOME"
|
|
||||||
|
|
@ -3,8 +3,8 @@
|
||||||
"name": "sdl",
|
"name": "sdl",
|
||||||
"description": "Software Defined Laboratory",
|
"description": "Software Defined Laboratory",
|
||||||
"abbr": "SDL",
|
"abbr": "SDL",
|
||||||
"version": "0.3.7",
|
"version": "0.3.9",
|
||||||
"version_date": "2026-02-01",
|
"version_date": "2026-02-08",
|
||||||
"maturity": "ALPHA",
|
"maturity": "ALPHA",
|
||||||
"author": "John Haverlack",
|
"author": "John Haverlack",
|
||||||
"copyright": "2026 John Haverlack",
|
"copyright": "2026 John Haverlack",
|
||||||
|
|
|
||||||
|
|
@ -605,7 +605,7 @@ function getUptimeDHMS() {
|
||||||
const h = Math.floor((uptimeSec % 86400) / 3600);
|
const h = Math.floor((uptimeSec % 86400) / 3600);
|
||||||
const m = Math.floor((uptimeSec % 3600) / 60);
|
const m = Math.floor((uptimeSec % 3600) / 60);
|
||||||
const s = Math.floor(uptimeSec % 60);
|
const s = Math.floor(uptimeSec % 60);
|
||||||
return `${d}:${h}:${m}:${s}`;
|
return `${d}d ${h}h ${m}m ${s}s`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,18 @@
|
||||||
const module = 'sdl-mgr'; // Module Name
|
const module = 'sdl-mgr'; // Module Name
|
||||||
|
|
||||||
import { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeDHMS } from '../nwa-lib/index.js';
|
import { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeDHMS, getUptimeSec } from '../nwa-lib/index.js';
|
||||||
import mqtt from 'mqtt';
|
import mqtt from 'mqtt';
|
||||||
import os from 'os';
|
import os from 'os';
|
||||||
import dgram from 'dgram';
|
import dgram from 'dgram';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
const config = load_config();
|
const config = load_config();
|
||||||
|
|
||||||
log(`Loaded module: ${module}`);
|
log(`Loaded module: ${module}`);
|
||||||
// log(`${module}: Cluster Conf: ${JSON.stringify(config.cluster, null, 2)}`, true);
|
|
||||||
// log(`${module}: Dirs: ${JSON.stringify(config.dirs, null, 2)}`, true);
|
// ✅ In-memory workers state (built from telemetry only)
|
||||||
|
let workersState = {};
|
||||||
|
|
||||||
function loadClusterState(config) {
|
function loadClusterState(config) {
|
||||||
const clusterDir = config.dirs.clstr;
|
const clusterDir = config.dirs.clstr;
|
||||||
|
|
@ -30,31 +29,7 @@ function loadClusterState(config) {
|
||||||
meta: {
|
meta: {
|
||||||
updated: new Date().toISOString(),
|
updated: new Date().toISOString(),
|
||||||
started_at: new Date(getProcessStartTs()).toISOString(),
|
started_at: new Date(getProcessStartTs()).toISOString(),
|
||||||
uptime: getUptimeDHMS(),
|
uptime: getUptimeDHMS()
|
||||||
stats: {
|
|
||||||
workers: {
|
|
||||||
allocated: 0,
|
|
||||||
available: 0,
|
|
||||||
used: 0
|
|
||||||
},
|
|
||||||
resources: {
|
|
||||||
cpus: {
|
|
||||||
allocated: 0,
|
|
||||||
available: 0,
|
|
||||||
used: 0
|
|
||||||
},
|
|
||||||
memory: {
|
|
||||||
allocated: 0,
|
|
||||||
available: 0,
|
|
||||||
used: 0
|
|
||||||
},
|
|
||||||
gpus: {
|
|
||||||
allocated: 0,
|
|
||||||
available: 0,
|
|
||||||
used: 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
cluster: config.cluster,
|
cluster: config.cluster,
|
||||||
"sdl-mgr": {
|
"sdl-mgr": {
|
||||||
|
|
@ -65,8 +40,7 @@ function loadClusterState(config) {
|
||||||
distro: config.host.os.pretty_name,
|
distro: config.host.os.pretty_name,
|
||||||
distro_name: config.host.os.name,
|
distro_name: config.host.os.name,
|
||||||
distro_version: config.host.os.version
|
distro_version: config.host.os.version
|
||||||
},
|
}
|
||||||
workers: {}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
fs.writeFileSync(clusterFile, JSON.stringify(initialState, null, 2));
|
fs.writeFileSync(clusterFile, JSON.stringify(initialState, null, 2));
|
||||||
|
|
@ -89,55 +63,15 @@ function saveClusterState(config, state) {
|
||||||
// Write to disk
|
// Write to disk
|
||||||
try {
|
try {
|
||||||
fs.writeFileSync(clusterFile, JSON.stringify(state, null, 2));
|
fs.writeFileSync(clusterFile, JSON.stringify(state, null, 2));
|
||||||
log(`${module}: cluster state saved to ${clusterFile}`);
|
// log(`${module}: cluster state saved to ${clusterFile}`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log(`${module}: failed to save cluster state: ${err}`);
|
log(`${module}: failed to save cluster state: ${err}`);
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function addWorker(state, workerData) {
|
// ✅ Compute stats from in-memory workers state
|
||||||
const sdl_id = workerData.sdl_id;
|
function computeStats() {
|
||||||
|
|
||||||
// Check if worker already exists
|
|
||||||
const isNewWorker = !state.workers[sdl_id];
|
|
||||||
|
|
||||||
// Add or update worker
|
|
||||||
state.workers[sdl_id] = {
|
|
||||||
sdl_id: workerData.sdl_id,
|
|
||||||
hostname: workerData.hostname,
|
|
||||||
version: workerData.sdl_version,
|
|
||||||
platform: workerData.platform,
|
|
||||||
arch: workerData.arch,
|
|
||||||
distro: workerData.distro || "Unknown",
|
|
||||||
distro_name: workerData.distro_name || "Unknown",
|
|
||||||
distro_version: workerData.distro_version || "Unknown",
|
|
||||||
resources: {
|
|
||||||
cpus: {
|
|
||||||
allocated: workerData.cpus || 0,
|
|
||||||
available: workerData.cpus || 0,
|
|
||||||
used: 0
|
|
||||||
},
|
|
||||||
memory: {
|
|
||||||
allocated: workerData.totalmem || 0,
|
|
||||||
available: workerData.totalmem || 0,
|
|
||||||
used: 0
|
|
||||||
},
|
|
||||||
gpus: {
|
|
||||||
allocated: workerData.gpus || 0,
|
|
||||||
available: workerData.gpus || 0,
|
|
||||||
used: 0
|
|
||||||
}
|
|
||||||
},
|
|
||||||
status: 'active',
|
|
||||||
joined_at: isNewWorker ? new Date().toISOString() : state.workers[sdl_id].joined_at,
|
|
||||||
last_seen: new Date().toISOString()
|
|
||||||
};
|
|
||||||
|
|
||||||
return state;
|
|
||||||
}
|
|
||||||
|
|
||||||
function computeStats(state) {
|
|
||||||
const stats = {
|
const stats = {
|
||||||
workers: {
|
workers: {
|
||||||
allocated: 0,
|
allocated: 0,
|
||||||
|
|
@ -164,29 +98,28 @@ function computeStats(state) {
|
||||||
};
|
};
|
||||||
|
|
||||||
// Count workers and aggregate resources
|
// Count workers and aggregate resources
|
||||||
for (const worker of Object.values(state.workers)) {
|
for (const worker of Object.values(workersState)) {
|
||||||
stats.workers.allocated++;
|
stats.workers.allocated++;
|
||||||
|
|
||||||
if (worker.status === 'active') {
|
if (worker.status === 'active') {
|
||||||
stats.workers.available++;
|
stats.workers.available++;
|
||||||
|
|
||||||
stats.resources.cpus.available += worker.resources.cpus.available;
|
stats.resources.cpus.available += worker.resources?.cpus?.available || 0;
|
||||||
stats.resources.memory.available += worker.resources.memory.available;
|
stats.resources.memory.available += worker.resources?.memory?.available || 0;
|
||||||
stats.resources.gpus.available += worker.resources.gpus.available;
|
stats.resources.gpus.available += worker.resources?.gpus?.available || 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
stats.resources.cpus.allocated += worker.resources.cpus.allocated;
|
stats.resources.cpus.allocated += worker.resources?.cpus?.allocated || 0;
|
||||||
stats.resources.cpus.used += worker.resources.cpus.used;
|
stats.resources.cpus.used += worker.usage?.cpu?.total || 0;
|
||||||
|
|
||||||
stats.resources.memory.allocated += worker.resources.memory.allocated;
|
stats.resources.memory.allocated += worker.resources?.memory?.allocated || 0;
|
||||||
stats.resources.memory.used += worker.resources.memory.used;
|
stats.resources.memory.used += worker.usage?.memory?.used || 0;
|
||||||
|
|
||||||
stats.resources.gpus.allocated += worker.resources.gpus.allocated;
|
stats.resources.gpus.allocated += worker.resources?.gpus?.allocated || 0;
|
||||||
stats.resources.gpus.used += worker.resources.gpus.used;
|
stats.resources.gpus.used += worker.usage?.gpu?.total_utilization || 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.meta.stats = stats;
|
return stats;
|
||||||
return state;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Worker Join Handler
|
// Worker Join Handler
|
||||||
|
|
@ -204,9 +137,6 @@ function startJoinHandler() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load current cluster state
|
|
||||||
let clusterState = loadClusterState(config);
|
|
||||||
|
|
||||||
const joinReqTopic = mqttCfg.topics?.[module]?.sub?.['sdl_join-req'];
|
const joinReqTopic = mqttCfg.topics?.[module]?.sub?.['sdl_join-req'];
|
||||||
const joinAuthzTopic = mqttCfg.topics?.[module]?.pub?.['sdl_join-authz'];
|
const joinAuthzTopic = mqttCfg.topics?.[module]?.pub?.['sdl_join-authz'];
|
||||||
|
|
||||||
|
|
@ -238,10 +168,8 @@ function startJoinHandler() {
|
||||||
|
|
||||||
log(`${module}: received join request from ${joinReq.host} (${joinReq.sdl_id})`);
|
log(`${module}: received join request from ${joinReq.host} (${joinReq.sdl_id})`);
|
||||||
|
|
||||||
// Validate version
|
|
||||||
const workerData = joinReq.msg['sdl-wkr'];
|
const workerData = joinReq.msg['sdl-wkr'];
|
||||||
const workerVersion = workerData?.sdl_version;
|
const workerVersion = workerData?.sdl_version;
|
||||||
// const workerVersion = joinReq.msg?.worker?.version;
|
|
||||||
const clusterVersion = config.package.version;
|
const clusterVersion = config.package.version;
|
||||||
|
|
||||||
let authorized = false;
|
let authorized = false;
|
||||||
|
|
@ -249,13 +177,7 @@ function startJoinHandler() {
|
||||||
|
|
||||||
if (workerVersion === clusterVersion) {
|
if (workerVersion === clusterVersion) {
|
||||||
authorized = true;
|
authorized = true;
|
||||||
|
log(`${module}: worker ${joinReq.host} authorized (version ${workerVersion})`);
|
||||||
// Add worker to cluster state
|
|
||||||
clusterState = addWorker(clusterState, workerData);
|
|
||||||
clusterState = computeStats(clusterState);
|
|
||||||
saveClusterState(config, clusterState);
|
|
||||||
|
|
||||||
log(`${module}: worker ${joinReq.host} authorized and added to cluster`);
|
|
||||||
} else {
|
} else {
|
||||||
authorized = false;
|
authorized = false;
|
||||||
reason = 'version_mismatch';
|
reason = 'version_mismatch';
|
||||||
|
|
@ -271,7 +193,8 @@ function startJoinHandler() {
|
||||||
type: 'join-authz',
|
type: 'join-authz',
|
||||||
msg: {
|
msg: {
|
||||||
sdl_id: joinReq.sdl_id,
|
sdl_id: joinReq.sdl_id,
|
||||||
authorized: authorized
|
authorized: authorized,
|
||||||
|
reason: reason
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -282,8 +205,6 @@ function startJoinHandler() {
|
||||||
err => {
|
err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log(`${module}: failed to publish join authz: ${err}`);
|
log(`${module}: failed to publish join authz: ${err}`);
|
||||||
} else {
|
|
||||||
// log(`${module}: published join authz to ${joinAuthzTopic}`);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
@ -298,8 +219,6 @@ function startJoinHandler() {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
function startSDLStatusPub() {
|
function startSDLStatusPub() {
|
||||||
const sdlCfg = config.modules[module];
|
const sdlCfg = config.modules[module];
|
||||||
const mqttCfg = config.modules.mqtt;
|
const mqttCfg = config.modules.mqtt;
|
||||||
|
|
@ -322,59 +241,49 @@ function startSDLStatusPub() {
|
||||||
|
|
||||||
const statusTopic = mqttCfg.topics?.[module]?.pub?.['sdl_cluster-status'];
|
const statusTopic = mqttCfg.topics?.[module]?.pub?.['sdl_cluster-status'];
|
||||||
if (!statusTopic) {
|
if (!statusTopic) {
|
||||||
log(`${module}: ${statusTopic} not configured`);
|
log(`${module}: cluster-status topic not configured`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// This is ok because we are running on localhost
|
|
||||||
const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`;
|
const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`;
|
||||||
const client = mqtt.connect(mqttUrl);
|
const client = mqtt.connect(mqttUrl);
|
||||||
|
|
||||||
client.on('connect', () => {
|
client.on('connect', () => {
|
||||||
log(`${module}: connected to MQTT at ${mqttUrl}`);
|
log(`${module}: status publisher connected to MQTT at ${mqttUrl}`);
|
||||||
|
|
||||||
publishStatus(client, statusTopic);
|
publishStatus(client, statusTopic);
|
||||||
setInterval(() => publishStatus(client, statusTopic), statusInterval);
|
setInterval(() => publishStatus(client, statusTopic), statusInterval);
|
||||||
});
|
});
|
||||||
|
|
||||||
client.on('error', err => {
|
client.on('error', err => {
|
||||||
log(`${module}: MQTT error: ${err}`);
|
log(`${module}: status publisher MQTT error: ${err}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function publishStatus(client, topic) {
|
function publishStatus(client, topic) {
|
||||||
// Load current cluster state
|
// Compute stats from in-memory workers
|
||||||
const clusterState = loadClusterState(config);
|
const stats = computeStats();
|
||||||
|
|
||||||
// Recompute stats to get latest resource totals
|
|
||||||
const updatedState = computeStats(clusterState);
|
|
||||||
|
|
||||||
//get ip addr
|
// Get IP for update command
|
||||||
const interfaces = os.networkInterfaces();
|
const interfaces = os.networkInterfaces();
|
||||||
let ip_addr = null;
|
let ip_addr = null;
|
||||||
|
|
||||||
for (const [iface, addrs] of Object.entries(interfaces)) {
|
for (const [iface, addrs] of Object.entries(interfaces)) {
|
||||||
for (const addr of addrs) {
|
for (const addr of addrs) {
|
||||||
// --- FILTERS ---
|
|
||||||
|
|
||||||
// IPv4 only
|
|
||||||
if (addr.family !== 'IPv4') continue;
|
if (addr.family !== 'IPv4') continue;
|
||||||
|
|
||||||
// Skip loopback
|
|
||||||
if (addr.internal === true) continue;
|
if (addr.internal === true) continue;
|
||||||
|
|
||||||
// Skip /32 networks (no broadcast: e.g. Tailscale)
|
|
||||||
const cidr = Number(addr.cidr?.split('/')[1]);
|
const cidr = Number(addr.cidr?.split('/')[1]);
|
||||||
if (!Number.isInteger(cidr) || cidr >= 32) continue;
|
if (!Number.isInteger(cidr) || cidr >= 32) continue;
|
||||||
|
|
||||||
// Compute broadcast address
|
ip_addr = addr.address;
|
||||||
const broadcastAddr = computeBroadcast(addr.address, addr.netmask);
|
break;
|
||||||
if (!broadcastAddr) continue;
|
|
||||||
|
|
||||||
ip_addr = addr.address
|
|
||||||
}
|
}
|
||||||
|
if (ip_addr) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const web_port = config.modules.web.port;
|
||||||
|
|
||||||
const status = {
|
const status = {
|
||||||
ts: new Date().toISOString(),
|
ts: new Date().toISOString(),
|
||||||
sdl_id: config.identity.sdl_id,
|
sdl_id: config.identity.sdl_id,
|
||||||
|
|
@ -384,12 +293,15 @@ function publishStatus(client, topic) {
|
||||||
msg: {
|
msg: {
|
||||||
sdl: {
|
sdl: {
|
||||||
version: config.package.version,
|
version: config.package.version,
|
||||||
|
uptime_secs: getUptimeSec(),
|
||||||
uptime: getUptimeDHMS(),
|
uptime: getUptimeDHMS(),
|
||||||
update_cmd: `curl -s http://${ip_addr}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${ip_addr}`
|
update_cmd: ip_addr
|
||||||
|
? `curl -s http://${ip_addr}:${web_port}/dist/install-sdl-wkr.sh | bash -s ${ip_addr}:${web_port}`
|
||||||
|
: null
|
||||||
},
|
},
|
||||||
cluster: config.cluster,
|
cluster: config.cluster,
|
||||||
resources: updatedState.meta.stats.resources, // ✅ Add cluster resources
|
resources: stats.resources,
|
||||||
workers: updatedState.meta.stats.workers, // ✅ Add worker counts
|
workers: stats.workers,
|
||||||
modules: Object.fromEntries(
|
modules: Object.fromEntries(
|
||||||
Object.entries(config.modules).map(([name, mod]) => [
|
Object.entries(config.modules).map(([name, mod]) => [
|
||||||
name,
|
name,
|
||||||
|
|
@ -406,15 +318,12 @@ function publishStatus(client, topic) {
|
||||||
err => {
|
err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log(`${module}: failed to publish status: ${err}`);
|
log(`${module}: failed to publish status: ${err}`);
|
||||||
} else {
|
|
||||||
// log(`${module}: published status to ${topic}`);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ✅ Publish workers from in-memory state
|
||||||
// ✅ NEW: Publish individual worker details
|
|
||||||
function startWorkersPub() {
|
function startWorkersPub() {
|
||||||
const sdlCfg = config.modules[module];
|
const sdlCfg = config.modules[module];
|
||||||
const mqttCfg = config.modules.mqtt;
|
const mqttCfg = config.modules.mqtt;
|
||||||
|
|
@ -457,9 +366,6 @@ function startWorkersPub() {
|
||||||
}
|
}
|
||||||
|
|
||||||
function publishWorkers(client, topic) {
|
function publishWorkers(client, topic) {
|
||||||
// Load current cluster state
|
|
||||||
const clusterState = loadClusterState(config);
|
|
||||||
|
|
||||||
const workersMsg = {
|
const workersMsg = {
|
||||||
ts: new Date().toISOString(),
|
ts: new Date().toISOString(),
|
||||||
sdl_id: config.identity.sdl_id,
|
sdl_id: config.identity.sdl_id,
|
||||||
|
|
@ -467,7 +373,7 @@ function publishWorkers(client, topic) {
|
||||||
host: config.identity.hostname,
|
host: config.identity.hostname,
|
||||||
type: 'cluster-workers',
|
type: 'cluster-workers',
|
||||||
msg: {
|
msg: {
|
||||||
workers: clusterState.workers
|
workers: workersState // ✅ Use in-memory state from telemetry
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -478,18 +384,13 @@ function publishWorkers(client, topic) {
|
||||||
err => {
|
err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
log(`${module}: failed to publish workers: ${err}`);
|
log(`${module}: failed to publish workers: ${err}`);
|
||||||
} else {
|
|
||||||
// log(`${module}: published workers to ${topic}`);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function startUdpBeacon() {
|
function startUdpBeacon() {
|
||||||
let sdlCfg = config.modules['sdl-mgr'];
|
let sdlCfg = config.modules['sdl-mgr'];
|
||||||
// log(`${module}: DEBUG: SDL config: ${JSON.stringify(sdlCfg, null, 2)}`, true);
|
|
||||||
// log(`${module}: DEBUG: Cluster Conf: ${JSON.stringify(config.cluster, null, 2)}`, true);
|
|
||||||
sdlCfg.cluster = config.cluster;
|
sdlCfg.cluster = config.cluster;
|
||||||
const mqttCfg = config.modules.mqtt;
|
const mqttCfg = config.modules.mqtt;
|
||||||
|
|
||||||
|
|
@ -520,7 +421,6 @@ function startUdpBeacon() {
|
||||||
};
|
};
|
||||||
|
|
||||||
log(`${module}: cluster: ${JSON.stringify(cluster)}`);
|
log(`${module}: cluster: ${JSON.stringify(cluster)}`);
|
||||||
|
|
||||||
|
|
||||||
if (!config.host?.network) {
|
if (!config.host?.network) {
|
||||||
log(`${module}: no host network metadata available for UDP beacon`);
|
log(`${module}: no host network metadata available for UDP beacon`);
|
||||||
|
|
@ -541,26 +441,17 @@ function startUdpBeacon() {
|
||||||
setInterval(() => {
|
setInterval(() => {
|
||||||
const interfaces = os.networkInterfaces();
|
const interfaces = os.networkInterfaces();
|
||||||
|
|
||||||
// for (const [iface, addrs] of Object.entries(config.host.network)) {
|
|
||||||
for (const [iface, addrs] of Object.entries(interfaces)) {
|
for (const [iface, addrs] of Object.entries(interfaces)) {
|
||||||
for (const addr of addrs) {
|
for (const addr of addrs) {
|
||||||
// --- FILTERS ---
|
|
||||||
|
|
||||||
// IPv4 only
|
|
||||||
if (addr.family !== 'IPv4') continue;
|
if (addr.family !== 'IPv4') continue;
|
||||||
|
|
||||||
// Skip loopback
|
|
||||||
if (addr.internal === true) continue;
|
if (addr.internal === true) continue;
|
||||||
|
|
||||||
// Skip /32 networks (no broadcast: e.g. Tailscale)
|
|
||||||
const cidr = Number(addr.cidr?.split('/')[1]);
|
const cidr = Number(addr.cidr?.split('/')[1]);
|
||||||
if (!Number.isInteger(cidr) || cidr >= 32) continue;
|
if (!Number.isInteger(cidr) || cidr >= 32) continue;
|
||||||
|
|
||||||
// Compute broadcast address
|
|
||||||
const broadcastAddr = computeBroadcast(addr.address, addr.netmask);
|
const broadcastAddr = computeBroadcast(addr.address, addr.netmask);
|
||||||
if (!broadcastAddr) continue;
|
if (!broadcastAddr) continue;
|
||||||
|
|
||||||
|
|
||||||
const beacon = {
|
const beacon = {
|
||||||
ts: new Date().toISOString(),
|
ts: new Date().toISOString(),
|
||||||
sdl_id: config.identity.sdl_id,
|
sdl_id: config.identity.sdl_id,
|
||||||
|
|
@ -591,8 +482,8 @@ function startUdpBeacon() {
|
||||||
web_ui_url: `http://${addr.address}:${config.modules.web.port}`,
|
web_ui_url: `http://${addr.address}:${config.modules.web.port}`,
|
||||||
},
|
},
|
||||||
sdl_wkr_install_cmd: {
|
sdl_wkr_install_cmd: {
|
||||||
curl: `curl -s http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}`,
|
curl: `curl -s http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}:${config.modules.web.port}`,
|
||||||
wget: `wget -O - http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}`
|
wget: `wget -O - http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}:${config.modules.web.port}`,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -620,9 +511,8 @@ function startUdpBeacon() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ✅ Build workers state from telemetry
|
||||||
// Update startHeartbeatListener to use telemetry topic
|
function startTelemetryListener() {
|
||||||
function startTelemetryListener() { // ✅ Renamed function
|
|
||||||
const sdlCfg = config.modules[module];
|
const sdlCfg = config.modules[module];
|
||||||
const mqttCfg = config.modules.mqtt;
|
const mqttCfg = config.modules.mqtt;
|
||||||
|
|
||||||
|
|
@ -636,9 +526,7 @@ function startTelemetryListener() { // ✅ Renamed function
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let clusterState = loadClusterState(config);
|
const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry'];
|
||||||
|
|
||||||
const telemetryTopic = mqttCfg.topics?.[module]?.sub?.['sdl_cluster-telemetry']; // ✅ Changed
|
|
||||||
if (!telemetryTopic) {
|
if (!telemetryTopic) {
|
||||||
log(`${module}: cluster-telemetry topic not configured`);
|
log(`${module}: cluster-telemetry topic not configured`);
|
||||||
return;
|
return;
|
||||||
|
|
@ -659,55 +547,29 @@ function startTelemetryListener() { // ✅ Renamed function
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
client.on('message', (topic, message) => {
|
client.on('message', (topic, message) => {
|
||||||
if (topic !== telemetryTopic) return;
|
if (topic !== telemetryTopic) return;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const telemetry = JSON.parse(message.toString());
|
const telemetry = JSON.parse(message.toString());
|
||||||
const sdl_id = telemetry.msg.sdl_id;
|
const sdl_id = telemetry.sdl_id;
|
||||||
|
const hostname = telemetry.host;
|
||||||
// Update worker's last_seen timestamp
|
const now = new Date();
|
||||||
if (clusterState.workers[sdl_id]) {
|
|
||||||
clusterState.workers[sdl_id].last_seen = new Date().toISOString();
|
|
||||||
clusterState.workers[sdl_id].status = 'active';
|
|
||||||
|
|
||||||
// ✅ MERGE resources instead of replacing
|
// ✅ Build/update worker state from telemetry
|
||||||
if (telemetry.msg.resources) {
|
workersState[sdl_id] = telemetry
|
||||||
// Merge each resource type
|
|
||||||
if (telemetry.msg.resources.cpus) {
|
|
||||||
clusterState.workers[sdl_id].resources.cpus = {
|
|
||||||
...clusterState.workers[sdl_id].resources.cpus,
|
|
||||||
...telemetry.msg.resources.cpus
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (telemetry.msg.resources.memory) {
|
|
||||||
clusterState.workers[sdl_id].resources.memory = {
|
|
||||||
...clusterState.workers[sdl_id].resources.memory,
|
|
||||||
...telemetry.msg.resources.memory
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (telemetry.msg.resources.gpus) {
|
|
||||||
clusterState.workers[sdl_id].resources.gpus = {
|
|
||||||
...clusterState.workers[sdl_id].resources.gpus,
|
|
||||||
...telemetry.msg.resources.gpus
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
clusterState = computeStats(clusterState);
|
} catch (err) {
|
||||||
saveClusterState(config, clusterState);
|
log(`${module}: failed to process telemetry: ${err}`);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
});
|
||||||
log(`${module}: failed to process telemetry: ${err}`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
client.on('error', err => {
|
client.on('error', err => {
|
||||||
log(`${module}: telemetry listener MQTT error: ${err}`);
|
log(`${module}: telemetry listener MQTT error: ${err}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add Stale Worker Detection
|
// ✅ Mark stale workers as inactive
|
||||||
function startStaleWorkerDetection() {
|
function startStaleWorkerDetection() {
|
||||||
const sdlCfg = config.modules[module];
|
const sdlCfg = config.modules[module];
|
||||||
|
|
||||||
|
|
@ -716,44 +578,36 @@ function startStaleWorkerDetection() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const checkInterval = 10000; // Check every 10 seconds
|
const checkInterval = 10000;
|
||||||
const staleThreshold =
|
|
||||||
Number.isInteger(sdlCfg.worker_stale_threshold) &&
|
// Try multiple config locations for stale threshold
|
||||||
sdlCfg.worker_stale_threshold > 0
|
const staleThreshold =
|
||||||
? sdlCfg.worker_stale_threshold
|
sdlCfg.expiration_timeout?.cluster_telemetry ||
|
||||||
: 30000; // Default 30 seconds
|
sdlCfg.worker_stale_threshold ||
|
||||||
|
30000;
|
||||||
|
|
||||||
setInterval(() => {
|
setInterval(() => {
|
||||||
let clusterState = loadClusterState(config);
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
let changed = false;
|
|
||||||
|
|
||||||
for (const [sdl_id, worker] of Object.entries(clusterState.workers)) {
|
for (const [sdl_id, worker] of Object.entries(workersState)) {
|
||||||
const lastSeenMs = Date.parse(worker.last_seen);
|
const lastSeenMs = Date.parse(worker.last_seen);
|
||||||
const ageMs = now - lastSeenMs;
|
const ageMs = now - lastSeenMs;
|
||||||
|
|
||||||
if (ageMs > staleThreshold && worker.status === 'active') {
|
if (ageMs > staleThreshold && worker.status === 'active') {
|
||||||
log(`${module}: worker ${worker.hostname} (${sdl_id}) marked as inactive (last seen ${Math.round(ageMs / 1000)}s ago)`);
|
log(`${module}: worker ${worker.hostname} (${sdl_id}) marked inactive (${Math.round(ageMs / 1000)}s)`);
|
||||||
clusterState.workers[sdl_id].status = 'inactive';
|
workersState[sdl_id].status = 'inactive';
|
||||||
changed = true;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (changed) {
|
|
||||||
clusterState = computeStats(clusterState);
|
|
||||||
saveClusterState(config, clusterState);
|
|
||||||
}
|
|
||||||
}, checkInterval);
|
}, checkInterval);
|
||||||
|
|
||||||
log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check interval: ${checkInterval}ms)`);
|
log(`${module}: stale worker detection active (threshold: ${staleThreshold}ms, check: ${checkInterval}ms)`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update entry point
|
// Entry point
|
||||||
startSDLStatusPub();
|
startSDLStatusPub();
|
||||||
startWorkersPub();
|
startWorkersPub();
|
||||||
startTelemetryListener();
|
startTelemetryListener();
|
||||||
startUdpBeacon();
|
startUdpBeacon();
|
||||||
loadClusterState(config);
|
loadClusterState(config);
|
||||||
startJoinHandler();
|
startJoinHandler();
|
||||||
startStaleWorkerDetection();
|
startStaleWorkerDetection();
|
||||||
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"name": "sdl-mgr",
|
"name": "sdl-mgr",
|
||||||
"version": "0.3.7",
|
"version": "0.3.9",
|
||||||
"version_date": "2026-02-01",
|
"version_date": "2026-02-08",
|
||||||
"description": "Software Defined Laboratory: Manager (sdl-mgr)",
|
"description": "Software Defined Laboratory: Manager (sdl-mgr)",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
|
|
@ -25,6 +25,7 @@
|
||||||
"@fortawesome/fontawesome-free": "^6.7.2",
|
"@fortawesome/fontawesome-free": "^6.7.2",
|
||||||
"aedes": "^0.51.3",
|
"aedes": "^0.51.3",
|
||||||
"bootstrap": "^5.3.7",
|
"bootstrap": "^5.3.7",
|
||||||
|
"bootstrap-icons": "^1.13.1",
|
||||||
"chart.js": "^4.5.1",
|
"chart.js": "^4.5.1",
|
||||||
"csv-parse": "^5.6.0",
|
"csv-parse": "^5.6.0",
|
||||||
"d3": "^7.9.0",
|
"d3": "^7.9.0",
|
||||||
|
|
|
||||||
|
|
@ -4,9 +4,9 @@
|
||||||
"worker_stale_threshold": 15000,
|
"worker_stale_threshold": 15000,
|
||||||
"update_interval": {
|
"update_interval": {
|
||||||
"udp_beacon": 2000,
|
"udp_beacon": 2000,
|
||||||
"cluster_status": 10000,
|
"cluster_status": 2000,
|
||||||
"cluster_workers": 10000,
|
"cluster_workers": 1000,
|
||||||
"cluster_telemetry": 3000
|
"cluster_telemetry": 500
|
||||||
},
|
},
|
||||||
"expiration_timeout": {
|
"expiration_timeout": {
|
||||||
"cluster_status": 60000,
|
"cluster_status": 60000,
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@
|
||||||
"open_browser": false,
|
"open_browser": false,
|
||||||
"weblibs": {
|
"weblibs": {
|
||||||
"bootstrap":"bootstrap/dist",
|
"bootstrap":"bootstrap/dist",
|
||||||
|
"bootstrap-icons":"bootstrap-icons",
|
||||||
"chartjs":"chart.js/dist",
|
"chartjs":"chart.js/dist",
|
||||||
"d3":"d3/dist",
|
"d3":"d3/dist",
|
||||||
"fontawesome":"@fortawesome/fontawesome-free",
|
"fontawesome":"@fortawesome/fontawesome-free",
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,9 @@
|
||||||
<!-- Bootstrap CSS -->
|
<!-- Bootstrap CSS -->
|
||||||
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
|
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
|
||||||
|
|
||||||
|
<!-- Bootstrap Icons -->
|
||||||
|
<link rel="stylesheet" href="/bootstrap-icons/font/bootstrap-icons.min.css">
|
||||||
|
|
||||||
<!-- Font Awesome -->
|
<!-- Font Awesome -->
|
||||||
<link rel="stylesheet" href="/fontawesome/css/all.min.css">
|
<link rel="stylesheet" href="/fontawesome/css/all.min.css">
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -54,14 +54,14 @@ if (Array.isArray(config.host.gpu) && config.host.gpu.length > 0) {
|
||||||
|
|
||||||
// Hostinfo: hostname, cpu, cores, ram, os
|
// Hostinfo: hostname, cpu, cores, ram, os
|
||||||
const hostinfo = {
|
const hostinfo = {
|
||||||
"hostname": '<span class="dash-val">' + config.host.hostname + '</span>',
|
|
||||||
"sdl_id": '<span class="dash-val" style=" font-size:0.8em">' + config.identity.sdl_id + '</span>',
|
"sdl_id": '<span class="dash-val" style=" font-size:0.8em">' + config.identity.sdl_id + '</span>',
|
||||||
"cpu cores": '<span class="dash-val">' + config.host.cpu.cores_logical + '</span> <span style="font-size:0.7em">' + config.host.cpu.model + '</span>',
|
"hostname": '<span class="dash-val">' + config.host.hostname + '</span>',
|
||||||
// "cores": config.host.cpu.cores_logical,
|
// "cpu cores": '<span class="dash-val">' + config.host.cpu.cores_logical + '</span> <span style="font-size:0.7em">' + config.host.cpu.model + '</span>',
|
||||||
"ram": '<span class="dash-val">' + config.host.memory.total_gb + '</span> GB ',
|
// // "cores": config.host.cpu.cores_logical,
|
||||||
"gpu": gpuSummary,
|
// "ram": '<span class="dash-val">' + config.host.memory.total_gb + '</span> GB ',
|
||||||
"net": "",
|
// "gpu": gpuSummary,
|
||||||
"os": config.host.os.pretty_name
|
// "net": "",
|
||||||
|
// "os": config.host.os.pretty_name
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -70,7 +70,7 @@ for (let intf in config.host.network) {
|
||||||
if (intf != 'lo') {
|
if (intf != 'lo') {
|
||||||
for (let addr in config.host.network[intf]) {
|
for (let addr in config.host.network[intf]) {
|
||||||
if (config.host.network[intf][addr].family == 'IPv4') {
|
if (config.host.network[intf][addr].family == 'IPv4') {
|
||||||
hostinfo["net"] += '<span class="dash-val" style=" font-size: 0.9em">' + config.host.network[intf][addr].cidr + '</span><br> ';
|
hostinfo["net"] += '<span class="dash-val" style=" font-size: 0.9em">' + config.host.network[intf][addr].cidr + '</span> ';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -79,12 +79,7 @@ for (let intf in config.host.network) {
|
||||||
|
|
||||||
let host_html = '<h5>Manager</h5>\n';
|
let host_html = '<h5>Manager</h5>\n';
|
||||||
host_html += '<table class="table table-sm table-striped">\n';
|
host_html += '<table class="table table-sm table-striped">\n';
|
||||||
// host_html += '<thead>\n';
|
|
||||||
// host_html += '<tr>\n';
|
|
||||||
// host_html += '<th>Key</th>\n';
|
|
||||||
// host_html += '<th>Value</th>\n';
|
|
||||||
// host_html += '</tr>\n';
|
|
||||||
// host_html += '</thead>\n';
|
|
||||||
host_html += '<tbody>\n';
|
host_html += '<tbody>\n';
|
||||||
|
|
||||||
for (let key in hostinfo) {
|
for (let key in hostinfo) {
|
||||||
|
|
@ -97,7 +92,7 @@ for (let key in hostinfo) {
|
||||||
host_html += '</tbody>\n';
|
host_html += '</tbody>\n';
|
||||||
host_html += '</table>\n';
|
host_html += '</table>\n';
|
||||||
|
|
||||||
document.getElementById('dash-sdl-mgr-info').innerHTML = host_html;
|
// document.getElementById('dash-sdl-mgr-info').innerHTML = host_html;
|
||||||
|
|
||||||
initMqtt();
|
initMqtt();
|
||||||
|
|
||||||
|
|
@ -245,9 +240,11 @@ async function initMqtt() {
|
||||||
lastSdlStatus = msg;
|
lastSdlStatus = msg;
|
||||||
lastSdlTimestamp = Date.parse(msg.ts);
|
lastSdlTimestamp = Date.parse(msg.ts);
|
||||||
renderModulesTable(msg);
|
renderModulesTable(msg);
|
||||||
|
// renderClusterSummary(msg);
|
||||||
} else if (msg.type === 'cluster-workers') {
|
} else if (msg.type === 'cluster-workers') {
|
||||||
lastWorkers = msg;
|
lastWorkers = msg;
|
||||||
renderWorkersTable(msg);
|
renderWorkersTable(msg);
|
||||||
|
renderClusterSummary(msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|
@ -302,109 +299,78 @@ function renderModulesTable(msg) {
|
||||||
// </tr>
|
// </tr>
|
||||||
// `;
|
// `;
|
||||||
|
|
||||||
sdl_html += `
|
|
||||||
<tr>
|
|
||||||
<th>Name</th>
|
|
||||||
<td class="dash-val" style=" font-size:1em">${msg.msg.cluster.name}</td>
|
|
||||||
</tr>
|
|
||||||
`;
|
|
||||||
|
|
||||||
sdl_html += `
|
|
||||||
<tr>
|
|
||||||
<th>Description</th>
|
|
||||||
<td class="dash-val" style=" font-size:0.8em">${msg.msg.cluster.desc}</td>
|
|
||||||
</tr>
|
|
||||||
`;
|
|
||||||
|
|
||||||
// Workers stats
|
|
||||||
const workers = msg.msg.workers || { allocated: 0, available: 0, used: 0 };
|
|
||||||
sdl_html += `
|
|
||||||
<tr>
|
|
||||||
<th>SDL Workers</th>
|
|
||||||
<td class="dash-val" style="font-size:1em">
|
|
||||||
<span class="dash-val">${workers.available}</span>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
`;
|
|
||||||
// sdl_html += `
|
// sdl_html += `
|
||||||
// <tr>
|
// <tr>
|
||||||
// <th>SDL Workers (active/total)</th>
|
// <th>Name</th>
|
||||||
|
// <td class="dash-val" style=" font-size:1em">${msg.msg.cluster.name}</td>
|
||||||
|
// </tr>
|
||||||
|
// `;
|
||||||
|
|
||||||
|
// sdl_html += `
|
||||||
|
// <tr>
|
||||||
|
// <th>Description</th>
|
||||||
|
// <td class="dash-val" style=" font-size:0.8em">${msg.msg.cluster.desc}</td>
|
||||||
|
// </tr>
|
||||||
|
// `;
|
||||||
|
|
||||||
|
// Workers stats
|
||||||
|
// const workers = msg.msg.workers || { allocated: 0, available: 0, used: 0 };
|
||||||
|
// sdl_html += `
|
||||||
|
// <tr>
|
||||||
|
// <th>SDL Workers</th>
|
||||||
// <td class="dash-val" style="font-size:1em">
|
// <td class="dash-val" style="font-size:1em">
|
||||||
// <span class="dash-val">${workers.available}</span> / ${workers.allocated}
|
// <span class="dash-val">${workers.available}</span>
|
||||||
// </td>
|
// </td>
|
||||||
// </tr>
|
// </tr>
|
||||||
// `;
|
// `;
|
||||||
|
|
||||||
|
|
||||||
// CPU resources
|
// CPU resources
|
||||||
const cpus = msg.msg.resources?.cpus || { allocated: 0, available: 0, used: 0 };
|
// const cpus = msg.msg.resources?.cpus || { allocated: 0, available: 0, used: 0 };
|
||||||
sdl_html += `
|
|
||||||
<tr>
|
|
||||||
<th><span class="fa fa-microchip" title="CPU Cores" style="font-size:1.2em"></span> CPU Cores</th>
|
|
||||||
<td class="dash-val" style="font-size:1em">
|
|
||||||
<span class="dash-val">${cpus.available}</span>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
`;
|
|
||||||
// sdl_html += `
|
// sdl_html += `
|
||||||
// <tr>
|
// <tr>
|
||||||
// <th><span class="fa fa-microchip" title="CPU Cores" style="font-size:1.2em"></span> CPU Cores</th>
|
// <th><span class="fa fa-microchip" title="CPU Cores" style="font-size:1.2em"></span> CPU Cores</th>
|
||||||
// <td class="dash-val" style="font-size:1em">
|
// <td class="dash-val" style="font-size:1em">
|
||||||
// <span class="dash-val">${cpus.available}</span> / ${cpus.allocated}
|
// <span class="dash-val">${cpus.available}</span>
|
||||||
// ${cpus.used > 0 ? `<span style="font-size:0.8em">(${cpus.used} in use)</span>` : ''}
|
|
||||||
// </td>
|
// </td>
|
||||||
// </tr>
|
// </tr>
|
||||||
// `;
|
// `;
|
||||||
|
|
||||||
// Memory resources
|
|
||||||
const memory = msg.msg.resources?.memory || { allocated: 0, available: 0, used: 0 };
|
|
||||||
const memAllocGB = Math.round(memory.allocated / (1024 * 1024 * 1024));
|
|
||||||
const memAvailGB = Math.round(memory.available / (1024 * 1024 * 1024));
|
|
||||||
const memUsedGB = memory.used > 0 ? Math.round(memory.used / 1024) : 0;
|
|
||||||
|
|
||||||
sdl_html += `
|
// Memory resources
|
||||||
<tr>
|
// const memory = msg.msg.resources?.memory || { allocated: 0, available: 0, used: 0 };
|
||||||
<th><span class="fa fa-memory" title="RAM" style="font-size:1.2em"></span> RAM</th>
|
// const memAllocGB = Math.round(memory.allocated / (1024 * 1024 * 1024));
|
||||||
<td class="dash-val" style="font-size:1em">
|
// const memAvailGB = Math.round(memory.available / (1024 * 1024 * 1024));
|
||||||
<span class="dash-val">${memAvailGB}</span> GB
|
// const memUsedGB = memory.used > 0 ? Math.round(memory.used / 1024) : 0;
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
`;
|
|
||||||
// sdl_html += `
|
// sdl_html += `
|
||||||
// <tr>
|
// <tr>
|
||||||
// <th><span class="fa fa-memory" title="RAM" style="font-size:1.2em"></span> RAM (GB)</th>
|
// <th><span class="fa fa-memory" title="RAM" style="font-size:1.2em"></span> RAM</th>
|
||||||
// <td class="dash-val" style="font-size:1em">
|
// <td class="dash-val" style="font-size:1em">
|
||||||
// <span class="dash-val">${memAvailGB}</span> / ${memAllocGB} GB
|
// <span class="dash-val">${memAvailGB}</span> GB
|
||||||
// ${memUsedGB > 0 ? `<span style="font-size:0.8em">(${memUsedGB} GB in use)</span>` : ''}
|
// </td>
|
||||||
// </td>
|
// </tr>
|
||||||
// </tr>
|
// `;
|
||||||
// `;
|
|
||||||
|
|
||||||
|
|
||||||
// GPU resources
|
// GPU resources
|
||||||
const gpus = msg.msg.resources?.gpus || { allocated: 0, available: 0, used: 0 };
|
// const gpus = msg.msg.resources?.gpus || { allocated: 0, available: 0, used: 0 };
|
||||||
sdl_html += `
|
|
||||||
<tr>
|
|
||||||
<th><span class="fa fa-dice-d20" title="GPU" style="font-size:1.2em"></span> GPU (VRAM)</th>
|
|
||||||
<td class="dash-val" style="font-size:1em">
|
|
||||||
<span class="dash-val">${gpus.available}</span>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
`;
|
|
||||||
// sdl_html += `
|
// sdl_html += `
|
||||||
// <tr>
|
// <tr>
|
||||||
// <th><span class="fa fa-dice-d20" title="GPU" style="font-size:1.2em"></span> GPU</th>
|
// <th><span class="fa fa-dice-d20" title="GPU" style="font-size:1.2em"></span> GPU (VRAM)</th>
|
||||||
// <td class="dash-val" style="font-size:1em">
|
// <td class="dash-val" style="font-size:1em">
|
||||||
// <span class="dash-val">${gpus.available}</span> / ${gpus.allocated}
|
// <span class="dash-val">${gpus.available}</span>
|
||||||
// ${gpus.used > 0 ? `<span style="font-size:0.8em">(${gpus.used} in use)</span>` : ''}
|
|
||||||
// </td>
|
// </td>
|
||||||
// </tr>
|
// </tr>
|
||||||
// `;
|
// `;
|
||||||
|
|
||||||
|
|
||||||
sdl_html += `
|
sdl_html += `
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
`;
|
`;
|
||||||
document.getElementById('dash-sdl-info').innerHTML = sdl_html;
|
|
||||||
|
|
||||||
|
sessionStorage.setItem('dash-sdl-uptime', msg.msg.sdl.uptime);
|
||||||
|
|
||||||
|
|
||||||
const modules = msg.msg.modules;
|
const modules = msg.msg.modules;
|
||||||
|
|
@ -431,7 +397,7 @@ function renderModulesTable(msg) {
|
||||||
</table>
|
</table>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
document.getElementById('dash-modules-list').innerHTML = html;
|
// document.getElementById('dash-modules-list').innerHTML = html;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -450,23 +416,33 @@ function startStatusAgeMonitor() {
|
||||||
|
|
||||||
const ageMs = Date.now() - lastSdlTimestamp;
|
const ageMs = Date.now() - lastSdlTimestamp;
|
||||||
|
|
||||||
const ageEl = document.getElementById('dash-modules-age');
|
// const ageEl = document.getElementById('dash-modules-age');
|
||||||
|
|
||||||
|
|
||||||
|
// if (ageMs > maxAgeMs) {
|
||||||
|
// ageEl.innerHTML =
|
||||||
|
// `<span class="text-danger">
|
||||||
|
// <i class="fa-solid fa-triangle-exclamation"></i>
|
||||||
|
// SDL status stale (${Math.round(ageMs / 1000)}s)
|
||||||
|
// </span>`;
|
||||||
|
|
||||||
|
// markAllModulesDown();
|
||||||
|
// } else {
|
||||||
|
// ageEl.innerHTML =
|
||||||
|
// `<span class="dash-val">
|
||||||
|
// <i class="fa-solid fa-circle-check"></i>
|
||||||
|
// SDL online (${Math.round(ageMs / 1000)}s ago)
|
||||||
|
// </span>`;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// Save dash-sdl-status to Session Storage
|
||||||
if (ageMs > maxAgeMs) {
|
if (ageMs > maxAgeMs) {
|
||||||
ageEl.innerHTML =
|
sessionStorage.setItem('dash-sdl-status', '<span class="text-danger"><i class="fa-solid fa-triangle-exclamation"></i> ' + Math.round(ageMs / 1000) + 's ago' + '</span>');
|
||||||
`<span class="text-danger">
|
|
||||||
<i class="fa-solid fa-triangle-exclamation"></i>
|
|
||||||
SDL status stale (${Math.round(ageMs / 1000)}s)
|
|
||||||
</span>`;
|
|
||||||
|
|
||||||
markAllModulesDown();
|
|
||||||
} else {
|
} else {
|
||||||
ageEl.innerHTML =
|
sessionStorage.setItem('dash-sdl-status', '<i class="fa-solid fa-circle-check"></i> ' + Math.round(ageMs / 1000) + 's ago');
|
||||||
`<span class="dash-val">
|
|
||||||
<i class="fa-solid fa-circle-check"></i>
|
|
||||||
SDL online (${Math.round(ageMs / 1000)}s ago)
|
|
||||||
</span>`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sessionStorage.setItem('dash-sdl-status', Math.round(ageMs / 1000) + 's ago');
|
||||||
}, 1000);
|
}, 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -500,53 +476,127 @@ function renderWorkersTable(msg) {
|
||||||
html += '<table class="table table-sm table-striped table-hover">';
|
html += '<table class="table table-sm table-striped table-hover">';
|
||||||
html += '<thead>';
|
html += '<thead>';
|
||||||
html += '<tr>';
|
html += '<tr>';
|
||||||
html += '<th>Hostname</th>';
|
html += '<th>System</th>';
|
||||||
html += '<th>Status</th>';
|
html += '<th>Status</th>';
|
||||||
html += '<th><i class="fa fa-microchip"></i> CPU</th>';
|
html += '<th><i class="fa fa-microchip"></i> CPU</th>';
|
||||||
html += '<th><i class="fa fa-memory"></i> RAM</th>';
|
html += '<th><i class="fa fa-memory"></i> RAM</th>';
|
||||||
html += '<th><i class="fa fa-dice-d20"></i> GPU</th>';
|
html += '<th><i class="fa fa-dice-d20"></i> GPU</th>';
|
||||||
html += '<th>Platform</th>';
|
|
||||||
html += '<th>Last Seen</th>';
|
|
||||||
html += '</tr>';
|
html += '</tr>';
|
||||||
html += '</thead>';
|
html += '</thead>';
|
||||||
html += '<tbody>';
|
html += '<tbody>';
|
||||||
|
|
||||||
|
const nowSec = Math.floor(Date.now() / 1000);
|
||||||
|
const staleThresholdSec = 60;
|
||||||
|
|
||||||
for (const [sdl_id, worker] of Object.entries(workers)) {
|
for (const [sdl_id, worker] of Object.entries(workers)) {
|
||||||
const status = worker.status === 'active'
|
const ageSec = nowSec - worker.ts_utime; // ✅ Use ts_utime from telemetry
|
||||||
? '<span class="badge bg-success">Active</span>'
|
const isActive = ageSec <= staleThresholdSec;
|
||||||
: '<span class="badge bg-secondary">Inactive</span>';
|
|
||||||
|
// Status badge
|
||||||
const cpuAvail = worker.resources?.cpus?.available || 0;
|
const statusBadge = isActive
|
||||||
const cpuTotal = worker.resources?.cpus?.allocated || 0;
|
? '<span class="badge bg-success">Active</span>'
|
||||||
const cpuUsed = worker.resources?.cpus?.used || 0;
|
: '<span class="badge bg-danger">Inactive</span>';
|
||||||
|
|
||||||
const memAvail = worker.resources?.memory?.available || 0;
|
// ✅ Hardware info from nested msg.system.hardware
|
||||||
const memTotal = worker.resources?.memory?.allocated || 0;
|
const hwType = worker.msg?.system?.hardware?.type || 'unknown';
|
||||||
const memUsed = worker.resources?.memory?.used || 0;
|
// const hwIcon = hwType === 'hw' ? '🖥️' : hwType === 'vm' ? '💠' : '❓';
|
||||||
|
const hwIcon = hwType === 'hw' ? '<i class="fa fa-desktop"></i>' :
|
||||||
|
hwType === 'vm' ? '<i class="fa fa-cube"></i>' :
|
||||||
|
'<i class="fa fa-question"></i>';
|
||||||
|
const hwManuf = worker.msg?.system?.hardware?.manufacturer || 'Unknown';
|
||||||
|
const hwModel = worker.msg?.system?.hardware?.model || 'Unknown';
|
||||||
|
const hwCPU = worker.msg?.system?.hardware?.cpu || 'Unknown';
|
||||||
|
|
||||||
|
const hardwareDisplay = hwType === 'vm'
|
||||||
|
? `${hwIcon} ${hwModel}`
|
||||||
|
: `${hwIcon} ${hwManuf} ${hwModel}`;
|
||||||
|
|
||||||
|
// ✅ OS info from nested msg.system
|
||||||
|
const osDistro = worker.msg?.system?.distro.replace(/\(\S+\)/, '') || 'Unknown';
|
||||||
|
const osArch = worker.msg?.system?.arch || 'unknown';
|
||||||
|
const osPlatform = worker.msg?.system?.platform || 'unknown';
|
||||||
|
const platformIcon = osPlatform === 'linux' ? '<i class="fab fa-linux"></i>' :
|
||||||
|
osPlatform === 'win32' ? '<i class="fab fa-windows"></i>' :
|
||||||
|
osPlatform === 'darwin' ? '<i class="fab fa-apple"></i>' :
|
||||||
|
'<i class="fa fa-question"></i>';
|
||||||
|
|
||||||
|
const osDisplay = `${platformIcon} ${osDistro} (${osArch})`;
|
||||||
|
|
||||||
|
// ✅ CPU from nested msg.resources and msg.usage
|
||||||
|
const cpuAvail = worker.msg?.resources?.cpus?.available || 0;
|
||||||
|
const cpuTotal = worker.msg?.resources?.cpus?.allocated || 0;
|
||||||
|
const cpuUsed = worker.msg?.usage?.cpu?.total || 0;
|
||||||
|
|
||||||
|
// ✅ Memory from nested msg
|
||||||
|
const memAvail = worker.msg?.resources?.memory?.available || 0;
|
||||||
|
const memTotal = worker.msg?.resources?.memory?.allocated || 0;
|
||||||
|
|
||||||
// Convert bytes to GB
|
|
||||||
const memAvailGB = Math.round(memAvail / (1024 * 1024 * 1024));
|
const memAvailGB = Math.round(memAvail / (1024 * 1024 * 1024));
|
||||||
const memTotalGB = Math.round(memTotal / (1024 * 1024 * 1024));
|
const memTotalGB = Math.round(memTotal / (1024 * 1024 * 1024));
|
||||||
const memUsedGB = memUsed > 0 ? Math.round(memUsed / (1024 * 1024 * 1024)) : 0;
|
const memPercent = worker.msg?.usage?.memory?.percent_used || 0;
|
||||||
|
|
||||||
const gpuAvail = worker.resources?.gpus?.available || 0;
|
// ✅ GPU from nested msg
|
||||||
const gpuTotal = worker.resources?.gpus?.allocated || 0;
|
const gpuAvail = worker.msg?.resources?.gpus?.available || 0;
|
||||||
const gpuUsed = worker.resources?.gpus?.used || 0;
|
const gpuTotal = worker.msg?.resources?.gpus?.allocated || 0;
|
||||||
|
const gpuData = worker.msg?.usage?.gpu;
|
||||||
|
const gpuUsed = gpuData?.total_utilization || 0;
|
||||||
|
const gpuMemMB = gpuData?.total_memory_total_mb || 0;
|
||||||
|
const gpuMemGB = Math.round(gpuMemMB / 1024);
|
||||||
|
|
||||||
const platform = `${worker.platform || 'unknown'} / ${worker.arch || 'unknown'}`;
|
// ✅ Uptime from nested msg.uptime
|
||||||
|
const procUptime = worker.msg?.uptime?.proc_dhms || 'N/A';
|
||||||
|
const sysUptime = worker.msg?.uptime?.sys_dhms || 'N/A';
|
||||||
|
|
||||||
const lastSeen = worker.last_seen
|
// Last seen age
|
||||||
? new Date(worker.last_seen).toLocaleString()
|
const days = Math.floor(ageSec / 86400);
|
||||||
: 'N/A';
|
const hours = Math.floor((ageSec % 86400) / 3600);
|
||||||
|
const minutes = Math.floor((ageSec % 3600) / 60);
|
||||||
|
const seconds = Math.floor(ageSec % 60);
|
||||||
|
|
||||||
|
let lastSeenAge = '';
|
||||||
|
if (days > 0) lastSeenAge += `${days}d `;
|
||||||
|
if (hours > 0 || days > 0) lastSeenAge += `${hours}h `;
|
||||||
|
if (minutes > 0 || hours > 0 || days > 0) lastSeenAge += `${minutes}m `;
|
||||||
|
lastSeenAge += `${seconds}s ago`;
|
||||||
|
|
||||||
html += '<tr>';
|
html += '<tr>';
|
||||||
html += `<td><strong>${worker.hostname}</strong><br><small class="text-muted">${worker.sdl_id}</small></td>`;
|
|
||||||
html += `<td>${status}</td>`;
|
// System column
|
||||||
html += `<td><span class="dash-val">${cpuAvail}</span> / ${cpuTotal}${cpuUsed > 0 ? `<br><small>(${cpuUsed} used)</small>` : ''}</td>`;
|
html += `<td>
|
||||||
html += `<td><span class="dash-val">${memAvailGB}</span> / ${memTotalGB} GB${memUsedGB > 0 ? `<br><small>(${memUsedGB} GB used)</small>` : ''}</td>`;
|
<strong>${worker.host}</strong><br>
|
||||||
html += `<td><span class="dash-val">${gpuAvail}</span> / ${gpuTotal}${gpuUsed > 0 ? `<br><small>(${gpuUsed} used)</small>` : ''}</td>`;
|
<small class="text-muted" style="font-size:0.7em">${worker.sdl_id.substring(0, 8)}...</small><br>
|
||||||
html += `<td><small>${platform}</small></td>`;
|
<small class="text-muted" style="font-size:0.8em">${hardwareDisplay}</small><br>
|
||||||
html += `<td><small>${lastSeen}</small></td>`;
|
<small class="text-muted" style="font-size:0.75em"><i title="CPU" class="fa fa-microchip"></i> <i class="fa fa-memory"></i> <span class="dash-val" style="font-weight:bold">${memTotalGB}</span> GB | <span class="dash-val" style="font-weight:bold"> ${cpuTotal}</span>x ${hwCPU}} </small><br>
|
||||||
|
<small class="text-muted" style="font-size:0.75em">${osDisplay}</small>
|
||||||
|
</td>`;
|
||||||
|
|
||||||
|
// Status column
|
||||||
|
html += `<td>
|
||||||
|
${statusBadge}<br>
|
||||||
|
<small class="text-muted">SDL: ${procUptime}</small><br>
|
||||||
|
<small class="text-muted">OS: ${sysUptime}</small><br>
|
||||||
|
<small class="text-muted">${lastSeenAge}</small>
|
||||||
|
</td>`;
|
||||||
|
|
||||||
|
// CPU
|
||||||
|
html += `<td>
|
||||||
|
<span class="dash-val">${cpuAvail}</span> / ${cpuTotal}
|
||||||
|
${cpuUsed > 0 ? `<br><small class="text-muted">${cpuUsed}% load</small>` : ''}
|
||||||
|
</td>`;
|
||||||
|
|
||||||
|
// Memory
|
||||||
|
html += `<td>
|
||||||
|
<span class="dash-val">${memAvailGB}</span> / ${memTotalGB} GB
|
||||||
|
${memPercent > 0 ? `<br><small class="text-muted">${memPercent}% used</small>` : ''}
|
||||||
|
</td>`;
|
||||||
|
|
||||||
|
// GPU
|
||||||
|
html += `<td>
|
||||||
|
<span class="dash-val">${gpuAvail}</span> / ${gpuTotal}
|
||||||
|
${gpuMemGB > 0 ? `<br><small class="text-muted">(${gpuMemGB} GB)</small>` : ''}
|
||||||
|
${gpuUsed > 0 ? `<br><small class="text-muted">${gpuUsed}% load</small>` : ''}
|
||||||
|
</td>`;
|
||||||
|
|
||||||
html += '</tr>';
|
html += '</tr>';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -555,3 +605,148 @@ function renderWorkersTable(msg) {
|
||||||
|
|
||||||
document.getElementById('dash-sdl-wkrs').innerHTML = html;
|
document.getElementById('dash-sdl-wkrs').innerHTML = html;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function renderClusterSummary(msg) {
|
||||||
|
const workers = msg.msg?.workers || {};
|
||||||
|
const workerCount = Object.keys(workers).length;
|
||||||
|
|
||||||
|
// Aggregate resources from all workers
|
||||||
|
let totalCpus = 0;
|
||||||
|
let totalMemory = 0;
|
||||||
|
let totalGpus = 0;
|
||||||
|
let activeCpus = 0;
|
||||||
|
let activeMemory = 0;
|
||||||
|
let activeGpus = 0;
|
||||||
|
let activeWorkers = 0;
|
||||||
|
|
||||||
|
const nowSec = Math.floor(Date.now() / 1000);
|
||||||
|
const staleThresholdSec = 60;
|
||||||
|
|
||||||
|
for (const worker of Object.values(workers)) {
|
||||||
|
const ageSec = nowSec - worker.ts_utime;
|
||||||
|
const isActive = ageSec <= staleThresholdSec;
|
||||||
|
|
||||||
|
// Totals (all workers)
|
||||||
|
totalCpus += worker.msg?.resources?.cpus?.allocated || 0;
|
||||||
|
totalMemory += worker.msg?.resources?.memory?.allocated || 0;
|
||||||
|
totalGpus += worker.msg?.resources?.gpus?.allocated || 0;
|
||||||
|
|
||||||
|
// Active workers only
|
||||||
|
if (isActive) {
|
||||||
|
activeWorkers++;
|
||||||
|
activeCpus += worker.msg?.resources?.cpus?.available || 0;
|
||||||
|
activeMemory += worker.msg?.resources?.memory?.allocated || 0;
|
||||||
|
activeGpus += worker.msg?.resources?.gpus?.available || 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert memory to GB
|
||||||
|
const totalMemoryGB = Math.round(totalMemory / (1024 * 1024 * 1024));
|
||||||
|
const activeMemoryGB = Math.round(activeMemory / (1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
// Get cluster info from last status message
|
||||||
|
const clusterName = lastSdlStatus?.msg?.cluster?.name || 'Unknown';
|
||||||
|
const clusterDesc = lastSdlStatus?.msg?.cluster?.desc || '';
|
||||||
|
|
||||||
|
let html = `
|
||||||
|
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||||
|
<h5>Cluster: <span class="dash-val">${clusterName}</span></h5>
|
||||||
|
<small class="dash-val">${clusterDesc}</small>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
html += '<div class="row">';
|
||||||
|
|
||||||
|
// SDL card
|
||||||
|
html += `
|
||||||
|
<div class="col-md-3">
|
||||||
|
<div class="card border mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<h6 class="card-title">
|
||||||
|
<i class="fa-solid fa-square-binary"></i> SDL
|
||||||
|
</h6>
|
||||||
|
<span class="card-text">Version: <span class="dash-val">v${config?.sdl?.version}</span></span><br>
|
||||||
|
<span class="card-text">Uptime: <span class="dash-val" id="dash-sdl-uptime"></span></span><br>
|
||||||
|
<span class="card-text">Status: <span class="dash-val" id="dash-sdl-status"></span></span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Workers card
|
||||||
|
html += `
|
||||||
|
<div class="col-md-2">
|
||||||
|
<div class="card border mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<h6 class="card-title">
|
||||||
|
<i class="fa fa-server"></i> Workers
|
||||||
|
</h6>
|
||||||
|
<h3 class="card-text dash-val">${activeWorkers}</h3>
|
||||||
|
<small class="text-muted">of ${workerCount} total</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
// CPU card
|
||||||
|
html += `
|
||||||
|
<div class="col-md-2">
|
||||||
|
<div class="card border mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<h6 class="card-title">
|
||||||
|
<i class="fa fa-microchip"></i> CPU Cores
|
||||||
|
</h6>
|
||||||
|
<h3 class="card-text dash-val">${activeCpus}</h3>
|
||||||
|
<small class="text-muted">of ${totalCpus} total</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Memory card
|
||||||
|
html += `
|
||||||
|
<div class="col-md-2">
|
||||||
|
<div class="card border mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<h6 class="card-title">
|
||||||
|
<i class="fa fa-memory"></i> RAM
|
||||||
|
</h6>
|
||||||
|
<h3 class="card-text dash-val">${activeMemoryGB} GB</h3>
|
||||||
|
<small class="text-muted">of ${totalMemoryGB} GB total</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
// GPU card
|
||||||
|
html += `
|
||||||
|
<div class="col-md-2">
|
||||||
|
<div class="card border mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<h6 class="card-title">
|
||||||
|
<i class="fa fa-dice-d20"></i> GPUs
|
||||||
|
</h6>
|
||||||
|
<h3 class="card-text dash-val">${activeGpus}</h3>
|
||||||
|
<small class="text-muted">of ${totalGpus} total</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
html += '</div>'; // end row
|
||||||
|
|
||||||
|
document.getElementById('dash-sdl-clstr').innerHTML = html;
|
||||||
|
|
||||||
|
// Read dash-sdl-uptime from Session Storage
|
||||||
|
const dashSdlUptime = sessionStorage.getItem('dash-sdl-uptime');
|
||||||
|
if (dashSdlUptime) {
|
||||||
|
document.getElementById('dash-sdl-uptime').innerHTML = dashSdlUptime;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read dash-sdl-status from Session Storage
|
||||||
|
const dashSdlStatus = sessionStorage.getItem('dash-sdl-status');
|
||||||
|
if (dashSdlStatus) {
|
||||||
|
document.getElementById('dash-sdl-status').innerHTML = dashSdlStatus;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -371,54 +371,99 @@ function loadNavTabs() {
|
||||||
// ===============================
|
// ===============================
|
||||||
// Side Panel (layout-collapsing)
|
// Side Panel (layout-collapsing)
|
||||||
// ===============================
|
// ===============================
|
||||||
async function loadSidePanel() {
|
// async function loadSidePanel() {
|
||||||
// console.log('DEBUG: loadSidePanel() called');
|
// // console.log('DEBUG: loadSidePanel() called');
|
||||||
|
|
||||||
|
// const panel = document.getElementById('sidepanel');
|
||||||
|
// const inner = document.getElementById('sidepanel-inner');
|
||||||
|
// const toggleBtn = document.getElementById('sidepanel-toggle');
|
||||||
|
|
||||||
|
// if (!panel || !inner || !toggleBtn) return;
|
||||||
|
|
||||||
|
// const currentPage =
|
||||||
|
// new URLSearchParams(window.location.search).get('page') || 'index';
|
||||||
|
|
||||||
|
// const isCollapsed =
|
||||||
|
// localStorage.getItem('sidepanel_collapsed') === 'true';
|
||||||
|
|
||||||
|
// if (isCollapsed) {
|
||||||
|
// panel.classList.add('collapsed');
|
||||||
|
// // toggleBtn.innerHTML = `<i class="fa-solid fa-folder-plus"></i>`;
|
||||||
|
// toggleBtn.innerHTML = '<i class="bi bi-layout-sidebar"></i>';
|
||||||
|
// } else {
|
||||||
|
// // toggleBtn.innerHTML = `<i class="fa-solid fa-folder-minus"></i>`;
|
||||||
|
// toggleBtn.innerHTML = '<i class="bi bi-layout-sidebar-reverse"></i>';
|
||||||
|
// }
|
||||||
|
|
||||||
|
// try {
|
||||||
|
// const res = await fetch('/api/nav/sidepanel');
|
||||||
|
// if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
|
||||||
|
// const sideNav = await res.json();
|
||||||
|
|
||||||
|
// renderAllSidePanelSections(inner, sideNav, currentPage);
|
||||||
|
// } catch (err) {
|
||||||
|
// console.error('Failed to load side panel:', err);
|
||||||
|
// inner.innerHTML = '';
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // Collapse / expand behavior
|
||||||
|
// toggleBtn.onclick = () => {
|
||||||
|
// const collapsed = panel.classList.toggle('collapsed');
|
||||||
|
// localStorage.setItem('sidepanel_collapsed', collapsed);
|
||||||
|
|
||||||
|
// // toggleBtn.innerHTML = `
|
||||||
|
// // <i class="fa-solid ${
|
||||||
|
// // collapsed ? 'fa-folder-plus' : 'fa-folder-minus'
|
||||||
|
// // }"></i>
|
||||||
|
// // `;
|
||||||
|
|
||||||
|
// toggleBtn.innerHTML = `
|
||||||
|
// <i class="bi ${
|
||||||
|
// collapsed ? 'bi-layout-sidebar' : 'bi-layout-sidebar-reverse'
|
||||||
|
// }"></i>
|
||||||
|
// `;
|
||||||
|
// };
|
||||||
|
// }
|
||||||
|
async function loadSidePanel() {
|
||||||
const panel = document.getElementById('sidepanel');
|
const panel = document.getElementById('sidepanel');
|
||||||
const inner = document.getElementById('sidepanel-inner');
|
const inner = document.getElementById('sidepanel-inner');
|
||||||
const toggleBtn = document.getElementById('sidepanel-toggle');
|
const toggleBtn = document.getElementById('sidepanel-toggle');
|
||||||
|
|
||||||
if (!panel || !inner || !toggleBtn) return;
|
if (!panel || !inner || !toggleBtn) return;
|
||||||
|
|
||||||
const currentPage =
|
const currentPage =
|
||||||
new URLSearchParams(window.location.search).get('page') || 'index';
|
new URLSearchParams(window.location.search).get('page') || 'index';
|
||||||
|
|
||||||
const isCollapsed =
|
const isCollapsed =
|
||||||
localStorage.getItem('sidepanel_collapsed') === 'true';
|
localStorage.getItem('sidepanel_collapsed') === 'true';
|
||||||
|
|
||||||
if (isCollapsed) {
|
if (isCollapsed) {
|
||||||
panel.classList.add('collapsed');
|
panel.classList.add('collapsed');
|
||||||
toggleBtn.innerHTML = `<i class="fa-solid fa-folder-plus"></i>`;
|
toggleBtn.innerHTML = '<i class="bi bi-layout-sidebar"></i>';
|
||||||
} else {
|
} else {
|
||||||
toggleBtn.innerHTML = `<i class="fa-solid fa-folder-minus"></i>`;
|
toggleBtn.innerHTML = '<i class="bi bi-layout-sidebar-reverse"></i>';
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/nav/sidepanel');
|
const res = await fetch('/api/nav/sidepanel');
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
if (!res.ok) throw new Error(`HTTP ${res.status}`); // ✅ Fixed template literal
|
||||||
|
|
||||||
const sideNav = await res.json();
|
const sideNav = await res.json();
|
||||||
|
|
||||||
renderAllSidePanelSections(inner, sideNav, currentPage);
|
renderAllSidePanelSections(inner, sideNav, currentPage);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to load side panel:', err);
|
console.error('Failed to load side panel:', err);
|
||||||
inner.innerHTML = '';
|
inner.innerHTML = '';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Collapse / expand behavior
|
// Collapse / expand behavior
|
||||||
toggleBtn.onclick = () => {
|
toggleBtn.onclick = () => {
|
||||||
const collapsed = panel.classList.toggle('collapsed');
|
const collapsed = panel.classList.toggle('collapsed');
|
||||||
localStorage.setItem('sidepanel_collapsed', collapsed);
|
localStorage.setItem('sidepanel_collapsed', collapsed);
|
||||||
|
|
||||||
toggleBtn.innerHTML = `
|
toggleBtn.innerHTML = `
|
||||||
<i class="fa-solid ${
|
<i class="bi ${collapsed ? 'bi-layout-sidebar' : 'bi-layout-sidebar-reverse'}"></i>
|
||||||
collapsed ? 'fa-folder-plus' : 'fa-folder-minus'
|
|
||||||
}"></i>
|
|
||||||
`;
|
`;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// -------------------------------
|
// -------------------------------
|
||||||
// Find matching section by page
|
// Find matching section by page
|
||||||
// -------------------------------
|
// -------------------------------
|
||||||
|
|
|
||||||
|
|
@ -5,14 +5,46 @@
|
||||||
| **Author** | John Haverlack |
|
| **Author** | John Haverlack |
|
||||||
| **Copyright** | 2026 John Haverlack |
|
| **Copyright** | 2026 John Haverlack |
|
||||||
| **License** | MIT |
|
| **License** | MIT |
|
||||||
| **Version** | 0.3.7 |
|
| **Version** | 0.3.9 |
|
||||||
| **Date** | 2026-02-01 |
|
| **Date** | 2026-02-08 |
|
||||||
|
|
||||||
|
## v0.3.9 - 2026-02-08 (ALPHA)
|
||||||
|
|
||||||
|
- Refactor Cluster Status Dash
|
||||||
|
- Refactor Cluster Status Dash
|
||||||
|
- System Columnn Display
|
||||||
|
- Start v0.3.9
|
||||||
|
|
||||||
|
## v0.3.8 - 2026-02-01 (ALPHA)
|
||||||
|
|
||||||
|
- v0.3.8
|
||||||
|
- Refactoing Worker Dashboard
|
||||||
|
- Refactoing Worker Dashboard
|
||||||
|
- Refactored Cluster Status
|
||||||
|
- Refactor of telemetry format + hw detection
|
||||||
|
- Switched Panel Icon to Bootstrap Icon
|
||||||
|
- Updating Worker Telemetry
|
||||||
|
- CP
|
||||||
|
- Fixing Wget/Curl Command Installer Port
|
||||||
|
- Fixing Wget/Curl Command Installer Port
|
||||||
|
- Fixing Wkr Start / Stop
|
||||||
|
- Update stop-sdl-wkr.sh
|
||||||
|
- WIP Install / Start / Stop
|
||||||
|
- WIP Install / Start /Stop
|
||||||
|
- Systemd Detectiion in Start/Stop/Status
|
||||||
|
- WIP: Start / Stop Scripts
|
||||||
|
- Non Systemd start stop scripts
|
||||||
|
- WIP: start stop scripts
|
||||||
|
- Updating Task List
|
||||||
|
- Adding Screeshot to README
|
||||||
|
- Start v0.3.8
|
||||||
|
|
||||||
## v0.3.7 - 2026-02-01 (ALPHA)
|
## v0.3.7 - 2026-02-01 (ALPHA)
|
||||||
|
|
||||||
**Summary**
|
**Summary**
|
||||||
sdl-wkr Telemetry
|
sdl-wkr Telemetry
|
||||||
|
|
||||||
|
- v0.3.7
|
||||||
- Fixing sdl-wkr RAM detection
|
- Fixing sdl-wkr RAM detection
|
||||||
- Start v0.3.7
|
- Start v0.3.7
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,8 @@
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col-3" id="dash-sdl-info">
|
<div class="col-10" id="dash-sdl-clstr">
|
||||||
|
<h5>Cluster</h5>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-4" id="dash-sdl-mgr-info">
|
<div class="col-10" id="dash-sdl-wkrs">
|
||||||
</div>
|
|
||||||
<div class="col-2" id="dash-modules">
|
|
||||||
<div id="dash-modules-list"></div>
|
|
||||||
<div id="dash-modules-age"></div>
|
|
||||||
</div>
|
|
||||||
<div class="col-8" id="dash-sdl-wkrs">
|
|
||||||
<h5>Workers</h5>
|
<h5>Workers</h5>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -0,0 +1,65 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# start-sdl-mgr.sh - Script to start SDL Manager
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
use_systemd() {
|
||||||
|
# Check if systemd user mode is available and service exists
|
||||||
|
if systemctl --user status >/dev/null 2>&1; then
|
||||||
|
# Check if our service file exists
|
||||||
|
if systemctl --user list-unit-files | grep -q "^sdl-mgr.service"; then
|
||||||
|
return 0 # Use systemd
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 1 # Use manual scripts
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get $SDL_HOME directory from SCRIPT PATH
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
SDL_HOME="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
|
if use_systemd; then
|
||||||
|
# Systemd mode - let systemd handle it
|
||||||
|
echo "Starting SDL Manager (systemd)..."
|
||||||
|
systemctl --user start sdl-mgr
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Manual mode - continue with checks
|
||||||
|
NODE_BIN_DIR="${SDL_HOME}/nodejs/current/bin"
|
||||||
|
NODE_BIN="${NODE_BIN_DIR}/node"
|
||||||
|
NPM_BIN="${NODE_BIN_DIR}/npm"
|
||||||
|
|
||||||
|
SDL_MGR_APP_DIR="${SDL_HOME}/sdl-mgr/current/app"
|
||||||
|
|
||||||
|
# Check that sdl-mgr is not already running
|
||||||
|
SDL_MGR_PID=""
|
||||||
|
for pid in $(pgrep -f 'node' 2>/dev/null || true); do
|
||||||
|
cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true)
|
||||||
|
if [[ "$cwd" == *"sdl-mgr"* ]]; then
|
||||||
|
SDL_MGR_PID="$pid"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -n "$SDL_MGR_PID" ]]; then
|
||||||
|
echo "ERROR: SDL Manager is already running with PID ${SDL_MGR_PID}"
|
||||||
|
echo " Stop it first: ${SDL_HOME}/scripts/stop-sdl-mgr.sh"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Verify paths exist
|
||||||
|
if [[ ! -f "$NODE_BIN" ]]; then
|
||||||
|
echo "ERROR: Node.js not found at: $NODE_BIN"
|
||||||
|
echo " Run install-sdl.sh first"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -d "$SDL_MGR_APP_DIR" ]]; then
|
||||||
|
echo "ERROR: SDL Manager app not found at: $SDL_MGR_APP_DIR"
|
||||||
|
echo " Run install-sdl.sh first"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Starting SDL Manager..."
|
||||||
|
echo " App Dir: ${SDL_MGR_APP_DIR}"
|
||||||
|
echo " Node:
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# status-sdl-mgr.sh - Script to show status of SDL Manager
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
use_systemd() {
|
||||||
|
# Check if systemd user mode is available and service exists
|
||||||
|
if systemctl --user status >/dev/null 2>&1; then
|
||||||
|
# Check if our service file exists
|
||||||
|
if systemctl --user list-unit-files | grep -q "^sdl-mgr.service"; then
|
||||||
|
return 0 # Use systemd
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 1 # Use manual scripts
|
||||||
|
}
|
||||||
|
|
||||||
|
if use_systemd; then
|
||||||
|
systemctl --user status sdl-mgr
|
||||||
|
else
|
||||||
|
# Get sdl-mgr PID
|
||||||
|
SDL_MGR_PID=""
|
||||||
|
for pid in $(pgrep -f 'node' 2>/dev/null || true); do
|
||||||
|
cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true)
|
||||||
|
if [[ "$cwd" == *"sdl-mgr"* ]]; then
|
||||||
|
SDL_MGR_PID="$pid"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$SDL_MGR_PID" ]]; then
|
||||||
|
echo "SDL Manager is not running"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get process uptime
|
||||||
|
PROCESS_START=$(ps -p "$SDL_MGR_PID" -o lstart= 2>/dev/null || echo "unknown")
|
||||||
|
|
||||||
|
echo "SDL Manager is running"
|
||||||
|
echo " PID: ${SDL_MGR_PID}"
|
||||||
|
echo " Started: ${PROCESS_START}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
@ -0,0 +1,65 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# stop-sdl-mgr.sh - Script to stop SDL Manager
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
use_systemd() {
|
||||||
|
# Check if systemd user mode is available and service exists
|
||||||
|
if systemctl --user status >/dev/null 2>&1; then
|
||||||
|
# Check if our service file exists
|
||||||
|
if systemctl --user list-unit-files | grep -q "^sdl-mgr.service"; then
|
||||||
|
return 0 # Use systemd
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 1 # Use manual scripts
|
||||||
|
}
|
||||||
|
|
||||||
|
if use_systemd; then
|
||||||
|
# Systemd mode - let systemd handle it
|
||||||
|
echo "Stopping SDL Manager (systemd)..."
|
||||||
|
systemctl --user stop sdl-mgr --quiet 2>/dev/null || true
|
||||||
|
echo "SDL Manager stopped"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Manual mode - find and kill process
|
||||||
|
SDL_MGR_PID=""
|
||||||
|
for pid in $(pgrep -f 'node' 2>/dev/null || true); do
|
||||||
|
cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true)
|
||||||
|
if [[ "$cwd" == *"sdl-mgr"* ]]; then
|
||||||
|
SDL_MGR_PID="$pid"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$SDL_MGR_PID" ]]; then
|
||||||
|
echo "SDL Manager is not running"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Stopping SDL Manager (PID: ${SDL_MGR_PID})..."
|
||||||
|
|
||||||
|
# Send SIGTERM for graceful shutdown
|
||||||
|
kill "$SDL_MGR_PID"
|
||||||
|
|
||||||
|
# Wait up to 10 seconds for process to exit
|
||||||
|
for i in {1..10}; do
|
||||||
|
if ! kill -0 "$SDL_MGR_PID" 2>/dev/null; then
|
||||||
|
echo "SDL Manager stopped successfully"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
# If still running after 10 seconds, force kill
|
||||||
|
if kill -0 "$SDL_MGR_PID" 2>/dev/null; then
|
||||||
|
echo "WARNING: Graceful shutdown failed, forcing kill..."
|
||||||
|
kill -9 "$SDL_MGR_PID"
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
if kill -0 "$SDL_MGR_PID" 2>/dev/null; then
|
||||||
|
echo "ERROR: Failed to stop SDL Manager"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "SDL Manager stopped"
|
||||||
|
|
@ -12,7 +12,7 @@ log('===========================================================================
|
||||||
log(config.package.name + ': STARTING: ' + config.package.description + ' v' + config.package.version);
|
log(config.package.name + ': STARTING: ' + config.package.description + ' v' + config.package.version);
|
||||||
// log(JSON.stringify(config, null, 2));
|
// log(JSON.stringify(config, null, 2));
|
||||||
// log(JSON.stringify(config.dirs, null, 2), false);
|
// log(JSON.stringify(config.dirs, null, 2), false);
|
||||||
log(JSON.stringify(config.host, null, 2), false);
|
// log(JSON.stringify(config.host, null, 2), false);
|
||||||
|
|
||||||
for (let m in config.modules) {
|
for (let m in config.modules) {
|
||||||
// log(`Loading module: ${m}`);
|
// log(`Loading module: ${m}`);
|
||||||
|
|
|
||||||
|
|
@ -602,8 +602,16 @@ function getUptimeDHMS() {
|
||||||
const h = Math.floor((uptimeSec % 86400) / 3600);
|
const h = Math.floor((uptimeSec % 86400) / 3600);
|
||||||
const m = Math.floor((uptimeSec % 3600) / 60);
|
const m = Math.floor((uptimeSec % 3600) / 60);
|
||||||
const s = Math.floor(uptimeSec % 60);
|
const s = Math.floor(uptimeSec % 60);
|
||||||
return `${d}.${h}.${m}.${s}`;
|
return `${d}d ${h}h ${m}m ${s}s`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getOSUptimeDHMS() {
|
||||||
|
const uptimeSec = os.uptime();
|
||||||
|
const d = Math.floor(uptimeSec / 86400);
|
||||||
|
const h = Math.floor((uptimeSec % 86400) / 3600);
|
||||||
|
const m = Math.floor((uptimeSec % 3600) / 60);
|
||||||
|
const s = Math.floor(uptimeSec % 60);
|
||||||
|
return `${d}d ${h}h ${m}m ${s}s`;
|
||||||
|
}
|
||||||
|
|
||||||
export { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeMs, getUptimeSec, getUptimeDHMS };
|
export { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeMs, getUptimeSec, getUptimeDHMS, getOSUptimeDHMS };
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,11 @@
|
||||||
const module = 'sdl-wkr'; // Module Name
|
const module = 'sdl-wkr'; // Module Name
|
||||||
import { load_config, log } from '../nwa-lib/index.js';
|
import { load_config, log, getUptimeDHMS, getOSUptimeDHMS } from '../nwa-lib/index.js';
|
||||||
import dgram from 'dgram';
|
import dgram from 'dgram';
|
||||||
import mqtt from 'mqtt';
|
import mqtt from 'mqtt';
|
||||||
import os from 'os';
|
import fs from 'fs';
|
||||||
import { exec } from 'child_process';
|
import os, { platform } from 'os';
|
||||||
|
import { exec, execSync } from 'child_process';
|
||||||
|
import { get } from 'http';
|
||||||
|
|
||||||
const config = load_config();
|
const config = load_config();
|
||||||
|
|
||||||
|
|
@ -19,6 +20,7 @@ let mqttClient = null;
|
||||||
let clusterConfig = null;
|
let clusterConfig = null;
|
||||||
let updating = false;
|
let updating = false;
|
||||||
let telemetryTimer = null;
|
let telemetryTimer = null;
|
||||||
|
let lastCpuInfo = null;
|
||||||
|
|
||||||
// -------------------------------
|
// -------------------------------
|
||||||
// Start UDP discovery
|
// Start UDP discovery
|
||||||
|
|
@ -237,6 +239,8 @@ function handleClusterStatus(payload) {
|
||||||
const clusterVersion = payload.msg?.sdl?.version;
|
const clusterVersion = payload.msg?.sdl?.version;
|
||||||
const updateCmd = payload.msg?.sdl?.update_cmd;
|
const updateCmd = payload.msg?.sdl?.update_cmd;
|
||||||
const localVersion = config.package.version;
|
const localVersion = config.package.version;
|
||||||
|
const uptime = payload.msg?.sdl?.uptime_secs;
|
||||||
|
|
||||||
|
|
||||||
if (!clusterVersion) return;
|
if (!clusterVersion) return;
|
||||||
|
|
||||||
|
|
@ -245,6 +249,11 @@ function handleClusterStatus(payload) {
|
||||||
`${module}: version mismatch detected (cluster=${clusterVersion}, local=${localVersion})`
|
`${module}: version mismatch detected (cluster=${clusterVersion}, local=${localVersion})`
|
||||||
);
|
);
|
||||||
triggerWorkerUpdate(clusterVersion, updateCmd);
|
triggerWorkerUpdate(clusterVersion, updateCmd);
|
||||||
|
} else if (uptime < 5) { // sld-mgr restarts for Development Testing
|
||||||
|
log(
|
||||||
|
`${module}: version mismatch detected (cluster=${clusterVersion}, local=${localVersion})`
|
||||||
|
);
|
||||||
|
triggerWorkerUpdate(clusterVersion, updateCmd);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -306,42 +315,137 @@ function startTelemetry() {
|
||||||
}, telemetryInterval);
|
}, telemetryInterval);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add this function for per-CPU usage
|
||||||
|
function getPerCPUUsage() {
|
||||||
|
const cpus = os.cpus();
|
||||||
|
|
||||||
|
if (!lastCpuInfo || !lastCpuInfo.perCpu) {
|
||||||
|
// First call - initialize
|
||||||
|
const perCpu = cpus.map(cpu => {
|
||||||
|
let total = 0;
|
||||||
|
for (let type in cpu.times) {
|
||||||
|
total += cpu.times[type];
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
idle: cpu.times.idle,
|
||||||
|
total: total
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
lastCpuInfo = lastCpuInfo || {};
|
||||||
|
lastCpuInfo.perCpu = perCpu;
|
||||||
|
|
||||||
|
return cpus.map(() => 0); // Return 0% for all CPUs on first call
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate per-CPU usage
|
||||||
|
const usage = cpus.map((cpu, i) => {
|
||||||
|
let total = 0;
|
||||||
|
for (let type in cpu.times) {
|
||||||
|
total += cpu.times[type];
|
||||||
|
}
|
||||||
|
|
||||||
|
const idleDelta = cpu.times.idle - lastCpuInfo.perCpu[i].idle;
|
||||||
|
const totalDelta = total - lastCpuInfo.perCpu[i].total;
|
||||||
|
|
||||||
|
const cpuUsage = 100 - (100 * idleDelta / totalDelta);
|
||||||
|
|
||||||
|
// Update stored values
|
||||||
|
lastCpuInfo.perCpu[i] = {
|
||||||
|
idle: cpu.times.idle,
|
||||||
|
total: total
|
||||||
|
};
|
||||||
|
|
||||||
|
return Math.max(0, Math.min(100, Math.round(cpuUsage)));
|
||||||
|
});
|
||||||
|
|
||||||
|
return usage;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update getCPUUsagePercent to calculate average from per-CPU
|
||||||
|
function getCPUUsagePercent() {
|
||||||
|
const perCpuUsage = getPerCPUUsage();
|
||||||
|
if (perCpuUsage.length === 0) return 0;
|
||||||
|
|
||||||
|
const total = perCpuUsage.reduce((sum, usage) => sum + usage, 0);
|
||||||
|
return Math.round(total / perCpuUsage.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------------------
|
||||||
|
// Publish telemetry
|
||||||
|
// -------------------------------
|
||||||
function publishTelemetry(topic) {
|
function publishTelemetry(topic) {
|
||||||
if (!mqttClient || !authorized) return;
|
if (!mqttClient || !authorized) return;
|
||||||
|
|
||||||
const gpuCount = Array.isArray(config.host.gpu) ? config.host.gpu.length : 0;
|
const gpuCount = Array.isArray(config.host.gpu) ? config.host.gpu.length : 0;
|
||||||
|
|
||||||
// ✅ Convert GB to bytes
|
// Get real-time memory usage
|
||||||
const memoryBytes = config.host.memory.total_gb
|
const totalMem = os.totalmem();
|
||||||
? config.host.memory.total_gb * 1024 * 1024 * 1024
|
const freeMem = os.freemem();
|
||||||
|
const usedMem = totalMem - freeMem;
|
||||||
|
|
||||||
|
// Get CPU usage
|
||||||
|
const perCpuUsage = getPerCPUUsage();
|
||||||
|
// const totalCpuUsage = getCPUUsagePercent();
|
||||||
|
const totalCpuUsage = perCpuUsage.length > 0
|
||||||
|
? Math.round(perCpuUsage.reduce((sum, usage) => sum + usage, 0) / perCpuUsage.length)
|
||||||
: 0;
|
: 0;
|
||||||
|
|
||||||
|
const now = new Date();
|
||||||
|
|
||||||
const telemetry = {
|
const telemetry = {
|
||||||
ts: new Date().toISOString(),
|
ts: now.toISOString(),
|
||||||
|
ts_utime: Math.floor(now.getTime() / 1000),
|
||||||
sdl_id: config.identity.sdl_id,
|
sdl_id: config.identity.sdl_id,
|
||||||
|
sdl_ver: config.sdl.version,
|
||||||
role: 'sdl-wkr',
|
role: 'sdl-wkr',
|
||||||
host: config.identity.hostname,
|
host: config.identity.hostname,
|
||||||
type: 'worker-telemetry',
|
type: 'worker-telemetry',
|
||||||
msg: {
|
msg: {
|
||||||
sdl_id: config.identity.sdl_id,
|
system: {
|
||||||
hostname: config.identity.hostname,
|
platform: config.host.os.platform,
|
||||||
status: 'active',
|
arch: config.host.cpu.arch,
|
||||||
|
distro: config.host.os.pretty_name,
|
||||||
|
distro_name: config.host.os.name,
|
||||||
|
distro_version: config.host.os.version,
|
||||||
|
hardware: getSystemHardware()
|
||||||
|
},
|
||||||
|
uptime: {
|
||||||
|
process: Math.floor(process.uptime()),
|
||||||
|
proc_dhms: getUptimeDHMS(),
|
||||||
|
system: Math.floor(os.uptime()),
|
||||||
|
sys_dhms: getOSUptimeDHMS()
|
||||||
|
},
|
||||||
resources: {
|
resources: {
|
||||||
cpus: {
|
cpus: {
|
||||||
allocated: config.host.cpu.cores_logical,
|
allocated: config.host.cpu.cores_logical,
|
||||||
available: config.host.cpu.cores_logical,
|
available: config.host.cpu.cores_logical
|
||||||
used: 0
|
|
||||||
},
|
},
|
||||||
memory: {
|
memory: {
|
||||||
allocated: memoryBytes, // ✅ Fixed
|
allocated: totalMem
|
||||||
available: memoryBytes, // ✅ Fixed
|
|
||||||
used: 0
|
|
||||||
},
|
},
|
||||||
gpus: {
|
gpus: {
|
||||||
allocated: gpuCount,
|
allocated: gpuCount,
|
||||||
available: gpuCount,
|
available: gpuCount
|
||||||
used: 0
|
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
usage: {
|
||||||
|
cpu: {
|
||||||
|
total: totalCpuUsage,
|
||||||
|
per_core: perCpuUsage
|
||||||
|
},
|
||||||
|
memory: {
|
||||||
|
total: totalMem,
|
||||||
|
used: usedMem,
|
||||||
|
free: freeMem,
|
||||||
|
percent_used: Math.round((usedMem / totalMem) * 100)
|
||||||
|
},
|
||||||
|
gpu: getGPUUsage()
|
||||||
|
},
|
||||||
|
load: {
|
||||||
|
1: os.loadavg()[0],
|
||||||
|
5: os.loadavg()[1],
|
||||||
|
15: os.loadavg()[2]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -358,6 +462,143 @@ function publishTelemetry(topic) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Add GPU telemetry function (exec-based)
|
||||||
|
function getGPUUsage() {
|
||||||
|
const gpuCount = Array.isArray(config.host.gpu) ? config.host.gpu.length : 0;
|
||||||
|
|
||||||
|
if (gpuCount === 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Try nvidia-smi for NVIDIA GPUs
|
||||||
|
const output = execSync(
|
||||||
|
'nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits',
|
||||||
|
{ encoding: 'utf8', timeout: 5000 }
|
||||||
|
).toString();
|
||||||
|
|
||||||
|
const lines = output.trim().split('\n');
|
||||||
|
const gpus = lines.map((line, index) => {
|
||||||
|
const [util, memUsed, memTotal] = line.split(',').map(v => parseInt(v.trim()));
|
||||||
|
return {
|
||||||
|
id: index,
|
||||||
|
utilization: util,
|
||||||
|
memory_used_mb: memUsed,
|
||||||
|
memory_total_mb: memTotal,
|
||||||
|
memory_percent: Math.round((memUsed / memTotal) * 100)
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Calculate totals
|
||||||
|
const totalUtil = Math.round(gpus.reduce((sum, gpu) => sum + gpu.utilization, 0) / gpus.length);
|
||||||
|
const totalMemUsed = gpus.reduce((sum, gpu) => sum + gpu.memory_used_mb, 0);
|
||||||
|
const totalMemTotal = gpus.reduce((sum, gpu) => sum + gpu.memory_total_mb, 0);
|
||||||
|
|
||||||
|
return {
|
||||||
|
count: gpus.length,
|
||||||
|
total_utilization: totalUtil,
|
||||||
|
total_memory_used_mb: totalMemUsed,
|
||||||
|
total_memory_total_mb: totalMemTotal,
|
||||||
|
total_memory_percent: Math.round((totalMemUsed / totalMemTotal) * 100),
|
||||||
|
gpus: gpus
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
// nvidia-smi not available or failed
|
||||||
|
return { error: 'unavailable' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
function getSystemHardware() {
|
||||||
|
const hardware = {
|
||||||
|
type: 'unknown',
|
||||||
|
manufacturer: 'Unknown',
|
||||||
|
model: 'Unknown',
|
||||||
|
cpu: config.host.cpu.model
|
||||||
|
};
|
||||||
|
|
||||||
|
// Only run Linux-specific detection on Linux
|
||||||
|
if (config.host.os.platform !== 'linux') {
|
||||||
|
return hardware;
|
||||||
|
}
|
||||||
|
|
||||||
|
// === Linux-specific detection below ===
|
||||||
|
|
||||||
|
// Check for Raspberry Pi FIRST
|
||||||
|
try {
|
||||||
|
const piModel = fs.readFileSync('/proc/device-tree/model', 'utf8')
|
||||||
|
.replace(/\0/g, '')
|
||||||
|
.replace(/Raspberry Pi/, '')
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
if (piModel) {
|
||||||
|
hardware.type = 'hw'; // ✅ Changed from 'physical'
|
||||||
|
hardware.manufacturer = 'Raspberry Pi';
|
||||||
|
hardware.model = piModel;
|
||||||
|
return hardware;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
// Not a Pi, continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for hypervisor flag
|
||||||
|
try {
|
||||||
|
const cpuinfo = fs.readFileSync('/proc/cpuinfo', 'utf8');
|
||||||
|
if (cpuinfo.includes('hypervisor')) {
|
||||||
|
hardware.type = 'vm';
|
||||||
|
|
||||||
|
// Detect hypervisor type from DMI
|
||||||
|
try {
|
||||||
|
const manufacturer = fs.readFileSync('/sys/class/dmi/id/sys_vendor', 'utf8').trim();
|
||||||
|
const product = fs.readFileSync('/sys/class/dmi/id/product_name', 'utf8').trim();
|
||||||
|
|
||||||
|
hardware.manufacturer = manufacturer;
|
||||||
|
|
||||||
|
// Set model to hypervisor type
|
||||||
|
if (manufacturer.includes('QEMU') || product.includes('KVM')) {
|
||||||
|
hardware.model = 'KVM';
|
||||||
|
} else if (manufacturer.includes('VMware')) {
|
||||||
|
hardware.model = 'VMware';
|
||||||
|
} else if (manufacturer.includes('innotek')) {
|
||||||
|
hardware.model = 'VirtualBox';
|
||||||
|
} else if (manufacturer.includes('Microsoft')) {
|
||||||
|
hardware.model = 'Hyper-V';
|
||||||
|
} else if (manufacturer.includes('Xen')) {
|
||||||
|
hardware.model = 'Xen';
|
||||||
|
} else {
|
||||||
|
hardware.model = 'Unknown VM';
|
||||||
|
}
|
||||||
|
|
||||||
|
return hardware;
|
||||||
|
} catch (err) {
|
||||||
|
hardware.manufacturer = 'Unknown';
|
||||||
|
hardware.model = 'Unknown VM';
|
||||||
|
return hardware;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
// Can't read cpuinfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// No hypervisor flag - likely physical, get DMI info
|
||||||
|
try {
|
||||||
|
const manufacturer = fs.readFileSync('/sys/class/dmi/id/sys_vendor', 'utf8').trim();
|
||||||
|
const product = fs.readFileSync('/sys/class/dmi/id/product_name', 'utf8').trim();
|
||||||
|
|
||||||
|
hardware.type = 'hw'; // ✅ Changed from 'physical'
|
||||||
|
hardware.manufacturer = manufacturer;
|
||||||
|
hardware.model = product;
|
||||||
|
|
||||||
|
return hardware;
|
||||||
|
} catch (err) {
|
||||||
|
// DMI not available
|
||||||
|
}
|
||||||
|
|
||||||
|
return hardware;
|
||||||
|
}
|
||||||
|
|
||||||
function stopTelemetry() {
|
function stopTelemetry() {
|
||||||
if (telemetryTimer) {
|
if (telemetryTimer) {
|
||||||
clearInterval(telemetryTimer);
|
clearInterval(telemetryTimer);
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
{
|
{
|
||||||
"name": "sdl-wkr",
|
"name": "sdl-wkr",
|
||||||
"description": "Software Defined Laboratory: Worker (sdl-wkr)",
|
"description": "Software Defined Laboratory: Worker (sdl-wkr)",
|
||||||
"version": "0.3.7",
|
"version": "0.3.9",
|
||||||
"version_date": "2026-02-01",
|
"version_date": "2026-02-08",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# install-sdl-wkr.sh - Script to install SDL Worker
|
# install-sdl-wkr.sh - Script to install SDL Worker
|
||||||
# Usage: curl -s http://<SDL_MGR_IP>:8081/dist/install-sdl-wkr.sh | bash -s <SDL_MGR_IP>
|
# Usage: curl -s http://<SDL_MGR_IP>:<SDL_MGR_PORT>/dist/install-sdl-wkr.sh | bash -s <SDL_MGR_IP>:<SDL_MGR_PORT>
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# ------------------------------------------------------------
|
# ------------------------------------------------------------
|
||||||
|
|
@ -79,11 +79,11 @@ install_nodejs() {
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
|
||||||
NODE_VER=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.nodejs.version')
|
NODE_VER=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.nodejs.version')
|
||||||
NODE_DIR="${SDL_HOME}/nodejs"
|
NODE_DIR="${SDL_HOME}/nodejs"
|
||||||
NODE_VER_BASE="node-${NODE_VER}-${OS_PLATFORM}-${OS_ARCH}"
|
NODE_VER_BASE="node-${NODE_VER}-${OS_PLATFORM}-${OS_ARCH}"
|
||||||
NODE_FILE="node-${NODE_VER}-${OS_PLATFORM}-${OS_ARCH}.${EXT}"
|
NODE_FILE="node-${NODE_VER}-${OS_PLATFORM}-${OS_ARCH}.${EXT}"
|
||||||
NODE_URL="http://${SDL_MGR_IP}:8081/dist/${NODE_FILE}"
|
NODE_URL="http://${SDL_MGR_IP}:${SDL_MGR_PORT}/dist/${NODE_FILE}"
|
||||||
|
|
||||||
|
|
||||||
#echo "NodeJS Version: ${NODE_VER}"
|
#echo "NodeJS Version: ${NODE_VER}"
|
||||||
|
|
@ -138,19 +138,21 @@ if [ -z "$1" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
SDL_MGR_IP=$1
|
ARG1=$1 # SDL_MGR_IP:PORT
|
||||||
|
SDL_MGR_IP=$(echo "$ARG1" | cut -d ':' -f 1)
|
||||||
|
SDL_MGR_PORT=$(echo "$ARG1" | cut -d ':' -f 2)
|
||||||
|
|
||||||
# SDL_HOME=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.dirs.sdlhome')
|
# SDL_HOME=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.dirs.sdlhome')
|
||||||
SDL_HOME=$HOME/.sdl
|
SDL_HOME=$HOME/.sdl
|
||||||
|
|
||||||
|
|
||||||
SDL_VER=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.version')
|
SDL_VER=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.version')
|
||||||
SDL_DESC=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.description')
|
SDL_DESC=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.description')
|
||||||
SDL_ABBR=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.abbr')
|
# SDL_ABBR=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.abbr')
|
||||||
SDL_COPYRT=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.copyright')
|
SDL_COPYRT=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.copyright')
|
||||||
|
|
||||||
echo "###############################################"
|
echo "###############################################"
|
||||||
echo "${SDL_DESC} (${SDL_ABBR})"
|
# echo "${SDL_DESC} (${SDL_ABBR})"
|
||||||
echo "Version: ${SDL_VER}"
|
echo "Version: ${SDL_VER}"
|
||||||
echo "Copyright: (C) ${SDL_COPYRT}"
|
echo "Copyright: (C) ${SDL_COPYRT}"
|
||||||
echo "Installing SDL Worker..."
|
echo "Installing SDL Worker..."
|
||||||
|
|
@ -160,7 +162,7 @@ echo ""
|
||||||
|
|
||||||
TMP_DIR="${SDL_HOME}/tmp"
|
TMP_DIR="${SDL_HOME}/tmp"
|
||||||
SDL_WKR_DIR="${SDL_HOME}/sdl-wkr"
|
SDL_WKR_DIR="${SDL_HOME}/sdl-wkr"
|
||||||
SDL_WKR_VERSION=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.package.version')
|
SDL_WKR_VERSION=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.package.version')
|
||||||
SDL_WKR_TGZ="sdl-wkr_${SDL_WKR_VERSION}.tgz"
|
SDL_WKR_TGZ="sdl-wkr_${SDL_WKR_VERSION}.tgz"
|
||||||
SDL_CONF_DIR="${SDL_HOME}/conf"
|
SDL_CONF_DIR="${SDL_HOME}/conf"
|
||||||
SLD_LOGS_DIR="${SDL_HOME}/logs"
|
SLD_LOGS_DIR="${SDL_HOME}/logs"
|
||||||
|
|
@ -177,8 +179,8 @@ mkdir -p "${TMP_DIR}"
|
||||||
|
|
||||||
|
|
||||||
# Download the latest sdl-wkr version to TMP_DIR
|
# Download the latest sdl-wkr version to TMP_DIR
|
||||||
WKR_TGZ_URL="http://${SDL_MGR_IP}:8081/dist/${SDL_WKR_TGZ}"
|
WKR_TGZ_URL="http://${SDL_MGR_IP}:${SDL_MGR_PORT}/dist/${SDL_WKR_TGZ}"
|
||||||
WKR_TGZ_SHA256_URL="http://${SDL_MGR_IP}:8081/dist/${SDL_WKR_TGZ}.sha256"
|
WKR_TGZ_SHA256_URL="http://${SDL_MGR_IP}:${SDL_MGR_PORT}/dist/${SDL_WKR_TGZ}.sha256"
|
||||||
|
|
||||||
curl -sSL --fail "${WKR_TGZ_URL}" -o "${TMP_DIR}/${SDL_WKR_TGZ}" || { echo "Failed to download sdl-wkr"; exit 1; }
|
curl -sSL --fail "${WKR_TGZ_URL}" -o "${TMP_DIR}/${SDL_WKR_TGZ}" || { echo "Failed to download sdl-wkr"; exit 1; }
|
||||||
curl -sSL --fail "${WKR_TGZ_SHA256_URL}" -o "${TMP_DIR}/${SDL_WKR_TGZ}.sha256" || { echo "Failed to download sdl-wkr SHA256"; exit 1; }
|
curl -sSL --fail "${WKR_TGZ_SHA256_URL}" -o "${TMP_DIR}/${SDL_WKR_TGZ}.sha256" || { echo "Failed to download sdl-wkr SHA256"; exit 1; }
|
||||||
|
|
@ -214,7 +216,7 @@ cp "$SDL_WKR_DIR/current/scripts/sdl-wkr.service" ~/.config/systemd/user/
|
||||||
|
|
||||||
|
|
||||||
# Install Node.js idempotently
|
# Install Node.js idempotently
|
||||||
NODE_VER=$(curl -s http://${SDL_MGR_IP}:8081/api/config | jq -r '.nodejs.version')
|
NODE_VER=$(curl -s http://${SDL_MGR_IP}:${SDL_MGR_PORT}/api/config | jq -r '.nodejs.version')
|
||||||
NODE_DIR="${SDL_HOME}/nodejs"
|
NODE_DIR="${SDL_HOME}/nodejs"
|
||||||
NODE_BIN="${NODE_DIR}/current/bin/node"
|
NODE_BIN="${NODE_DIR}/current/bin/node"
|
||||||
NPM_BIN="${NODE_DIR}/current/bin/npm"
|
NPM_BIN="${NODE_DIR}/current/bin/npm"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# start-sdl-wkr.sh - Script to start SDL Worker
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
use_systemd() {
|
||||||
|
# Check if systemd user mode is available and service exists
|
||||||
|
if systemctl --user status >/dev/null 2>&1; then
|
||||||
|
# Check if our service file exists
|
||||||
|
if systemctl --user list-unit-files | grep -q "^sdl-wkr.service"; then
|
||||||
|
return 0 # Use systemd
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 1 # Use manual scripts
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get $SDL_HOME directory from SCRIPT PATH
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
SDL_HOME="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
|
NODE_BIN_DIR="${SDL_HOME}/nodejs/current/bin"
|
||||||
|
NODE_BIN="${NODE_BIN_DIR}/node"
|
||||||
|
NPM_BIN="${NODE_BIN_DIR}/npm"
|
||||||
|
|
||||||
|
SDL_WKR_APP_DIR="${SDL_HOME}/sdl-wkr/current/app"
|
||||||
|
|
||||||
|
# Check that sdl-wkr is not already running
|
||||||
|
SDL_WKR_PID=""
|
||||||
|
for pid in $(pgrep -f 'node' 2>/dev/null || true); do
|
||||||
|
cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true)
|
||||||
|
if [[ "$cwd" == *"sdl-wkr"* ]]; then
|
||||||
|
SDL_WKR_PID="$pid"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -n "$SDL_WKR_PID" ]]; then
|
||||||
|
echo "ERROR: SDL Worker is already running with PID ${SDL_WKR_PID}"
|
||||||
|
echo " Stop it first: kill ${SDL_WKR_PID}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Verify paths exist
|
||||||
|
if [[ ! -f "$NODE_BIN" ]]; then
|
||||||
|
echo "ERROR: Node.js not found at: $NODE_BIN"
|
||||||
|
echo " Run install-sdl.sh first"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -d "$SDL_WKR_APP_DIR" ]]; then
|
||||||
|
echo "ERROR: SDL Worker app not found at: $SDL_WKR_APP_DIR"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Starting SDL Worker..."
|
||||||
|
echo " App Dir: ${SDL_WKR_APP_DIR}"
|
||||||
|
echo " Node: ${NODE_BIN}"
|
||||||
|
|
||||||
|
export PATH="${NODE_BIN_DIR}:${PATH}"
|
||||||
|
cd "${SDL_WKR_APP_DIR}"
|
||||||
|
|
||||||
|
if use_systemd; then
|
||||||
|
systemctl --user start sdl-wkr
|
||||||
|
else
|
||||||
|
# Manual background start logic
|
||||||
|
"${NPM_BIN}" start &
|
||||||
|
fi
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# status-sdl-wkr.sh - Script to show status of SDL Worker
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
use_systemd() {
|
||||||
|
# Check if systemd user mode is available and service exists
|
||||||
|
if systemctl --user status >/dev/null 2>&1; then
|
||||||
|
# Check if our service file exists
|
||||||
|
if systemctl --user list-unit-files | grep -q "^sdl-wkr.service"; then
|
||||||
|
return 0 # Use systemd
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 1 # Use manual scripts
|
||||||
|
}
|
||||||
|
|
||||||
|
if use_systemd; then
|
||||||
|
systemctl --user status sdl-wkr
|
||||||
|
else
|
||||||
|
# Get sdl-wkr PID
|
||||||
|
SDL_MGR_PID=""
|
||||||
|
for pid in $(pgrep -f 'node' 2>/dev/null || true); do
|
||||||
|
cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true)
|
||||||
|
if [[ "$cwd" == *"sdl-wkr"* ]]; then
|
||||||
|
SDL_MGR_PID="$pid"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$SDL_MGR_PID" ]]; then
|
||||||
|
echo "SDL Worker is not running"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "SDL Worker is running with PID: ${SDL_MGR_PID}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
@ -0,0 +1,62 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# stop-sdl-wkr.sh - Script to stop SDL Worker
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
use_systemd() {
|
||||||
|
# Check if systemd user mode is available and service exists
|
||||||
|
if systemctl --user status >/dev/null 2>&1; then
|
||||||
|
# Check if our service file exists
|
||||||
|
if systemctl --user list-unit-files | grep -q "^sdl-wkr.service"; then
|
||||||
|
return 0 # Use systemd
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 1 # Use manual scripts
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get sdl-wkr PID
|
||||||
|
SDL_WKR_PID=""
|
||||||
|
for pid in $(pgrep -f 'node' 2>/dev/null || true); do
|
||||||
|
cwd=$(readlink /proc/$pid/cwd 2>/dev/null || true)
|
||||||
|
if [[ "$cwd" == *"sdl-wkr"* ]]; then
|
||||||
|
SDL_WKR_PID="$pid"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$SDL_WKR_PID" ]]; then
|
||||||
|
echo "SDL Worker is not running"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Stopping SDL Worker (PID: ${SDL_WKR_PID})..."
|
||||||
|
|
||||||
|
if use_systemd; then
|
||||||
|
systemctl --user stop sdl-wkr
|
||||||
|
else
|
||||||
|
# Manual PID-based stop logic
|
||||||
|
# Send SIGTERM for graceful shutdown
|
||||||
|
kill "$SDL_WKR_PID"
|
||||||
|
|
||||||
|
# Wait up to 10 seconds for process to exit
|
||||||
|
for i in {1..10}; do
|
||||||
|
if ! kill -0 "$SDL_WKR_PID" 2>/dev/null; then
|
||||||
|
echo "SDL Worker stopped successfully"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
# If still running after 10 seconds, force kill
|
||||||
|
if kill -0 "$SDL_WKR_PID" 2>/dev/null; then
|
||||||
|
echo "WARNING: Graceful shutdown failed, forcing kill..."
|
||||||
|
kill -9 "$SDL_WKR_PID"
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
if kill -0 "$SDL_WKR_PID" 2>/dev/null; then
|
||||||
|
echo "ERROR: Failed to stop SDL Worker"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "SDL Worker stopped"
|
||||||
Loading…
Reference in New Issue