diff --git a/README.md b/README.md index fde0343..b947bdf 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.1 | +| **Version** | 0.3.2 | | **Date** | 2026-01-31 | ## Overview @@ -23,7 +23,7 @@ See also: [Installation](docs/INSTALL.md) ``` $ git clone https://github.com/jhaverlack/sdl.git $ cd sdl -$ install-sdl.sh +$ ./install-sdl.sh ``` Point you browser to http://IP-ADDR:8081 diff --git a/changelog.json b/changelog.json index 12beb7b..0b16d57 100644 --- a/changelog.json +++ b/changelog.json @@ -1,10 +1,17 @@ { "releases": [ + { + "version": "0.3.2", + "date": "2026-01-31", + "maturity": "ALPHA", + "summary": "", + "notes": "" + }, { "version": "0.3.1", "date": "2026-01-31", "maturity": "ALPHA", - "summary": "", + "summary": "README Cleanup", "notes": "" }, { @@ -12,7 +19,7 @@ "date": "2026-01-31", "maturity": "ALPHA", "summary": "Initial Codebase", - "notes": "" + "notes": "Initial GitHub Release" } ] } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4babb40..d33e12e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,11 +5,26 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.1 | +| **Version** | 0.3.2 | | **Date** | 2026-01-31 | +## v0.3.2 - 2026-01-31 (ALPHA) + +- v0.3.2 +- Squetching Logs +- WORKING: join/authz +- WIP: Join Worker +- WIP: Join Process +- sdl-wkr Cluster Join +- Fixing Service Module Render Bug +- Start v0.3.2 + ## v0.3.1 - 2026-01-31 (ALPHA) +**Summary** +README Cleanup + +- v0.3.1 - Start v0.3.1 - Removing duplicate LICENSE file - README Cleanup @@ -19,6 +34,9 @@ **Summary** Initial Codebase +**Notes** +Initial GitHub Release + - Preparing GitHub Version - Start v0.3.0 - Initial commit diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 458fb53..252bc50 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -5,9 +5,12 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.1 | +| **Version** | 0.3.2 | | **Date** | 2026-01-31 | +## Overview + +The Software Defined Laboratory (SDL) project provides a generalized distributed computing platform for managing parallel computational workflows across a cluster of distributed nodes. SDL is a minimalistic High Performance Compute (HPC) platform focused on extreme minimizaton of technical debt associated configuration, deployability, and maintenance of the cluster. # Roadmap @@ -26,16 +29,16 @@ MVP - Minimum Viable Product Task List - [x] SDL_ID Node ID - [x] Version Updater Script - [x] Deploy sld-mgr to $SDL_HOME/sdl-mgr -- [ ] Create Worker Join Process -- [ ] Create Worker Heartbeat Process - [x] Create SDL Worker Dist Build Process - [x] Create SDL Worker Install to $SDL_HOME - [x] SDL Install Script +- [ ] Create SDL Worker Auto Update Process +- [x] Create Worker Join Process +- [ ] Create Worker Telementry Process - [ ] Data Storage Organizational Structure - [ ] MinIO S3 Storage Server -- [ ] Worker Heartbeat Hardware Inventory (CPU, RAM, GPU, etc) -- [ ] Create SDL Worker Auto Update Process -- [ ] Worker Heartbeat Load Status (CPU, RAM, GPU, Disk IO, etc) +- [ ] Worker Telemetry Hardware Inventory (CPU, RAM, GPU, etc) +- [ ] Worker Telementry Load Status (CPU, RAM, GPU, Disk IO, etc) # Design diff --git a/docs/INSTALL.md b/docs/INSTALL.md index be35a50..02fd478 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -49,7 +49,7 @@ git pull origin main (Debian, Ubuntu, Linux Mint, Zorin OS, Raspbian) ``` -sudo apt -y install gawk coreutils curl jq grep sed sha256sum tar unzip netcat-openbsd +sudo apt -y install gawk coreutils curl jq grep sed tar unzip netcat-openbsd ``` #### Installation diff --git a/metadata.json b/metadata.json index 9e574e2..c6005fa 100644 --- a/metadata.json +++ b/metadata.json @@ -3,7 +3,7 @@ "name": "sdl", "description": "Software Defined Laboratory", "abbr": "SDL", - "version": "0.3.1", + "version": "0.3.2", "version_date": "2026-01-31", "maturity": "ALPHA", "author": "John Haverlack", diff --git a/sdl-mgr/app/index.js b/sdl-mgr/app/index.js index 7cc9cdf..a81d329 100644 --- a/sdl-mgr/app/index.js +++ b/sdl-mgr/app/index.js @@ -11,7 +11,7 @@ const config = load_config(); log('======================================================================================='); log(config.package.name + ': STARTING: ' + config.package.description + ' v' + config.package.version); // log(JSON.stringify(config, null, 2)); -log(JSON.stringify(config.dirs, null, 2), false); +// log(JSON.stringify(config.dirs, null, 2), false); for (let m in config.modules) { // log(`Loading module: ${m}`); diff --git a/sdl-mgr/app/modules/mqtt/index.js b/sdl-mgr/app/modules/mqtt/index.js index 1c2b0d9..497a3cd 100644 --- a/sdl-mgr/app/modules/mqtt/index.js +++ b/sdl-mgr/app/modules/mqtt/index.js @@ -88,6 +88,6 @@ broker.on('publish', (packet, client) => { // console.log( // `Publish from ${client.id}: ${packet.topic}` // ); - log(`${module}: Publish from ${client.id}: ${packet.topic}`); + // log(`${module}: Publish from ${client.id}: ${packet.topic}`); } }); diff --git a/sdl-mgr/app/modules/nwa-lib/index.js b/sdl-mgr/app/modules/nwa-lib/index.js index 3ae3a4f..df7665f 100644 --- a/sdl-mgr/app/modules/nwa-lib/index.js +++ b/sdl-mgr/app/modules/nwa-lib/index.js @@ -146,6 +146,9 @@ function load_config() { config.package = JSON.parse(fs.readFileSync(path.join(config.dirs.app, 'package.json'), 'utf8')); config.dependencies = {} + // SDL Version + config.cluster.sdl_version = config.package.version + for (let d in config.package.dependencies) { config.dependencies[d] = JSON.parse(fs.readFileSync(path.join(config.dirs.app, 'node_modules', d, 'package.json'), 'utf8')); } @@ -158,6 +161,7 @@ function load_config() { ips: [], os: { + platform: null, id: null, name: null, version: null, @@ -181,6 +185,10 @@ function load_config() { }; + config.sdl = { + version: config.package.version + } + config.identity = { sdl_id: null, created_at: null, @@ -218,6 +226,8 @@ function load_config() { config.host.cpu.cores_logical = cpus.length; } + config.host.os.platform = os.platform(); + // ------------------------------- // OS Info from /etc/os-release (Linux) // ------------------------------- diff --git a/sdl-mgr/app/modules/sdl-mgr/index.js b/sdl-mgr/app/modules/sdl-mgr/index.js index d3c2618..9d1a3ba 100644 --- a/sdl-mgr/app/modules/sdl-mgr/index.js +++ b/sdl-mgr/app/modules/sdl-mgr/index.js @@ -1,14 +1,304 @@ const module = 'sdl-mgr'; // Module Name -import { load_config, log, ipToInt, intToIp, computeBroadcast, getUptimeDHMS } from '../nwa-lib/index.js'; +import { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeDHMS } from '../nwa-lib/index.js'; import mqtt from 'mqtt'; import os from 'os'; import dgram from 'dgram'; +import fs from 'fs'; +import path from 'path'; + + const config = load_config(); log(`Loaded module: ${module}`); // log(`${module}: Cluster Conf: ${JSON.stringify(config.cluster, null, 2)}`, true); +// log(`${module}: Dirs: ${JSON.stringify(config.dirs, null, 2)}`, true); + +function loadClusterState(config) { + const clusterDir = config.dirs.clstr; + const clusterFile = path.join(clusterDir, 'cluster.json'); + + // Ensure cluster directory exists + if (!fs.existsSync(clusterDir)) { + fs.mkdirSync(clusterDir, { recursive: true }); + } + + // If cluster.json doesn't exist, create initial state + if (!fs.existsSync(clusterFile)) { + const initialState = { + meta: { + updated: new Date().toISOString(), + started_at: new Date(getProcessStartTs()).toISOString(), + uptime: getUptimeDHMS(), + stats: { + workers: { + allocated: 0, + available: 0, + used: 0 + }, + resources: { + cpus: { + allocated: 0, + available: 0, + used: 0 + }, + memory: { + allocated: 0, + available: 0, + used: 0 + }, + gpus: { + allocated: 0, + available: 0, + used: 0 + } + } + } + }, + cluster: config.cluster, + "sdl-mgr": { + sdl_id: config.identity.sdl_id, + hostname: config.identity.hostname, + platform: config.host.os.platform, + arch: config.host.cpu.arch, + distro: config.host.os.pretty_name, + distro_name: config.host.os.name, + distro_version: config.host.os.version + }, + workers: {} + }; + + fs.writeFileSync(clusterFile, JSON.stringify(initialState, null, 2)); + return initialState; + } + + // Load existing cluster state + const data = fs.readFileSync(clusterFile, 'utf8'); + return JSON.parse(data); +} + +function saveClusterState(config, state) { + const clusterDir = config.dirs.clstr; + const clusterFile = path.join(clusterDir, 'cluster.json'); + + // Update metadata timestamp and uptime + state.meta.updated = new Date().toISOString(); + state.meta.uptime = getUptimeDHMS(); + + // Write to disk + try { + fs.writeFileSync(clusterFile, JSON.stringify(state, null, 2)); + log(`${module}: cluster state saved to ${clusterFile}`); + } catch (err) { + log(`${module}: failed to save cluster state: ${err}`); + throw err; + } +} + +function addWorker(state, workerData) { + const sdl_id = workerData.sdl_id; + + // Check if worker already exists + const isNewWorker = !state.workers[sdl_id]; + + // Add or update worker + state.workers[sdl_id] = { + sdl_id: workerData.sdl_id, + hostname: workerData.hostname, + version: workerData.sdl_version, + platform: workerData.platform, + arch: workerData.arch, + distro: workerData.distro || "Unknown", + distro_name: workerData.distro_name || "Unknown", + distro_version: workerData.distro_version || "Unknown", + resources: { + cpus: { + allocated: workerData.cpus || 0, + available: workerData.cpus || 0, + used: 0 + }, + memory: { + allocated: workerData.totalmem || 0, + available: workerData.totalmem || 0, + used: 0 + }, + gpus: { + allocated: workerData.gpus || 0, + available: workerData.gpus || 0, + used: 0 + } + }, + status: 'active', + joined_at: isNewWorker ? new Date().toISOString() : state.workers[sdl_id].joined_at, + last_seen: new Date().toISOString() + }; + + return state; +} + +function computeStats(state) { + const stats = { + workers: { + allocated: 0, + available: 0, + used: 0 + }, + resources: { + cpus: { + allocated: 0, + available: 0, + used: 0 + }, + memory: { + allocated: 0, + available: 0, + used: 0 + }, + gpus: { + allocated: 0, + available: 0, + used: 0 + } + } + }; + + // Count workers and aggregate resources + for (const worker of Object.values(state.workers)) { + stats.workers.allocated++; + + if (worker.status === 'active') { + stats.workers.available++; + + stats.resources.cpus.available += worker.resources.cpus.available; + stats.resources.memory.available += worker.resources.memory.available; + stats.resources.gpus.available += worker.resources.gpus.available; + } + + stats.resources.cpus.allocated += worker.resources.cpus.allocated; + stats.resources.cpus.used += worker.resources.cpus.used; + + stats.resources.memory.allocated += worker.resources.memory.allocated; + stats.resources.memory.used += worker.resources.memory.used; + + stats.resources.gpus.allocated += worker.resources.gpus.allocated; + stats.resources.gpus.used += worker.resources.gpus.used; + } + + state.meta.stats = stats; + return state; +} + +// Worker Join Handler +function startJoinHandler() { + const sdlCfg = config.modules[module]; + const mqttCfg = config.modules.mqtt; + + if (!sdlCfg?.enabled) { + log(`${module}: disabled, not starting join handler`); + return; + } + + if (!mqttCfg?.enabled) { + log(`${module}: MQTT disabled, cannot handle joins`); + return; + } + + // Load current cluster state + let clusterState = loadClusterState(config); + + const joinReqTopic = mqttCfg.topics?.[module]?.sub?.['sdl_join-req']; + const joinAuthzTopic = mqttCfg.topics?.[module]?.pub?.['sdl_join-authz']; + + if (!joinReqTopic || !joinAuthzTopic) { + log(`${module}: join topics not configured`); + return; + } + + const mqttUrl = `mqtt://127.0.0.1:${mqttCfg.mqtt_port}`; + const client = mqtt.connect(mqttUrl); + + client.on('connect', () => { + log(`${module}: join handler connected to MQTT at ${mqttUrl}`); + + client.subscribe(joinReqTopic, { qos: 1 }, err => { + if (err) { + log(`${module}: failed to subscribe to ${joinReqTopic}: ${err}`); + } else { + log(`${module}: subscribed to ${joinReqTopic}`); + } + }); + }); + + client.on('message', (topic, message) => { + if (topic !== joinReqTopic) return; + + try { + const joinReq = JSON.parse(message.toString()); + + log(`${module}: received join request from ${joinReq.host} (${joinReq.sdl_id})`); + + // Validate version + const workerData = joinReq.msg['sdl-wkr']; + const workerVersion = workerData?.sdl_version; + // const workerVersion = joinReq.msg?.worker?.version; + const clusterVersion = config.package.version; + + let authorized = false; + let reason = null; + + if (workerVersion === clusterVersion) { + authorized = true; + + // Add worker to cluster state + clusterState = addWorker(clusterState, workerData); + clusterState = computeStats(clusterState); + saveClusterState(config, clusterState); + + log(`${module}: worker ${joinReq.host} authorized and added to cluster`); + } else { + authorized = false; + reason = 'version_mismatch'; + log(`${module}: worker ${joinReq.host} denied - version mismatch (worker: ${workerVersion}, cluster: ${clusterVersion})`); + } + + // Publish authorization response + const authzResponse = { + ts: new Date().toISOString(), + sdl_id: config.identity.sdl_id, + role: 'sdl-mgr', + host: config.identity.hostname, + type: 'join-authz', + msg: { + sdl_id: joinReq.sdl_id, + authorized: authorized + } + }; + + client.publish( + joinAuthzTopic, + JSON.stringify(authzResponse), + { qos: 1 }, + err => { + if (err) { + log(`${module}: failed to publish join authz: ${err}`); + } else { + // log(`${module}: published join authz to ${joinAuthzTopic}`); + } + } + ); + + } catch (err) { + log(`${module}: failed to process join request: ${err}`); + } + }); + + client.on('error', err => { + log(`${module}: join handler MQTT error: ${err}`); + }); +} + + function startSDLStatusPub() { const sdlCfg = config.modules[module]; @@ -54,6 +344,32 @@ function startSDLStatusPub() { } function publishStatus(client, topic) { + //get ip addr + const interfaces = os.networkInterfaces(); + let ip_addr = null; + + for (const [iface, addrs] of Object.entries(interfaces)) { + for (const addr of addrs) { + // --- FILTERS --- + + // IPv4 only + if (addr.family !== 'IPv4') continue; + + // Skip loopback + if (addr.internal === true) continue; + + // Skip /32 networks (no broadcast: e.g. Tailscale) + const cidr = Number(addr.cidr?.split('/')[1]); + if (!Number.isInteger(cidr) || cidr >= 32) continue; + + // Compute broadcast address + const broadcastAddr = computeBroadcast(addr.address, addr.netmask); + if (!broadcastAddr) continue; + + ip_addr = addr.address + } + } + const status = { ts: new Date().toISOString(), sdl_id: config.identity.sdl_id, @@ -63,7 +379,8 @@ function publishStatus(client, topic) { msg: { sdl: { version: config.package.version, - uptime: getUptimeDHMS() + uptime: getUptimeDHMS(), + update_cmd: `curl -s http://${ip_addr}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${ip_addr}` }, cluster: config.cluster, modules: Object.fromEntries( @@ -83,7 +400,7 @@ function publishStatus(client, topic) { if (err) { log(`${module}: failed to publish status: ${err}`); } else { - log(`${module}: published status to ${topic}`); + // log(`${module}: published status to ${topic}`); } } ); @@ -123,7 +440,7 @@ function startUdpBeacon() { desc: '' }; - log(`${module}: DEBUG: cluster: ${JSON.stringify(cluster)}`); + log(`${module}: cluster: ${JSON.stringify(cluster)}`); if (!config.host?.network) { @@ -191,7 +508,8 @@ function startUdpBeacon() { host: addr.address, port: config.modules.web.port, api_config: '/api/config', - config_url: `http://${addr.address}:${config.modules.web.port}/api/config` + config_url: `http://${addr.address}:${config.modules.web.port}/api/config`, + web_ui_url: `http://${addr.address}:${config.modules.web.port}`, }, sdl_wkr_install_cmd: { curl: `curl -s http://${addr.address}:${config.modules.web.port}/dist/install-sdl-wkr.sh | bash -s ${addr.address}`, @@ -230,3 +548,5 @@ function startUdpBeacon() { // ------------------------------- startSDLStatusPub(); startUdpBeacon(); +loadClusterState(config); +startJoinHandler(); \ No newline at end of file diff --git a/sdl-mgr/app/package.json b/sdl-mgr/app/package.json index 16a23d6..ea7145d 100644 --- a/sdl-mgr/app/package.json +++ b/sdl-mgr/app/package.json @@ -1,6 +1,6 @@ { "name": "sdl-mgr", - "version": "0.3.1", + "version": "0.3.2", "version_date": "2026-01-31", "description": "Software Defined Laboratory: Manager (sdl-mgr)", "main": "index.js", diff --git a/sdl-mgr/conf/default-sdl-mgr-config.json b/sdl-mgr/conf/default-sdl-mgr-config.json index 966ea86..6cbcb5c 100644 --- a/sdl-mgr/conf/default-sdl-mgr-config.json +++ b/sdl-mgr/conf/default-sdl-mgr-config.json @@ -12,6 +12,7 @@ "dist": "SDLHOME/dist", "logs": "SDLHOME/logs", "data": "SDLHOME/data", + "clstr": "DATA/cluster", "proj": "DATA/proj", "jobs": "DATA/jobs", "conf": "SDLHOME/conf", diff --git a/sdl-mgr/conf/modules/mqtt.json b/sdl-mgr/conf/modules/mqtt.json index ad9b9ae..8da1b22 100644 --- a/sdl-mgr/conf/modules/mqtt.json +++ b/sdl-mgr/conf/modules/mqtt.json @@ -6,7 +6,7 @@ "topics":{ "sdl-mgr":{ "pub": { - "sdl_join-auth": "sdl/join/auth", + "sdl_join-authz": "sdl/join/authz", "sdl_cluster-status": "sdl/cluster/status", "sdl_cluster-workers": "sdl/cluster/workers" }, @@ -21,7 +21,7 @@ "sdl_cluster-telemetry": "sdl/cluster/telemetry" }, "sub": { - "sdl_join-auth": "sdl/join/auth", + "sdl_join-authz": "sdl/join/authz", "sdl_cluster-status": "sdl/cluster/status" } }, diff --git a/sdl-mgr/conf/modules/sdl-mgr.json b/sdl-mgr/conf/modules/sdl-mgr.json index 6222d93..5124e4d 100644 --- a/sdl-mgr/conf/modules/sdl-mgr.json +++ b/sdl-mgr/conf/modules/sdl-mgr.json @@ -9,6 +9,7 @@ }, "expiration_timeout": { "cluster_status": 60000, - "cluster_workers": 60000 + "cluster_workers": 60000, + "cluster_telemetry": 60000 } } \ No newline at end of file diff --git a/sdl-mgr/html/js/dash.js b/sdl-mgr/html/js/dash.js index ed97257..b398c78 100644 --- a/sdl-mgr/html/js/dash.js +++ b/sdl-mgr/html/js/dash.js @@ -314,12 +314,6 @@ function renderModulesTable(msg) { const modules = msg.msg.modules; let html = '
Service Modules
'; html += ''; - // - // - // - // - // - // html += ` `; for (const [name, info] of Object.entries(modules)) { @@ -330,7 +324,7 @@ function renderModulesTable(msg) { html += ` - + `; diff --git a/sdl-mgr/html/md/changelog.md b/sdl-mgr/html/md/changelog.md index 4babb40..d33e12e 100644 --- a/sdl-mgr/html/md/changelog.md +++ b/sdl-mgr/html/md/changelog.md @@ -5,11 +5,26 @@ | **Author** | John Haverlack | | **Copyright** | 2026 John Haverlack | | **License** | MIT | -| **Version** | 0.3.1 | +| **Version** | 0.3.2 | | **Date** | 2026-01-31 | +## v0.3.2 - 2026-01-31 (ALPHA) + +- v0.3.2 +- Squetching Logs +- WORKING: join/authz +- WIP: Join Worker +- WIP: Join Process +- sdl-wkr Cluster Join +- Fixing Service Module Render Bug +- Start v0.3.2 + ## v0.3.1 - 2026-01-31 (ALPHA) +**Summary** +README Cleanup + +- v0.3.1 - Start v0.3.1 - Removing duplicate LICENSE file - README Cleanup @@ -19,6 +34,9 @@ **Summary** Initial Codebase +**Notes** +Initial GitHub Release + - Preparing GitHub Version - Start v0.3.0 - Initial commit diff --git a/sdl-mgr/html/md/docs.md b/sdl-mgr/html/md/docs.md index be35a50..02fd478 100644 --- a/sdl-mgr/html/md/docs.md +++ b/sdl-mgr/html/md/docs.md @@ -49,7 +49,7 @@ git pull origin main (Debian, Ubuntu, Linux Mint, Zorin OS, Raspbian) ``` -sudo apt -y install gawk coreutils curl jq grep sed sha256sum tar unzip netcat-openbsd +sudo apt -y install gawk coreutils curl jq grep sed tar unzip netcat-openbsd ``` #### Installation diff --git a/sdl-wkr/app/index.js b/sdl-wkr/app/index.js index 7cc9cdf..a81d329 100644 --- a/sdl-wkr/app/index.js +++ b/sdl-wkr/app/index.js @@ -11,7 +11,7 @@ const config = load_config(); log('======================================================================================='); log(config.package.name + ': STARTING: ' + config.package.description + ' v' + config.package.version); // log(JSON.stringify(config, null, 2)); -log(JSON.stringify(config.dirs, null, 2), false); +// log(JSON.stringify(config.dirs, null, 2), false); for (let m in config.modules) { // log(`Loading module: ${m}`); diff --git a/sdl-wkr/app/modules/nwa-lib/index.js b/sdl-wkr/app/modules/nwa-lib/index.js index d6054cb..a0de696 100644 --- a/sdl-wkr/app/modules/nwa-lib/index.js +++ b/sdl-wkr/app/modules/nwa-lib/index.js @@ -1,3 +1,4 @@ +// Node Web App (NWA) Library Module import fs from 'fs'; import path from 'path'; import os from 'os'; @@ -6,12 +7,16 @@ import { hideBin } from 'yargs/helpers'; import { fileURLToPath } from 'url'; import crypto from 'crypto'; import { execFileSync } from 'child_process'; +import { get } from 'http'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); // console.log("DEBUG: __dirname: " + __dirname); +// Process start time (set once per Node.js process) +const PROCESS_START_TS = Date.now(); + const config = load_config(); function load_config() { @@ -153,6 +158,7 @@ function load_config() { ips: [], os: { + platform: null, id: null, name: null, version: null, @@ -176,6 +182,10 @@ function load_config() { }; + config.sdl = { + version: config.package.version + } + config.identity = { sdl_id: null, created_at: null, @@ -213,6 +223,8 @@ function load_config() { config.host.cpu.cores_logical = cpus.length; } + config.host.os.platform = os.platform(); + // ------------------------------- // OS Info from /etc/os-release (Linux) // ------------------------------- @@ -572,4 +584,26 @@ function pciShortId(id) { return m ? m[1].toLowerCase() : null; } -export { load_config, log, ipToInt, intToIp, computeBroadcast }; +function getProcessStartTs() { + return PROCESS_START_TS; +} + +function getUptimeMs() { + return Date.now() - PROCESS_START_TS; +} + +function getUptimeSec() { + return Math.floor((Date.now() - PROCESS_START_TS) / 1000); +} + +function getUptimeDHMS() { + const uptimeSec = getUptimeSec(); + const d = Math.floor(uptimeSec / 86400); + const h = Math.floor((uptimeSec % 86400) / 3600); + const m = Math.floor((uptimeSec % 3600) / 60); + const s = Math.floor(uptimeSec % 60); + return `${d}.${h}.${m}.${s}`; +} + + +export { load_config, log, ipToInt, intToIp, computeBroadcast, getProcessStartTs, getUptimeMs, getUptimeSec, getUptimeDHMS }; diff --git a/sdl-wkr/app/modules/sdl-wkr/index.js b/sdl-wkr/app/modules/sdl-wkr/index.js index fb921df..bb6dddd 100644 --- a/sdl-wkr/app/modules/sdl-wkr/index.js +++ b/sdl-wkr/app/modules/sdl-wkr/index.js @@ -3,18 +3,21 @@ import { load_config, log } from '../nwa-lib/index.js'; import dgram from 'dgram'; import mqtt from 'mqtt'; import os from 'os'; +import { exec } from 'child_process'; + const config = load_config(); log(`Loaded module: ${module}`); -// log(`${module}: CONFIG: ${JSON.stringify(config, null, 2)}`, true); - // ------------------------------- // Worker state // ------------------------------- let joined = false; +let authorized = false; let mqttClient = null; +let clusterConfig = null; +let updating = false; // ------------------------------- // Start UDP discovery @@ -43,11 +46,11 @@ function startUdpListener() { try { const beacon = JSON.parse(msg.toString()); - if (beacon?.sdl?.cluster?.id == null) return; - if (beacon?.sdl?.mqtt?.host == null) return; + if (beacon?.type !== 'udp-beacon') return; + if (beacon?.msg?.cluster?.id == null) return; + if (beacon?.msg?.mqtt?.host == null) return; log(`${module}: received SDL beacon from ${rinfo.address}`); - log(`${module}: UDP beacon: ${JSON.stringify(beacon)}`, false); handleBeacon(beacon); } catch { @@ -64,26 +67,37 @@ function startUdpListener() { // ------------------------------- // Handle beacon // ------------------------------- -function handleBeacon(beacon) { - const sdl = beacon.sdl; - const mqttInfo = sdl.mqtt; +async function handleBeacon(beacon) { + const beaconMsg = beacon.msg; + const mqttInfo = beaconMsg.mqtt; + const cluster = beaconMsg.cluster; - // Cluster match (strict) - const expectedClusterId = - config.modules[module].cluster_id || sdl.cluster.id; - - if (sdl.cluster.id !== expectedClusterId) { - log( - `${module}: ignoring beacon for cluster ${sdl.cluster.id}` - ); + // Optional: If cluster_id is configured, validate it. Otherwise auto-join first beacon. + const expectedClusterId = config.modules[module].cluster_id; + if (expectedClusterId && cluster.id !== expectedClusterId) { + log(`${module}: ignoring beacon for cluster ${cluster.id}`); return; } - log( - `${module}: joining cluster '${sdl.cluster.name}' via MQTT ${mqttInfo.host}:${mqttInfo.port}` - ); + log(`${module}: cluster '${cluster.name}' discovered via ${beacon.host}`); - joinCluster(mqttInfo, sdl.cluster); + // Fetch full cluster config from API + const configUrl = beaconMsg.web.config_url; + try { + const response = await fetch(configUrl); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + clusterConfig = await response.json(); + log(`${module}: fetched cluster config from ${configUrl}`); + } catch (err) { + log(`${module}: failed to fetch cluster config: ${err}`); + return; + } + + log(`${module}: joining cluster '${cluster.name}' via MQTT ${mqttInfo.host}:${mqttInfo.port}`); + + joinCluster(mqttInfo, cluster); } // ------------------------------- @@ -98,33 +112,161 @@ function joinCluster(mqttInfo, cluster) { mqttClient.on('connect', () => { log(`${module}: connected to MQTT broker at ${mqttUrl}`); + const mqttTopics = clusterConfig.modules.mqtt.topics[module]; + + // Subscribe to authz topic + const authzTopic = mqttTopics.sub['sdl_join-authz']; + mqttClient.subscribe(authzTopic, { qos: 1 }, err => { + if (err) { + log(`${module}: failed to subscribe to ${authzTopic}: ${err}`); + } else { + log(`${module}: subscribed to ${authzTopic}`); + } + }); + + const joinTopic = mqttTopics.pub['sdl_join-req']; + const joinPayload = { - type: 'sdl_worker_join', - timestamp: new Date().toISOString(), - sdl_wkr: config.identity.hostname, - sdl_id: config.identity.sdl_id + ts: new Date().toISOString(), + sdl_id: config.identity.sdl_id, + role: 'sdl-wkr', + host: config.identity.hostname, + type: 'join-request', + msg: { + cluster_id: cluster.id, + "sdl-wkr": { + sdl_id: config.identity.sdl_id, + hostname: config.identity.hostname, + sdl_version: config.package.version, + platform: config.host.os.platform, + arch: config.host.cpu.arch, + distro: config.host.os.pretty_name, + distro_name: config.host.os.name, + distro_version: config.host.os.version, + cpus: os.cpus().length, + totalmem: os.totalmem() + } + } }; mqttClient.publish( - mqttInfo.join_topic, + joinTopic, JSON.stringify(joinPayload), { qos: 1 }, err => { if (err) { - log(`${module}: failed to publish join message: ${err}`); + log(`${module}: failed to publish join request: ${err}`); } else { - log(`${module}: published join request to ${mqttInfo.join_topic}`); + log(`${module}: published join request to ${joinTopic}`); } } ); }); + mqttClient.on('message', (topic, message) => { + handleMqttMessage(topic, message); + }); + mqttClient.on('error', err => { log(`${module}: MQTT error: ${err}`); }); } +// ------------------------------- +// Handle MQTT messages +// ------------------------------- +function handleMqttMessage(topic, message) { + + try { + const payload = JSON.parse(message.toString()); + const mqttTopics = clusterConfig.modules.mqtt.topics[module]; + + if (topic === mqttTopics.sub['sdl_join-authz']) { + handleJoinAuthz(payload); + return; + } + + if ( + authorized && + topic === mqttTopics.sub['sdl_cluster-status'] + ) { + handleClusterStatus(payload); + } + + } catch (err) { + log(`${module}: failed to parse MQTT message: ${err}`); + } +} + +// ------------------------------- +// Handle join authorization +// ------------------------------- +function handleJoinAuthz(payload) { + // Check if this authz is for us + if (payload.msg?.sdl_id !== config.identity.sdl_id) { + return; + } + + if (payload.msg?.authorized === true) { + log(`${module}: join authorized by cluster`); + authorized = true; + + const mqttTopics = clusterConfig.modules.mqtt.topics[module]; + const statusTopic = mqttTopics.sub['sdl_cluster-status']; + + mqttClient.subscribe(statusTopic, { qos: 1 }, err => { + if (err) { + log(`${module}: failed to subscribe to ${statusTopic}: ${err}`); + } else { + log(`${module}: subscribed to ${statusTopic}`); + } + }); + } else { + log(`${module}: join denied: ${payload.msg?.reason || 'unknown'}`); + } +} + +function handleClusterStatus(payload) { + const clusterVersion = payload.msg?.sdl?.version; + const updateCmd = payload.msg?.sdl?.update_cmd; + const localVersion = config.package.version; + + if (!clusterVersion) return; + + if (clusterVersion !== localVersion) { + log( + `${module}: version mismatch detected (cluster=${clusterVersion}, local=${localVersion})` + ); + triggerWorkerUpdate(clusterVersion, updateCmd); + } +} + + +function triggerWorkerUpdate(targetVersion, updateCmd) { + if (updating) return; + updating = true; + + log( + `${module}: triggering self-update to SDL version ${targetVersion}` + ); + + // Stop reacting to further control messages + authorized = false; + + try { + exec(updateCmd, { stdio: 'inherit' }); + } catch (err) { + log(`${module}: failed to exec update command: ${err}`); + return; + } + + log(`${module}: exiting for self-update`); + process.exit(0); +} + + + // ------------------------------- // Entry point // ------------------------------- -startUdpListener(); +startUdpListener(); \ No newline at end of file diff --git a/sdl-wkr/app/package.json b/sdl-wkr/app/package.json index f360c3f..a427617 100644 --- a/sdl-wkr/app/package.json +++ b/sdl-wkr/app/package.json @@ -1,7 +1,7 @@ { "name": "sdl-wkr", "description": "Software Defined Laboratory: Worker (sdl-wkr)", - "version": "0.3.1", + "version": "0.3.2", "version_date": "2026-01-31", "main": "index.js", "type": "module", diff --git a/sdl-wkr/conf/default-sdl-wkr-config.json b/sdl-wkr/conf/default-sdl-wkr-config.json index ef898e7..0de61b6 100644 --- a/sdl-wkr/conf/default-sdl-wkr-config.json +++ b/sdl-wkr/conf/default-sdl-wkr-config.json @@ -6,8 +6,7 @@ "modules": "APP/modules", "logs": "SDLHOME/logs", "data": "SDLHOME/data", - "proj": "DATA/projects", - "jobs": "DATA/jobs", + "sdl-wkr": "DATA/sdl-wkr", "conf": "SDLHOME/conf", "modconf": "BASE/conf/modules" },
ModuleStatus
${name} ${icon}