From 85734b1a7b5f3971e746a171f1b3a5579f8dbcd8 Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Mon, 4 Nov 2024 12:00:30 +0100 Subject: [PATCH 01/16] feat(ms2/engines): initial mqtt endpoints --- .../lib/engines/mqtt-endpoints.ts | 72 +++++++++ src/management-system-v2/lib/env-vars.ts | 9 ++ src/management-system-v2/package.json | 3 +- yarn.lock | 151 +++++++++++++++++- 4 files changed, 230 insertions(+), 5 deletions(-) create mode 100644 src/management-system-v2/lib/engines/mqtt-endpoints.ts diff --git a/src/management-system-v2/lib/engines/mqtt-endpoints.ts b/src/management-system-v2/lib/engines/mqtt-endpoints.ts new file mode 100644 index 000000000..5276d9664 --- /dev/null +++ b/src/management-system-v2/lib/engines/mqtt-endpoints.ts @@ -0,0 +1,72 @@ +import mqtt from 'mqtt'; +import { env } from '@/lib/env-vars'; + +const mqttTimeout = 1000; + +const mqttCredentials = { + password: env.MQTT_PASSWORD, + username: env.MQTT_USERNAME, +}; + +const baseTopicPrefix = env.MQTT_BASETOPIC ? env.MQTT_BASETOPIC + '/' : ''; + +export function getClient(options?: mqtt.IClientOptions): Promise { + if (!env.MQTT_SERVER_ADDRESS) throw new Error('MQTT_SERVER_ADDRESS is not set'); + + return new Promise((res, rej) => { + const client = mqtt.connect(env.MQTT_SERVER_ADDRESS, { + ...mqttCredentials, + ...options, + }); + client.on('connect', () => res(client)); + client.on('error', (err) => rej(err)); + }); +} + +function subscribeToTopic(client: mqtt.MqttClient, topic: string) { + return new Promise((res, rej) => { + setTimeout(rej, mqttTimeout); // Timeout if the subscription takes too long + client.subscribe(topic, (err) => { + if (err) rej(err); + res(); + }); + }); +} + +function getEnginePrefix(engineId: string) { + return `${baseTopicPrefix}proceed-pms/engine/${engineId}`; +} + +export async function getEngines() { + const client = await getClient({ + connectTimeout: mqttTimeout, + }); + + const engines: { engineId: string; running: boolean; version: string }[] = []; + + await subscribeToTopic(client, `${getEnginePrefix('+')}/status`); + + // All retained messages are sent at once + // The broker should bundle them in one tcp packet, + // after it is parsed all messages are in the queue, and handled before close + // is handled, as the packets where pushed to the queue before the close event was emitted. + // This is of course subject to the implementation of the broker, + // however for a small amount of engines it should be fine. + await new Promise((res) => { + setTimeout(res, mqttTimeout); // Timeout in case we receive no messages + + client.on('message', (topic, message) => { + const match = topic.match(new RegExp(`^${getEnginePrefix('')}([^\/]+)\/status`)); + if (match) { + const engineId = match[1]; + const status = JSON.parse(message.toString()); + engines.push({ engineId, ...status }); + res(); + } + }); + }); + + await client.endAsync(); + + return engines; +} diff --git a/src/management-system-v2/lib/env-vars.ts b/src/management-system-v2/lib/env-vars.ts index 8face3c3b..1c68ebd40 100644 --- a/src/management-system-v2/lib/env-vars.ts +++ b/src/management-system-v2/lib/env-vars.ts @@ -26,6 +26,11 @@ const environmentVariables = { } }) .optional(), + + MQTT_SERVER_ADDRESS: z.string().url().optional(), + MQTT_USERNAME: z.string().optional(), + MQTT_PASSWORD: z.string().optional(), + MQTT_BASETOPIC: z.string().optional(), }, production: { NEXTAUTH_SECRET: z.string(), @@ -52,6 +57,10 @@ const environmentVariables = { TWITTER_CLIENT_SECRET: z.string(), SHARING_ENCRYPTION_SECRET: z.string(), + + MQTT_SERVER_ADDRESS: z.string().url(), + MQTT_USERNAME: z.string(), + MQTT_PASSWORD: z.string(), }, development: { SHARING_ENCRYPTION_SECRET: z.string().default('T8VB/r1dw0kJAXjanUvGXpDb+VRr4dV5y59BT9TBqiQ='), diff --git a/src/management-system-v2/package.json b/src/management-system-v2/package.json index cec0fd246..7011e1aef 100644 --- a/src/management-system-v2/package.json +++ b/src/management-system-v2/package.json @@ -67,7 +67,8 @@ "winston": "^3.3.3", "yup": "^0.32.9", "zod": "3.22.4", - "zustand": "4.5.2" + "zustand": "4.5.2", + "mqtt": "^5.10.1" }, "devDependencies": { "@tanstack/eslint-plugin-query": "5.28.11", diff --git a/yarn.lock b/yarn.lock index 94d728104..47b45281f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1303,6 +1303,13 @@ dependencies: regenerator-runtime "^0.14.0" +"@babel/runtime@^7.23.8", "@babel/runtime@^7.24.5": + version "7.26.0" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.0.tgz#8600c2f595f277c60815256418b85356a65173c1" + integrity sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw== + dependencies: + regenerator-runtime "^0.14.0" + "@babel/template@^7.22.15", "@babel/template@^7.24.0", "@babel/template@^7.3.3": version "7.24.0" resolved "https://registry.npmjs.org/@babel/template/-/template-7.24.0.tgz" @@ -3513,6 +3520,14 @@ "@types/prop-types" "*" csstype "^3.0.2" +"@types/readable-stream@^4.0.0", "@types/readable-stream@^4.0.5": + version "4.0.18" + resolved "https://registry.yarnpkg.com/@types/readable-stream/-/readable-stream-4.0.18.tgz#5d8d15d26c776500ce573cae580787d149823bfc" + integrity sha512-21jK/1j+Wg+7jVw1xnSwy/2Q1VgVjWuFssbYGTREPUBeZ+rqVFl2udq0IkxzPC0ZhOzVceUbyIACFZKLqKEBlA== + dependencies: + "@types/node" "*" + safe-buffer "~5.1.1" + "@types/responselike@^1.0.0": version "1.0.3" resolved "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.3.tgz" @@ -3620,6 +3635,13 @@ anymatch "^3.0.0" source-map "^0.6.0" +"@types/ws@^8.5.9": + version "8.5.12" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.12.tgz#619475fe98f35ccca2a2f6c137702d85ec247b7e" + integrity sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ== + dependencies: + "@types/node" "*" + "@types/yargs-parser@*": version "21.0.3" resolved "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz" @@ -5960,6 +5982,16 @@ bl@^4.0.2, bl@^4.0.3, bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" +bl@^6.0.8: + version "6.0.16" + resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.16.tgz#29b190f1a754e2d168de3dc8c74ed8d12bf78e6e" + integrity sha512-V/kz+z2Mx5/6qDfRCilmrukUXcXuCoXKg3/3hDvzKKoSUx8CJKudfIoT29XZc3UE9xBvxs5qictiHdprwtteEg== + dependencies: + "@types/readable-stream" "^4.0.0" + buffer "^6.0.3" + inherits "^2.0.4" + readable-stream "^4.2.0" + blob@0.0.5: version "0.0.5" resolved "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz" @@ -6352,6 +6384,14 @@ buffer@^5.1.0, buffer@^5.2.1, buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + builder-util-runtime@8.9.2: version "8.9.2" resolved "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-8.9.2.tgz" @@ -7359,6 +7399,11 @@ commist@^1.0.0: leven "^2.1.0" minimist "^1.1.0" +commist@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/commist/-/commist-3.2.0.tgz#da9c8e5f245ac21510badc4b10c46b5bcc9b56cd" + integrity sha512-4PIMoPniho+LqXmpS5d3NuGYncG6XWlkBSVGiWycL22dd42OYdUGil2CWuzklaJoNxyxUSpO4MKIBU94viWNAw== + common-ancestor-path@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/common-ancestor-path/-/common-ancestor-path-1.0.1.tgz#4f7d2d1394d91b7abdf51871c62f71eadb0182a7" @@ -10232,7 +10277,7 @@ eventemitter3@^5.0.1: resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz" integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA== -events@^3.0.0, events@^3.2.0: +events@^3.0.0, events@^3.2.0, events@^3.3.0: version "3.3.0" resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== @@ -10594,6 +10639,14 @@ fast-text-encoding@^1.0.0: resolved "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz" integrity sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w== +fast-unique-numbers@^8.0.13: + version "8.0.13" + resolved "https://registry.yarnpkg.com/fast-unique-numbers/-/fast-unique-numbers-8.0.13.tgz#3c87232061ff5f408a216e1f0121232f76f695d7" + integrity sha512-7OnTFAVPefgw2eBJ1xj2PGGR9FwYzSUso9decayHgCDX4sJkHLdcsYTytTg+tYv+wKF3U8gJuSBz2jJpQV4u/g== + dependencies: + "@babel/runtime" "^7.23.8" + tslib "^2.6.2" + fast-xml-parser@3.15.0: version "3.15.0" resolved "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-3.15.0.tgz" @@ -11943,6 +11996,11 @@ help-me@^3.0.0: glob "^7.1.6" readable-stream "^3.6.0" +help-me@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/help-me/-/help-me-5.0.0.tgz#b1ebe63b967b74060027c2ac61f9be12d354a6f6" + integrity sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg== + hex-color-regex@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/hex-color-regex/-/hex-color-regex-1.1.0.tgz" @@ -15885,7 +15943,7 @@ minimatch@^9.0.1: dependencies: brace-expansion "^2.0.1" -minimist@^1.1.0, minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: +minimist@^1.1.0, minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.8: version "1.2.8" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -16124,6 +16182,15 @@ mqtt-packet@^6.8.0: debug "^4.1.1" process-nextick-args "^2.0.1" +mqtt-packet@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/mqtt-packet/-/mqtt-packet-9.0.0.tgz#fd841854d8c0f1f5211b00de388c4ced45b59216" + integrity sha512-8v+HkX+fwbodsWAZIZTI074XIoxVBOmPeggQuDFCGg1SqNcC+uoRMWu7J6QlJPqIUIJXmjNYYHxBBLr1Y/Df4w== + dependencies: + bl "^6.0.8" + debug "^4.3.4" + process-nextick-args "^2.0.1" + mqtt@^4.3.7: version "4.3.8" resolved "https://registry.npmjs.org/mqtt/-/mqtt-4.3.8.tgz" @@ -16147,6 +16214,28 @@ mqtt@^4.3.7: ws "^7.5.5" xtend "^4.0.2" +mqtt@^5.10.1: + version "5.10.1" + resolved "https://registry.yarnpkg.com/mqtt/-/mqtt-5.10.1.tgz#d4f45ffdd825bad331c18f08796a744dabbe16de" + integrity sha512-hXCOki8sANoQ7w+2OzJzg6qMBxTtrH9RlnVNV8panLZgnl+Gh0J/t4k6r8Az8+C7y3KAcyXtn0mmLixyUom8Sw== + dependencies: + "@types/readable-stream" "^4.0.5" + "@types/ws" "^8.5.9" + commist "^3.2.0" + concat-stream "^2.0.0" + debug "^4.3.4" + help-me "^5.0.0" + lru-cache "^10.0.1" + minimist "^1.2.8" + mqtt-packet "^9.0.0" + number-allocator "^1.0.14" + readable-stream "^4.4.2" + reinterval "^1.1.0" + rfdc "^1.3.0" + split2 "^4.2.0" + worker-timers "^7.1.4" + ws "^8.17.1" + ms@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" @@ -16794,7 +16883,7 @@ num2fraction@^1.2.2: resolved "https://registry.npmjs.org/num2fraction/-/num2fraction-1.2.2.tgz" integrity sha512-Y1wZESM7VUThYY+4W+X4ySH2maqcA+p7UR+w8VWNWVAd6lwuXXWz/w/Cz43J/dI2I+PS6wD5N+bJUF+gjWvIqg== -number-allocator@^1.0.9: +number-allocator@^1.0.14, number-allocator@^1.0.9: version "1.0.14" resolved "https://registry.npmjs.org/number-allocator/-/number-allocator-1.0.14.tgz" integrity sha512-OrL44UTVAvkKdOdRQZIJpLkAdjXGTRda052sN4sO77bKEzYYqWKMBjQvrJFzqygI99gL6Z4u2xctPW1tB8ErvA== @@ -19300,6 +19389,17 @@ readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stre string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^4.2.0, readable-stream@^4.4.2: + version "4.5.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.5.2.tgz#9e7fc4c45099baeed934bff6eb97ba6cf2729e09" + integrity sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g== + dependencies: + abort-controller "^3.0.0" + buffer "^6.0.3" + events "^3.3.0" + process "^0.11.10" + string_decoder "^1.3.0" + readable-web-to-node-stream@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz" @@ -20748,6 +20848,11 @@ split2@^3.0.0, split2@^3.1.0: dependencies: readable-stream "^3.0.0" +split2@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4" + integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg== + sprintf-js@^1.1.2, sprintf-js@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz" @@ -21080,7 +21185,7 @@ string.prototype.trimstart@^1.0.8: define-properties "^1.2.1" es-object-atoms "^1.0.0" -string_decoder@^1.0.0, string_decoder@^1.1.1: +string_decoder@^1.0.0, string_decoder@^1.1.1, string_decoder@^1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== @@ -21945,6 +22050,11 @@ tslib@^2.0.0, tslib@^2.1.0, tslib@^2.4.0: resolved "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tslib@^2.6.2: + version "2.8.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== + tsscmp@1.0.6: version "1.0.6" resolved "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz" @@ -23265,6 +23375,34 @@ worker-farm@^1.7.0: dependencies: errno "~0.1.7" +worker-timers-broker@^6.1.8: + version "6.1.8" + resolved "https://registry.yarnpkg.com/worker-timers-broker/-/worker-timers-broker-6.1.8.tgz#08f64e5931b77fadc55f0c7388c077a7dd17e4c7" + integrity sha512-FUCJu9jlK3A8WqLTKXM9E6kAmI/dR1vAJ8dHYLMisLNB/n3GuaFIjJ7pn16ZcD1zCOf7P6H62lWIEBi+yz/zQQ== + dependencies: + "@babel/runtime" "^7.24.5" + fast-unique-numbers "^8.0.13" + tslib "^2.6.2" + worker-timers-worker "^7.0.71" + +worker-timers-worker@^7.0.71: + version "7.0.71" + resolved "https://registry.yarnpkg.com/worker-timers-worker/-/worker-timers-worker-7.0.71.tgz#f96138bafbcfaabea116603ce23956e05e76db6a" + integrity sha512-ks/5YKwZsto1c2vmljroppOKCivB/ma97g9y77MAAz2TBBjPPgpoOiS1qYQKIgvGTr2QYPT3XhJWIB6Rj2MVPQ== + dependencies: + "@babel/runtime" "^7.24.5" + tslib "^2.6.2" + +worker-timers@^7.1.4: + version "7.1.8" + resolved "https://registry.yarnpkg.com/worker-timers/-/worker-timers-7.1.8.tgz#f53072c396ac4264fd3027914f4ab793c92d90be" + integrity sha512-R54psRKYVLuzff7c1OTFcq/4Hue5Vlz4bFtNEIarpSiCYhpifHU3aIQI29S84o1j87ePCYqbmEJPqwBTf+3sfw== + dependencies: + "@babel/runtime" "^7.24.5" + tslib "^2.6.2" + worker-timers-broker "^6.1.8" + worker-timers-worker "^7.0.71" + "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" @@ -23396,6 +23534,11 @@ ws@^7.5.5: resolved "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== +ws@^8.17.1: + version "8.18.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" + integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== + ws@~7.4.2: version "7.4.6" resolved "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz" From b6b4b13b0a48ddfaeed665a9451f77f1e1ac3832 Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Tue, 5 Nov 2024 23:55:53 +0100 Subject: [PATCH 02/16] feat(ms2/engines): endpoints for engines --- .../lib/engines/endpoint.ts | 30 +++++++++ .../lib/engines/endpoints.json | 67 +++++++++++++++++++ .../{endpoints.ts => http-endpoints.ts} | 0 3 files changed, 97 insertions(+) create mode 100644 src/management-system-v2/lib/engines/endpoint.ts create mode 100644 src/management-system-v2/lib/engines/endpoints.json rename src/management-system-v2/lib/engines/{endpoints.ts => http-endpoints.ts} (100%) diff --git a/src/management-system-v2/lib/engines/endpoint.ts b/src/management-system-v2/lib/engines/endpoint.ts new file mode 100644 index 000000000..140529bca --- /dev/null +++ b/src/management-system-v2/lib/engines/endpoint.ts @@ -0,0 +1,30 @@ +type EndpointSchema = typeof import('./endpoints.json'); +type Endpoints = EndpointSchema; +type Methods = 'get' | 'post' | 'put' | 'delete'; + +type GetParamsFromString< + Str extends string, + Count extends unknown[] = [], +> = Str extends `${infer Start}:${string}/${infer Rest}` + ? Str extends `${Start}:${infer Param}/${Rest}` + ? GetParamsFromString + : Count + : Str extends `${string}:${infer End}` + ? [...Count, End] + : Count; + +type EndpointArgsArray = ParamsArray extends [] + ? [] + : [Record]; +type EndpointArgs = EndpointArgsArray>; + +type AvailableEndpoints = keyof Endpoints[Method] extends string + ? keyof Endpoints[Method] + : never; +export function endpointBuilder>( + _: Method, + endpoint: Url, + ...options: EndpointArgs +) { + return endpoint.replace(/:([^/]+)/g, (_, capture_group) => options[0]?.[capture_group] || ''); +} diff --git a/src/management-system-v2/lib/engines/endpoints.json b/src/management-system-v2/lib/engines/endpoints.json new file mode 100644 index 000000000..ebeb5fcfd --- /dev/null +++ b/src/management-system-v2/lib/engines/endpoints.json @@ -0,0 +1,67 @@ +{ + "get": { + "/machine/:properties": { + "params": true + }, + "/machine/": {}, + "/capabilities/": {}, + "/configuration/": {}, + "/configuration/:key": {}, + "/logging": {}, + "/logging/status": {}, + "/logging/standard": {}, + "/logging/process": {}, + "/logging/process/:definitionId": {}, + "/logging/process/:definitionId/instance/:instanceId": {}, + "/monitoring/": {}, + "/tasklist/api/": {}, + "/tasklist/api/userTask": {}, + "/configuration/api/config": {}, + "/logging/api/log": {}, + "/": {}, + "/process/": {}, + "/process/:definitionId": {}, + "/process/:definitionId/versions": {}, + "/process/:definitionId/versions/:version": {}, + "/process/:definitionId/instance": {}, + "/process/:definitionId/instance/:instanceID": {}, + "/process/:definitionId/user-tasks/:fileName": {}, + "/process/:definitionId/user-tasks": {}, + "/status/": {}, + "/resources/process/:definitionId/images/:fileName": {}, + "/resources/process/:definitionId/images/": {} + }, + "post": { + "/capabilities/execute": {}, + "/capabilities/return": {}, + "/evaluation/": {}, + "/tasklist/api/userTask": {}, + "/configuration/api/config": {}, + "/process/": {}, + "/process/:definitionId/versions/:version/instance": {}, + "/process/:definitionId/instance/:instanceId/tokens": {}, + "/process/:definitionId/instance/:instanceId/variables": {}, + "/process/:definitionId/versions/:version/instance/migration": {} + }, + "put": { + "/configuration/": {}, + "/tasklist/api/variable": {}, + "/tasklist/api/milestone": {}, + "/process/:definitionId/instance/:instanceID": {}, + "/process/:definitionId/instance/:instanceID/instanceState": {}, + "/process/:definitionId/instance/:instanceId/tokens/:tokenId": {}, + "/process/:definitionId/instance/:instanceId/tokens/:tokenId/currentFlowNodeState": {}, + "/process/:definitionId/user-tasks/:fileName": {}, + "/resources/process/:definitionId/images/:fileName": {} + }, + "delete": { + "/configuration/": {}, + "/logging": {}, + "/logging/standard": {}, + "/logging/process": {}, + "/logging/process/:definitionId": {}, + "/logging/process/:definitionId/instance/:instanceId": {}, + "/process/:definitionId": {}, + "/process/:definitionId/instance/:instanceId/tokens/:tokenId": {} + } +} \ No newline at end of file diff --git a/src/management-system-v2/lib/engines/endpoints.ts b/src/management-system-v2/lib/engines/http-endpoints.ts similarity index 100% rename from src/management-system-v2/lib/engines/endpoints.ts rename to src/management-system-v2/lib/engines/http-endpoints.ts From ff6c609a1d55a4a198043a2d66dd0b07492aed9c Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Tue, 5 Nov 2024 23:57:43 +0100 Subject: [PATCH 03/16] feat(ms2/engines): mqtt requests --- .../lib/engines/mqtt-endpoints.ts | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/management-system-v2/lib/engines/mqtt-endpoints.ts b/src/management-system-v2/lib/engines/mqtt-endpoints.ts index 5276d9664..7803d7613 100644 --- a/src/management-system-v2/lib/engines/mqtt-endpoints.ts +++ b/src/management-system-v2/lib/engines/mqtt-endpoints.ts @@ -70,3 +70,57 @@ export async function getEngines() { return engines; } + +const requestClient = getClient(); + +export async function mqttRequest( + engineId: string, + url: string, + message: { + method: 'GET' | 'POST' | 'PUT' | 'DELETE'; + body: Record; + query: Record; + page?: number; + }, +) { + const client = await requestClient; + + const requestId = crypto.randomUUID(); + const requestTopic = getEnginePrefix(engineId) + '/api' + url; + await subscribeToTopic(client, requestTopic); + + // handler for the response + let res: (res: any) => void, rej: (Err: any) => void; + const receivedAnswer = new Promise((_res, _rej) => { + res = _res; + rej = _rej; + }); + function handler(topic: string, _message: any) { + const message = JSON.parse(_message.toString()); + if (topic !== requestTopic) return; + if ( + !message || + typeof message !== 'object' || + !('type' in message) || + message.type !== 'response' || + !('id' in message) || + message.id !== requestId + ) + return; + + res(JSON.parse(message.body)); + } + client.on('message', handler); + + // send request + client.publish(requestTopic, JSON.stringify({ ...message, type: 'request', id: requestId })); + + // await for response or timeout + setTimeout(rej!, mqttTimeout); + const response = await receivedAnswer; + + // cleanup + client.removeListener('message', handler); + + return response; +} From 1099bb6c628946ff8c3ec70c57b566ec79538a17 Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Wed, 6 Nov 2024 00:02:40 +0100 Subject: [PATCH 04/16] feat(ms2/admin): engine view --- .../app/admin/engines/engines-table.tsx | 57 +++++++++++++++++++ .../app/admin/engines/page.tsx | 39 +++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/management-system-v2/app/admin/engines/engines-table.tsx create mode 100644 src/management-system-v2/app/admin/engines/page.tsx diff --git a/src/management-system-v2/app/admin/engines/engines-table.tsx b/src/management-system-v2/app/admin/engines/engines-table.tsx new file mode 100644 index 000000000..26f4c8c23 --- /dev/null +++ b/src/management-system-v2/app/admin/engines/engines-table.tsx @@ -0,0 +1,57 @@ +'use client'; + +import { Tag } from 'antd'; +import { useState } from 'react'; +import { type TableEngine } from './page'; +import ElementList from '@/components/item-list-view'; +import Bar from '@/components/bar'; +import useFuzySearch from '@/lib/useFuzySearch'; + +export default function EnginesTable({ engines }: { engines: TableEngine[] }) { + const { filteredData, searchQuery, setSearchQuery } = useFuzySearch({ + data: engines, + keys: ['engineId'], + highlightedKeys: ['engineId'], + transformData: (matches) => matches.map((match) => match.item), + }); + + const [selectedEngines, setSelectedEngines] = useState([]); + + return ( + <> + setSearchQuery(e.target.value), + onPressEnter: (e) => setSearchQuery(e.currentTarget.value), + placeholder: 'Search spaces ...', + }} + /> + + engine.engineId.highlighted, + }, + { + title: 'Status', + dataIndex: 'owner', + sorter: (a, b) => +a.running - +b.running, + render: (_, engine) => ( + + {engine.running ? 'Online' : 'Offline'} + + ), + }, + ]} + /> + + ); +} diff --git a/src/management-system-v2/app/admin/engines/page.tsx b/src/management-system-v2/app/admin/engines/page.tsx new file mode 100644 index 000000000..c9262751e --- /dev/null +++ b/src/management-system-v2/app/admin/engines/page.tsx @@ -0,0 +1,39 @@ +import { getCurrentUser } from '@/components/auth'; +import Content from '@/components/content'; +import { getEngines } from '@/lib/engines/mqtt-endpoints'; +import { Result, Skeleton } from 'antd'; +import { notFound, redirect } from 'next/navigation'; +import { env } from 'process'; +import { Suspense } from 'react'; +import { getSystemAdminByUserId } from '@/lib/data/DTOs'; +import EnginesTable from './engines-table'; + +export type TableEngine = Awaited>[number] & { id: string }; + +async function Engines() { + const user = await getCurrentUser(); + if (!user.session) redirect('/'); + const adminData = getSystemAdminByUserId(user.userId); + if (!adminData) redirect('/'); + + try { + const engines = (await getEngines()).map((e) => ({ ...e, id: e.engineId })); + + return ; + } catch (e) { + console.error(e); + return ; + } +} + +export default function EnginesPage() { + if (!env.NEXT_PUBLIC_ENABLE_EXECUTION) return notFound(); + + return ( + + }> + + + + ); +} From eeef5cc21da4cd2b7755166f76bd9df496e279d8 Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Wed, 6 Nov 2024 00:17:40 +0100 Subject: [PATCH 05/16] refactor(ms2/mqtt) --- src/management-system-v2/lib/engines/mqtt-endpoints.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/management-system-v2/lib/engines/mqtt-endpoints.ts b/src/management-system-v2/lib/engines/mqtt-endpoints.ts index 7803d7613..69fb73d39 100644 --- a/src/management-system-v2/lib/engines/mqtt-endpoints.ts +++ b/src/management-system-v2/lib/engines/mqtt-endpoints.ts @@ -11,8 +11,6 @@ const mqttCredentials = { const baseTopicPrefix = env.MQTT_BASETOPIC ? env.MQTT_BASETOPIC + '/' : ''; export function getClient(options?: mqtt.IClientOptions): Promise { - if (!env.MQTT_SERVER_ADDRESS) throw new Error('MQTT_SERVER_ADDRESS is not set'); - return new Promise((res, rej) => { const client = mqtt.connect(env.MQTT_SERVER_ADDRESS, { ...mqttCredentials, From 0c815e3f02a8739436f0a378452d87593b6104eb Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Wed, 6 Nov 2024 00:17:54 +0100 Subject: [PATCH 06/16] fix(ms2/engine-endpoints): import --- src/management-system-v2/lib/engines/deployment.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/management-system-v2/lib/engines/deployment.ts b/src/management-system-v2/lib/engines/deployment.ts index 786680921..7c005036f 100644 --- a/src/management-system-v2/lib/engines/deployment.ts +++ b/src/management-system-v2/lib/engines/deployment.ts @@ -9,7 +9,7 @@ import { // @ts-ignore // import decider from '@proceed/decider'; import { Machine, getMachines } from './machines'; -import * as endpoints from './endpoints'; +import * as endpoints from './http-endpoints'; import { prepareExport } from '../process-export/export-preparation'; import { Prettify } from '../typescript-utils'; From 28f6d1025642350ee0175453bec2070cad527f1c Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Wed, 6 Nov 2024 00:18:52 +0100 Subject: [PATCH 07/16] fix(ms2/admin-engines): make page dynamic --- src/management-system-v2/app/admin/engines/page.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/management-system-v2/app/admin/engines/page.tsx b/src/management-system-v2/app/admin/engines/page.tsx index c9262751e..7ea7fb131 100644 --- a/src/management-system-v2/app/admin/engines/page.tsx +++ b/src/management-system-v2/app/admin/engines/page.tsx @@ -37,3 +37,5 @@ export default function EnginesPage() { ); } + +export const dynamic = 'force-dynamic'; From 201ff18ddfdb6a9e61dc9d584203e9414e810c4e Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Wed, 6 Nov 2024 17:02:28 +0100 Subject: [PATCH 08/16] feat(ms2/admin): added link to engines in sidebar --- src/management-system-v2/app/admin/layout.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/management-system-v2/app/admin/layout.tsx b/src/management-system-v2/app/admin/layout.tsx index d0f034ca4..b8789c0fe 100644 --- a/src/management-system-v2/app/admin/layout.tsx +++ b/src/management-system-v2/app/admin/layout.tsx @@ -1,6 +1,7 @@ import Link from 'next/link'; import Layout from '@/app/(dashboard)/[environmentId]/layout-client'; import { AreaChartOutlined, AppstoreOutlined, FileOutlined } from '@ant-design/icons'; +import { MdOutlineComputer } from 'react-icons/md'; import { FaUsers } from 'react-icons/fa'; import { RiAdminFill } from 'react-icons/ri'; @@ -45,6 +46,12 @@ export default function AdminLayout({ children }: { children: React.ReactNode }) label: Manage admins, icon: , }, + + { + key: 'engines', + label: Engines, + icon: , + }, ], }, ]} From e2f0b0187dac92d683a24e8f2635bc127dfb701c Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Wed, 6 Nov 2024 17:03:32 +0100 Subject: [PATCH 09/16] chore: format --- src/management-system-v2/lib/engines/endpoints.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/management-system-v2/lib/engines/endpoints.json b/src/management-system-v2/lib/engines/endpoints.json index ebeb5fcfd..bfaa4aa11 100644 --- a/src/management-system-v2/lib/engines/endpoints.json +++ b/src/management-system-v2/lib/engines/endpoints.json @@ -64,4 +64,4 @@ "/process/:definitionId": {}, "/process/:definitionId/instance/:instanceId/tokens/:tokenId": {} } -} \ No newline at end of file +} From 9ee7945bf7d18f6924382878025e2d2e2ccc7700 Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Mon, 11 Nov 2024 21:34:04 +0100 Subject: [PATCH 10/16] fix(ms2/env-vars): mqtt env vars shouldn't be required yet --- src/management-system-v2/lib/env-vars.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/management-system-v2/lib/env-vars.ts b/src/management-system-v2/lib/env-vars.ts index 1c68ebd40..70eb4f3ba 100644 --- a/src/management-system-v2/lib/env-vars.ts +++ b/src/management-system-v2/lib/env-vars.ts @@ -57,10 +57,6 @@ const environmentVariables = { TWITTER_CLIENT_SECRET: z.string(), SHARING_ENCRYPTION_SECRET: z.string(), - - MQTT_SERVER_ADDRESS: z.string().url(), - MQTT_USERNAME: z.string(), - MQTT_PASSWORD: z.string(), }, development: { SHARING_ENCRYPTION_SECRET: z.string().default('T8VB/r1dw0kJAXjanUvGXpDb+VRr4dV5y59BT9TBqiQ='), From 60fa7e3d23a527d54c050624608f7c8e8228748b Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Mon, 11 Nov 2024 21:40:55 +0100 Subject: [PATCH 11/16] fix: possible undefined --- src/management-system-v2/lib/engines/mqtt-endpoints.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/management-system-v2/lib/engines/mqtt-endpoints.ts b/src/management-system-v2/lib/engines/mqtt-endpoints.ts index 69fb73d39..cb726a913 100644 --- a/src/management-system-v2/lib/engines/mqtt-endpoints.ts +++ b/src/management-system-v2/lib/engines/mqtt-endpoints.ts @@ -11,8 +11,11 @@ const mqttCredentials = { const baseTopicPrefix = env.MQTT_BASETOPIC ? env.MQTT_BASETOPIC + '/' : ''; export function getClient(options?: mqtt.IClientOptions): Promise { + const address = env.MQTT_SERVER_ADDRESS; + if (!address) throw new Error('MQTT_SERVER_ADDRESS is not set'); + return new Promise((res, rej) => { - const client = mqtt.connect(env.MQTT_SERVER_ADDRESS, { + const client = mqtt.connect(address, { ...mqttCredentials, ...options, }); @@ -40,7 +43,7 @@ export async function getEngines() { connectTimeout: mqttTimeout, }); - const engines: { engineId: string; running: boolean; version: string }[] = []; + const engines: { id: string; running: boolean; version: string }[] = []; await subscribeToTopic(client, `${getEnginePrefix('+')}/status`); From f5ce66c146a3f276d4dbeb8aa844c35315efe4dd Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Tue, 12 Nov 2024 10:28:35 +0100 Subject: [PATCH 12/16] fix: engine name key --- .../app/admin/engines/engines-table.tsx | 6 +++--- src/management-system-v2/app/admin/engines/page.tsx | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/management-system-v2/app/admin/engines/engines-table.tsx b/src/management-system-v2/app/admin/engines/engines-table.tsx index 26f4c8c23..54f159246 100644 --- a/src/management-system-v2/app/admin/engines/engines-table.tsx +++ b/src/management-system-v2/app/admin/engines/engines-table.tsx @@ -10,8 +10,8 @@ import useFuzySearch from '@/lib/useFuzySearch'; export default function EnginesTable({ engines }: { engines: TableEngine[] }) { const { filteredData, searchQuery, setSearchQuery } = useFuzySearch({ data: engines, - keys: ['engineId'], - highlightedKeys: ['engineId'], + keys: ['name'], + highlightedKeys: ['name'], transformData: (matches) => matches.map((match) => match.item), }); @@ -38,7 +38,7 @@ export default function EnginesTable({ engines }: { engines: TableEngine[] }) { { title: 'Engine ID', dataIndex: 'name', - render: (_, engine) => engine.engineId.highlighted, + render: (_, engine) => engine.name.highlighted, }, { title: 'Status', diff --git a/src/management-system-v2/app/admin/engines/page.tsx b/src/management-system-v2/app/admin/engines/page.tsx index 7ea7fb131..aec62c111 100644 --- a/src/management-system-v2/app/admin/engines/page.tsx +++ b/src/management-system-v2/app/admin/engines/page.tsx @@ -8,7 +8,7 @@ import { Suspense } from 'react'; import { getSystemAdminByUserId } from '@/lib/data/DTOs'; import EnginesTable from './engines-table'; -export type TableEngine = Awaited>[number] & { id: string }; +export type TableEngine = Awaited>[number] & { name: string }; async function Engines() { const user = await getCurrentUser(); @@ -17,7 +17,7 @@ async function Engines() { if (!adminData) redirect('/'); try { - const engines = (await getEngines()).map((e) => ({ ...e, id: e.engineId })); + const engines = (await getEngines()).map((e) => ({ ...e, name: e.id })); return ; } catch (e) { From ba762b96d71f1bb867bbc995424655350045968f Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Tue, 12 Nov 2024 10:30:12 +0100 Subject: [PATCH 13/16] fix(ms2/build): unset env-var --- src/management-system-v2/lib/engines/mqtt-endpoints.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/management-system-v2/lib/engines/mqtt-endpoints.ts b/src/management-system-v2/lib/engines/mqtt-endpoints.ts index cb726a913..3ee944f8d 100644 --- a/src/management-system-v2/lib/engines/mqtt-endpoints.ts +++ b/src/management-system-v2/lib/engines/mqtt-endpoints.ts @@ -11,8 +11,7 @@ const mqttCredentials = { const baseTopicPrefix = env.MQTT_BASETOPIC ? env.MQTT_BASETOPIC + '/' : ''; export function getClient(options?: mqtt.IClientOptions): Promise { - const address = env.MQTT_SERVER_ADDRESS; - if (!address) throw new Error('MQTT_SERVER_ADDRESS is not set'); + const address = env.MQTT_SERVER_ADDRESS || ''; return new Promise((res, rej) => { const client = mqtt.connect(address, { From 380fe3c7ffbbf067166ed8bd665948c2b800a27e Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Sat, 23 Nov 2024 02:44:24 +0100 Subject: [PATCH 14/16] fix(ms2/mqtt): correct engine id key --- src/management-system-v2/lib/engines/mqtt-endpoints.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/management-system-v2/lib/engines/mqtt-endpoints.ts b/src/management-system-v2/lib/engines/mqtt-endpoints.ts index 3ee944f8d..06d45f6b0 100644 --- a/src/management-system-v2/lib/engines/mqtt-endpoints.ts +++ b/src/management-system-v2/lib/engines/mqtt-endpoints.ts @@ -58,9 +58,9 @@ export async function getEngines() { client.on('message', (topic, message) => { const match = topic.match(new RegExp(`^${getEnginePrefix('')}([^\/]+)\/status`)); if (match) { - const engineId = match[1]; + const id = match[1]; const status = JSON.parse(message.toString()); - engines.push({ engineId, ...status }); + engines.push({ id, ...status }); res(); } }); From f9c841f90651f6f78109838ea47750b4e8bd3e50 Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Mon, 25 Nov 2024 00:24:56 +0100 Subject: [PATCH 15/16] admin-dashboard/engines: better error message --- src/management-system-v2/app/admin/engines/page.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/management-system-v2/app/admin/engines/page.tsx b/src/management-system-v2/app/admin/engines/page.tsx index aec62c111..5860a86f1 100644 --- a/src/management-system-v2/app/admin/engines/page.tsx +++ b/src/management-system-v2/app/admin/engines/page.tsx @@ -3,10 +3,10 @@ import Content from '@/components/content'; import { getEngines } from '@/lib/engines/mqtt-endpoints'; import { Result, Skeleton } from 'antd'; import { notFound, redirect } from 'next/navigation'; -import { env } from 'process'; import { Suspense } from 'react'; import { getSystemAdminByUserId } from '@/lib/data/DTOs'; import EnginesTable from './engines-table'; +import { env } from '@/lib/env-vars'; export type TableEngine = Awaited>[number] & { name: string }; @@ -29,6 +29,9 @@ async function Engines() { export default function EnginesPage() { if (!env.NEXT_PUBLIC_ENABLE_EXECUTION) return notFound(); + if (!env.MQTT_SERVER_ADDRESS) + return ; + return ( }> From 83150ef37038f8bddb9c1097bc31d8f2a1b12e56 Mon Sep 17 00:00:00 2001 From: Felipe Trost Date: Tue, 26 Nov 2024 16:44:05 +0100 Subject: [PATCH 16/16] fix: import --- .../executions/[processId]/element-status.tsx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/executions/[processId]/element-status.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/executions/[processId]/element-status.tsx index 65ad7febb..045d50a10 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/executions/[processId]/element-status.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/executions/[processId]/element-status.tsx @@ -4,7 +4,7 @@ import { ClockCircleFilled } from '@ant-design/icons'; import React from 'react'; import { statusToType } from './instance-helpers'; import { convertISODurationToMiliseconds, getMetaDataFromElement } from '@proceed/bpmn-helper'; -import { generateRequestUrl } from '@/lib/engines/endpoints'; +import { endpointBuilder } from '@/lib/engines/endpoint'; import { DisplayTable, RelevantInstanceInfo } from './instance-info-panel'; function transformMilisecondsToTimeFormat(milliseconds: number | undefined) { @@ -47,10 +47,11 @@ export function ElementStatus({ info }: { info: RelevantInstanceInfo }) { }} > , ]);