From 02290c3c75ed47a042a7996e3a8972e979dc4893 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 13:21:37 -0300 Subject: [PATCH 01/27] Rolling out marquez web for staging --- web/src/auth/AuthContext.tsx | 2 +- web/src/components/ga4.tsx | 2 +- web/src/index.prod.html | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/web/src/auth/AuthContext.tsx b/web/src/auth/AuthContext.tsx index 1b31ea6851..b191f90fa8 100644 --- a/web/src/auth/AuthContext.tsx +++ b/web/src/auth/AuthContext.tsx @@ -4,7 +4,7 @@ import { trackEvent } from '../components/ga4' export const oktaAuth = new OktaAuth({ issuer: 'https://nubank.okta.com/oauth2/default', - clientId: '0oa20d6n6jb6nG5Mn0h8', + clientId: '0oa20ehmjv97g8jZP0h8', redirectUri: window.location.origin + '/login/callback', pkce: true, scopes: ['openid', 'profile', 'email'], diff --git a/web/src/components/ga4.tsx b/web/src/components/ga4.tsx index 11c6663e6a..9410c4d17c 100644 --- a/web/src/components/ga4.tsx +++ b/web/src/components/ga4.tsx @@ -1,7 +1,7 @@ import ReactGA from 'react-ga4'; const initializeGA = () => { - ReactGA.initialize('G-QF2RHX3HRJ'); + ReactGA.initialize('G-J6G5BV3EV5'); }; const trackPageView = () => { diff --git a/web/src/index.prod.html b/web/src/index.prod.html index 9d6224c956..1c40d388d2 100644 --- a/web/src/index.prod.html +++ b/web/src/index.prod.html @@ -11,14 +11,14 @@ Nu Data Lineage - --> @@ -32,8 +32,8 @@ - +
From d82eab77e0cdaf36164d04150ab1650f89ae9225 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 14:10:12 -0300 Subject: [PATCH 02/27] Adding kafka to Marquez Web --- web/package-lock.json | 10 ++++++++++ web/package.json | 1 + web/services/appMetrics.js | 17 ++++++++++++++--- web/services/kafkaProducer.js | 34 ++++++++++++++++++++++++++++++++++ web/setupProxy.js | 13 +++++++++++++ 5 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 web/services/kafkaProducer.js diff --git a/web/package-lock.json b/web/package-lock.json index 08eb0d647d..6c9b712300 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -45,6 +45,7 @@ "http-proxy-middleware": "^2.0.6", "i18next": "^22.5.0", "i18next-browser-languagedetector": "^7.0.1", + "kafkajs": "^2.2.4", "lodash": "^4.17.21", "loglevel": "^1.9.2", "micromatch": "^4.0.8", @@ -16339,6 +16340,15 @@ "node": ">=4.0" } }, + "node_modules/kafkajs": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", + "integrity": "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", diff --git a/web/package.json b/web/package.json index 91f6052b94..953f3d2d35 100644 --- a/web/package.json +++ b/web/package.json @@ -53,6 +53,7 @@ "http-proxy-middleware": "^2.0.6", "i18next": "^22.5.0", "i18next-browser-languagedetector": "^7.0.1", + "kafkajs": "^2.2.4", "lodash": "^4.17.21", "loglevel": "^1.9.2", "micromatch": "^4.0.8", diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 7079d51da9..7d0ea3c92e 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -1,8 +1,11 @@ -const client = require('prom-client'); -const crypto = require('crypto'); +const client = require('prom-client') +const crypto = require('crypto') + +// Import Kafka producer functions from your kafkaProducer.js file +const { sendLogToKafka } = require('./kafkaProducer') // Import Redis write client from your redisClient.js file -const { redisWriteClient, redisReadClient } = require('./redisClient'); +const { redisWriteClient, redisReadClient } = require('./redisClient') // Centralize the excluded emails list const excludedEmails = new Set([ @@ -100,6 +103,14 @@ class appMetrics { // Set the key with expiration (7 days) await redisWriteClient.set(key, currentTime, { EX: 7 * 24 * 60 * 60 }); this.uniqueUserLoginCounter.inc(); + + const logData = { + accessLog: { + email: encodedEmail, + dateTime: new Date().toISOString(), + }, + }; + sendLogToKafka(logData); } } catch (err) { console.error('Error in incrementUniqueLogins:', err); diff --git a/web/services/kafkaProducer.js b/web/services/kafkaProducer.js new file mode 100644 index 0000000000..3379413b77 --- /dev/null +++ b/web/services/kafkaProducer.js @@ -0,0 +1,34 @@ +// filepath: /Users/jonathan.moraes.gft/Projects/new-numarquez/NuMarquez/web/services/kafkaProducer.js +const { Kafka } = require('kafkajs'); + +const KAFKA_LOADBALANCER_DNS = process.env.KAFKA_LOADBALANCER_DNS +const KAFKA_PORT = process.env.KAFKA_PORT + +// Configure Kafka client with your broker list (can be set via environment variables) +const kafka = new Kafka({ + clientId: 'log-producer', + brokers: [`${KAFKA_LOADBALANCER_DNS}:${KAFKA_PORT}`] +}); + +const producer = kafka.producer(); + +const connectProducer = async () => { + await producer.connect(); + console.log('Kafka producer connected.'); +}; + +const sendLogToKafka = async (log) => { + try { + await producer.send({ + topic: process.env.KAFKA_LOG_TOPIC || 'logs-topic', + messages: [ + { value: JSON.stringify(log) } + ] + }); + console.log('Log sent to Kafka.'); + } catch (error) { + console.error('Error sending log to Kafka:', error); + } +}; + +module.exports = { connectProducer, sendLogToKafka, producer }; \ No newline at end of file diff --git a/web/setupProxy.js b/web/setupProxy.js index 9961e80f9b..ce4c592f67 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -3,6 +3,7 @@ const express = require('express') const { createProxyMiddleware } = require('http-proxy-middleware') const path = require('path') const appMetrics = require('./services/appMetrics'); +const { sendLogToKafka } = require('./services/kafkaProducer'); const app = express(); const router = express.Router(); @@ -21,6 +22,17 @@ app.get('/metrics', async (req, res) => { } }); +const { connectProducer } = require('./services/kafkaProducer'); + +(async () => { + try { + await connectProducer(); + } catch (error) { + console.error('Error connecting Kafka producer:', error); + process.exit(1); + } +})(); + const environmentVariable = (variableName) => { const value = process.env[variableName] if (!value) { @@ -108,6 +120,7 @@ app.post('/api/loguserinfo', (req, res) => { }, }; console.log(JSON.stringify(logData)); + sendLogToKafka(logData); res.sendStatus(200); }); From 9c5f7e5850f9d65eebbcd827c92bf9451097c46f Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 15:27:10 -0300 Subject: [PATCH 03/27] Adjusting kafka payload --- web/services/appMetrics.js | 21 +++++++++++++++------ web/services/logFormatter.js | 24 ++++++++++++++++++++++++ web/setupProxy.js | 11 +++++++---- 3 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 web/services/logFormatter.js diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 7d0ea3c92e..0a240c5adb 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -79,12 +79,25 @@ class appMetrics { * @param {string} email - The user's email */ incrementTotalLogins(email) { + if (typeof email !== 'string') { + console.error('Invalid email provided to incrementTotalLogins:', email); + return; // Early exit if email is not a string. + } const encodedEmail = this.encodeEmail(email); if (excludedEmails.has(encodedEmail)) { return; // Do not increment if user is in the excluded list } this.totalUserLoginCounter.inc(); } + + encodeEmail(email) { + // Optionally check here: + if (!email) { + console.error('No email provided to encodeEmail.'); + return ''; + } + return Buffer.from(email).toString('base64'); + } /** * Increments the unique user logins counter if the user hasn't logged in within the retention period. @@ -104,12 +117,8 @@ class appMetrics { await redisWriteClient.set(key, currentTime, { EX: 7 * 24 * 60 * 60 }); this.uniqueUserLoginCounter.inc(); - const logData = { - accessLog: { - email: encodedEmail, - dateTime: new Date().toISOString(), - }, - }; + const userInfo = { email}; + const logData = buildLogData(userInfo); sendLogToKafka(logData); } } catch (err) { diff --git a/web/services/logFormatter.js b/web/services/logFormatter.js new file mode 100644 index 0000000000..28abca03fa --- /dev/null +++ b/web/services/logFormatter.js @@ -0,0 +1,24 @@ +/** + * Build enriched log data. + * @param {object} userInfo - The Okta userinfo payload + * @returns {object} - The enriched log data + */ + +const { getFormattedDateTime } = require('./setupProxy') + +function buildLogData(userInfo) { + const timestamp = getFormattedDateTime(); + const podName = process.env.POD_NAME || "unknown-pod"; + + return { + timestamp, + podName, + username: userInfo.name, + locale: userInfo.locale, + email: userInfo.email, + zoneinfo: userInfo.zoneinfo, + email_verified: userInfo.email_verified + }; + } + + module.exports = { buildLogData }; \ No newline at end of file diff --git a/web/setupProxy.js b/web/setupProxy.js index ce4c592f67..a2a2bc9504 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -2,8 +2,8 @@ const express = require('express') const { createProxyMiddleware } = require('http-proxy-middleware') const path = require('path') -const appMetrics = require('./services/appMetrics'); -const { sendLogToKafka } = require('./services/kafkaProducer'); +const appMetrics = require('./services/appMetrics') +const { sendLogToKafka } = require('./services/kafkaProducer') const app = express(); const router = express.Router(); @@ -97,6 +97,8 @@ function getFormattedDateTime() { return `${year}-${month}-${day} ${hour}:${minute}:${second}.${ms}`; } +const { buildLogData } = require('./services/logFormatter'); + // Endpoint to log user info and increment counters app.post('/api/loguserinfo', (req, res) => { const { email = '' } = req.body; @@ -105,6 +107,8 @@ app.post('/api/loguserinfo', (req, res) => { return res.status(400).json({ error: 'Invalid email format' }); } + // Build the enriched log data using the helper + const kafkaData = buildLogData(userInfo); const encodedEmail = Buffer.from(email).toString('base64'); // Increment total logins counter @@ -120,10 +124,9 @@ app.post('/api/loguserinfo', (req, res) => { }, }; console.log(JSON.stringify(logData)); - sendLogToKafka(logData); + sendLogToKafka(kafkaData); res.sendStatus(200); }); -// Export the app for testing or further integration module.exports = app; From b500a9dd6f48ca7f5865c8b82395ef5bee15d8a1 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 15:47:08 -0300 Subject: [PATCH 04/27] adding dateformater to a class --- web/services/dateTimeHelper.js | 14 ++++++++++++++ web/services/logFormatter.js | 2 +- web/setupProxy.js | 16 +--------------- 3 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 web/services/dateTimeHelper.js diff --git a/web/services/dateTimeHelper.js b/web/services/dateTimeHelper.js new file mode 100644 index 0000000000..e39c442549 --- /dev/null +++ b/web/services/dateTimeHelper.js @@ -0,0 +1,14 @@ +function getFormattedDateTime() { + const d = new Date(); + const pad = (n, size = 2) => n.toString().padStart(size, '0'); + const year = d.getFullYear(); + const month = pad(d.getMonth() + 1); + const day = pad(d.getDate()); + const hour = pad(d.getHours()); + const minute = pad(d.getMinutes()); + const second = pad(d.getSeconds()); + const ms = pad(d.getMilliseconds(), 3); + return `${year}-${month}-${day} ${hour}:${minute}:${second}.${ms}`; + } + + module.exports = { getFormattedDateTime }; \ No newline at end of file diff --git a/web/services/logFormatter.js b/web/services/logFormatter.js index 28abca03fa..d6bcb174b8 100644 --- a/web/services/logFormatter.js +++ b/web/services/logFormatter.js @@ -4,7 +4,7 @@ * @returns {object} - The enriched log data */ -const { getFormattedDateTime } = require('./setupProxy') +const { getFormattedDateTime } = require('./dateTimeHelper') function buildLogData(userInfo) { const timestamp = getFormattedDateTime(); diff --git a/web/setupProxy.js b/web/setupProxy.js index a2a2bc9504..7873b2c014 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -4,6 +4,7 @@ const { createProxyMiddleware } = require('http-proxy-middleware') const path = require('path') const appMetrics = require('./services/appMetrics') const { sendLogToKafka } = require('./services/kafkaProducer') +const { getFormattedDateTime } = require('./services/dateTimeHelper') const app = express(); const router = express.Router(); @@ -82,21 +83,6 @@ app.listen(port, () => { app.use(express.json()); -// Helper function to format datetime as "YYYY-MM-DD HH:mm:SS.sss" -function getFormattedDateTime() { - const d = new Date(); - const pad = (n, size = 2) => n.toString().padStart(size, '0'); - const year = d.getFullYear(); - const month = pad(d.getMonth() + 1); - const day = pad(d.getDate()); - const hour = pad(d.getHours()); - const minute = pad(d.getMinutes()); - const second = pad(d.getSeconds()); - // JavaScript Date only provides milliseconds (0-999), so we pad to 3 digits - const ms = pad(d.getMilliseconds(), 3); - return `${year}-${month}-${day} ${hour}:${minute}:${second}.${ms}`; -} - const { buildLogData } = require('./services/logFormatter'); // Endpoint to log user info and increment counters From 7fb6c42dd0041048477a75d327030cd9d9442f72 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 16:05:29 -0300 Subject: [PATCH 05/27] adding userInfo to a class --- web/setupProxy.js | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index 7873b2c014..6e680cdde6 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -84,33 +84,49 @@ app.listen(port, () => { app.use(express.json()); const { buildLogData } = require('./services/logFormatter'); +const { userInfo, userInfo, userInfo } = require('os') // Endpoint to log user info and increment counters app.post('/api/loguserinfo', (req, res) => { const { email = '' } = req.body; + // Guard against invalid email values if (typeof email !== 'string') { return res.status(400).json({ error: 'Invalid email format' }); } - // Build the enriched log data using the helper + // Define a minimal userInfo object with email + const userInfo = { + email, + name: userInfo.name, + locale: userInfo.locale, + zoneinfo: userInfo.zoneinfo, + email_verified: true + }; + + // Build enriched log data using the helper const kafkaData = buildLogData(userInfo); - const encodedEmail = Buffer.from(email).toString('base64'); - // Increment total logins counter - metrics.incrementTotalLogins(); + // Encode the email for your own logs + const encodedEmail = Buffer.from(email).toString('base64'); - // Check if the user is logging in for the first time in the last 8 hours + // Update metrics + metrics.incrementTotalLogins(); metrics.incrementUniqueLogins(email); + // Console log for local debugging const logData = { accessLog: { email: encodedEmail, - dateTime: getFormattedDateTime(), - }, + dateTime: getFormattedDateTime() + } }; console.log(JSON.stringify(logData)); + + // Send meta info to Kafka sendLogToKafka(kafkaData); + + // Response res.sendStatus(200); }); From d7bc3f1f26f87cd9b3c2e7d82a20da559da4d701 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 16:23:03 -0300 Subject: [PATCH 06/27] adjusting call to userInfo --- web/setupProxy.js | 1 - 1 file changed, 1 deletion(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index 6e680cdde6..74f8288898 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -84,7 +84,6 @@ app.listen(port, () => { app.use(express.json()); const { buildLogData } = require('./services/logFormatter'); -const { userInfo, userInfo, userInfo } = require('os') // Endpoint to log user info and increment counters app.post('/api/loguserinfo', (req, res) => { From c8c6a34204e7798f9416738b58a666dd5ad70fc6 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 16:59:31 -0300 Subject: [PATCH 07/27] Implementing logic to deal with vars --- web/setupProxy.js | 23 ++++++++++++++--------- web/src/auth/AuthContext.tsx | 5 ++++- web/src/components/ga4.tsx | 6 ++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index 74f8288898..e9a52be3e8 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -87,21 +87,26 @@ const { buildLogData } = require('./services/logFormatter'); // Endpoint to log user info and increment counters app.post('/api/loguserinfo', (req, res) => { - const { email = '' } = req.body; + const { + email = '', + name, + locale, + zoneinfo + } = req.body; // Guard against invalid email values if (typeof email !== 'string') { return res.status(400).json({ error: 'Invalid email format' }); } - // Define a minimal userInfo object with email - const userInfo = { - email, - name: userInfo.name, - locale: userInfo.locale, - zoneinfo: userInfo.zoneinfo, - email_verified: true - }; + // Create userInfo from request data (without circular references) + const userInfo = { + email, + name, + locale, + zoneinfo, + email_verified: true + }; // Build enriched log data using the helper const kafkaData = buildLogData(userInfo); diff --git a/web/src/auth/AuthContext.tsx b/web/src/auth/AuthContext.tsx index b191f90fa8..36a2230634 100644 --- a/web/src/auth/AuthContext.tsx +++ b/web/src/auth/AuthContext.tsx @@ -2,9 +2,12 @@ import { AuthState, OktaAuth } from '@okta/okta-auth-js' import React, { createContext, useContext, useEffect, useState } from 'react' import { trackEvent } from '../components/ga4' +const isStaging = window.location.origin.includes('staging') +const oktaClientId = isStaging ? '0oa20ehmjv97g8jZP0h8' : '0oa20d6n6jb6nG5Mn0h8' + export const oktaAuth = new OktaAuth({ issuer: 'https://nubank.okta.com/oauth2/default', - clientId: '0oa20ehmjv97g8jZP0h8', + clientId: oktaClientId, redirectUri: window.location.origin + '/login/callback', pkce: true, scopes: ['openid', 'profile', 'email'], diff --git a/web/src/components/ga4.tsx b/web/src/components/ga4.tsx index 9410c4d17c..6edcd51bd2 100644 --- a/web/src/components/ga4.tsx +++ b/web/src/components/ga4.tsx @@ -1,8 +1,10 @@ import ReactGA from 'react-ga4'; const initializeGA = () => { - ReactGA.initialize('G-J6G5BV3EV5'); - }; + const isStaging = window.location.origin.includes('staging'); + const gaId = isStaging ? 'G-J6G5BV3EV5' : 'G-QF2RHX3HRJ'; + ReactGA.initialize(gaId); +}; const trackPageView = () => { ReactGA.send({ hitType: 'pageview', page: window.location.pathname }); From dde16ae953b5bbf6c556136e5828d3029840271b Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Mon, 17 Feb 2025 17:17:27 -0300 Subject: [PATCH 08/27] Adjusting userInfo build function --- web/services/appMetrics.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 0a240c5adb..a714e29a6e 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -1,5 +1,6 @@ const client = require('prom-client') const crypto = require('crypto') +const { buildLogData } = require('./logFormatter') // Import Kafka producer functions from your kafkaProducer.js file const { sendLogToKafka } = require('./kafkaProducer') @@ -126,15 +127,6 @@ class appMetrics { } } - /** - * Encodes the email using Base64 encoding - * @param {string} email - * @returns {string} - */ - encodeEmail(email) { - return Buffer.from(email).toString('base64'); - } - /** * Hashes the email using SHA-256 for enhanced security * @param {string} email From 1a26c3c1c619df57dda501750923222b5a8b6183 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Tue, 18 Feb 2025 11:59:58 -0300 Subject: [PATCH 09/27] fixing up incrementTotalLogins function --- web/setupProxy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index e9a52be3e8..d024b712df 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -115,7 +115,7 @@ app.post('/api/loguserinfo', (req, res) => { const encodedEmail = Buffer.from(email).toString('base64'); // Update metrics - metrics.incrementTotalLogins(); + metrics.incrementTotalLogins(email); metrics.incrementUniqueLogins(email); // Console log for local debugging From 7e2bc63404c4dee2f529c5991f09bbfaddfa9801 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Tue, 18 Feb 2025 13:10:42 -0300 Subject: [PATCH 10/27] encoding vars with base64 --- web/src/auth/AuthContext.tsx | 8 +++++++- web/src/components/ga4.tsx | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/web/src/auth/AuthContext.tsx b/web/src/auth/AuthContext.tsx index 36a2230634..943ea0f47e 100644 --- a/web/src/auth/AuthContext.tsx +++ b/web/src/auth/AuthContext.tsx @@ -2,8 +2,14 @@ import { AuthState, OktaAuth } from '@okta/okta-auth-js' import React, { createContext, useContext, useEffect, useState } from 'react' import { trackEvent } from '../components/ga4' +function decodeBase64(str: string) { + return window.atob(str); +} + const isStaging = window.location.origin.includes('staging') -const oktaClientId = isStaging ? '0oa20ehmjv97g8jZP0h8' : '0oa20d6n6jb6nG5Mn0h8' +const oktaClientId = isStaging + ? decodeBase64('MG9hMjBlaG1qdjk3ZzhqWlAwaDg=') + : decodeBase64('MG9hMjBkNm42amI2bkc1TW4waDg=') export const oktaAuth = new OktaAuth({ issuer: 'https://nubank.okta.com/oauth2/default', diff --git a/web/src/components/ga4.tsx b/web/src/components/ga4.tsx index 6edcd51bd2..fc56c66394 100644 --- a/web/src/components/ga4.tsx +++ b/web/src/components/ga4.tsx @@ -1,8 +1,14 @@ import ReactGA from 'react-ga4'; +function decodeBase64(str: string) { + return window.atob(str); +} + const initializeGA = () => { const isStaging = window.location.origin.includes('staging'); - const gaId = isStaging ? 'G-J6G5BV3EV5' : 'G-QF2RHX3HRJ'; + const gaId = isStaging + ? decodeBase64('Ry1KNkc1QlYzRVY1') + : decodeBase64('Ry1RRjJSSFgzSFJK') ReactGA.initialize(gaId); }; From 6151101afae92c90f9d549b5c2a60f092aa5cf7c Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Wed, 19 Feb 2025 15:26:23 -0300 Subject: [PATCH 11/27] Fixing name of topic --- web/services/kafkaProducer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/services/kafkaProducer.js b/web/services/kafkaProducer.js index 3379413b77..2bd012000d 100644 --- a/web/services/kafkaProducer.js +++ b/web/services/kafkaProducer.js @@ -20,7 +20,7 @@ const connectProducer = async () => { const sendLogToKafka = async (log) => { try { await producer.send({ - topic: process.env.KAFKA_LOG_TOPIC || 'logs-topic', + topic: 'DATA-LINEAGE.USER-ACCESS-LOGS', messages: [ { value: JSON.stringify(log) } ] From b9b5d7644d154e40713ad663d86c76c622ed4619 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 20 Feb 2025 10:31:27 -0300 Subject: [PATCH 12/27] Adding header to classes and changing kafka topic --- web/services/appMetrics.js | 42 +++++++++------ web/services/{ => helpers}/dateTimeHelper.js | 12 +++++ web/services/helpers/excludedEmails.js | 25 +++++++++ web/services/{ => helpers}/logFormatter.js | 12 +++-- web/services/kafkaProducer.js | 14 ++++- web/services/redisClient.js | 11 ++++ web/setupProxy.js | 57 +++++++++++--------- web/src/auth/AuthContext.tsx | 12 +++++ web/src/components/ErrorBoundary.tsx | 11 ++++ web/src/components/GAInitializer.tsx | 12 +++++ web/src/components/Login.tsx | 11 ++++ web/src/components/LoginCallback.tsx | 12 +++++ web/src/components/PrivateRoute.tsx | 12 +++++ web/src/components/ga4.tsx | 11 ++++ 14 files changed, 210 insertions(+), 44 deletions(-) rename web/services/{ => helpers}/dateTimeHelper.js (57%) create mode 100644 web/services/helpers/excludedEmails.js rename web/services/{ => helpers}/logFormatter.js (54%) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index a714e29a6e..5ef0bd346b 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -1,6 +1,25 @@ +/** + * appMetrics Module + * + * This module is responsible for collecting and exposing various Prometheus + * metrics related to the application's performance and user activity. It + * tracks total and unique user logins, application uptime, and user activity + * over a specified duration. Additionally, it integrates with Redis for + * storing login timestamps and Kafka for logging unique user events, all while + * ensuring that emails on the excluded list are not processed. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To monitor application performance and user activity while protecting + * sensitive internal data. + */ + const client = require('prom-client') const crypto = require('crypto') -const { buildLogData } = require('./logFormatter') +const { buildLogData } = require('./helpers/logFormatter') + +// Centralize the excluded emails list +const { excludedEmails } = require('./helpers/excludedEmails') // Import Kafka producer functions from your kafkaProducer.js file const { sendLogToKafka } = require('./kafkaProducer') @@ -8,20 +27,7 @@ const { sendLogToKafka } = require('./kafkaProducer') // Import Redis write client from your redisClient.js file const { redisWriteClient, redisReadClient } = require('./redisClient') -// Centralize the excluded emails list -const excludedEmails = new Set([ - 'bWF0ZXVzLmNhcmRvc29AbnViYW5rLmNvbS5icg==', - 'bHVpcy55YW1hZGFAbnViYW5rLmNvbS5icg==', - 'cmFmYWVsLmJyYWdlcm9sbGlAbnViYW5rLmNvbS5icg==', - 'a2Fpby5iZW5pY2lvQG51YmFuay5jb20uYnI=', - 'bWljaGFlbC5zYW50YUBudWJhbmsuY29tLmJy', - 'cGVkcm8uYXJhdWpvMUBudWJhbmsuY29tLmJy', - 'amhvbmF0YXMucm9zZW5kb0BudWJhbmsuY29tLmJy', - 'dml2aWFuLm1pcmFuZGFAbnViYW5rLmNvbS5icg==', - 'YnJ1bmEucGVyaW5AbnViYW5rLmNvbS5icg==' -]); - -class appMetrics { +class AppMetrics { constructor() { // Create a Registry to hold all metrics this.register = new client.Registry(); @@ -110,6 +116,10 @@ class appMetrics { const key = `unique_user:${encodedEmail}`; const currentTime = Date.now(); const eightHours = 8 * 60 * 60 * 1000; + + if (excludedEmails.has(encodedEmail)) { + return; // skip everything for excluded emails + } try { const storedTime = await redisReadClient.get(key); @@ -184,4 +194,4 @@ class appMetrics { } } -module.exports = appMetrics; \ No newline at end of file +module.exports = AppMetrics; \ No newline at end of file diff --git a/web/services/dateTimeHelper.js b/web/services/helpers/dateTimeHelper.js similarity index 57% rename from web/services/dateTimeHelper.js rename to web/services/helpers/dateTimeHelper.js index e39c442549..71ba878e40 100644 --- a/web/services/dateTimeHelper.js +++ b/web/services/helpers/dateTimeHelper.js @@ -1,3 +1,15 @@ +/** + * DateTime Helper Module + * + * This module provides a function to format the current date and time + * into a standardized, human-readable string. This formatted date/time + * is used across the application to timestamp logs and other events. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To ensure consistent and formatted timestamps in logging and monitoring. + */ + function getFormattedDateTime() { const d = new Date(); const pad = (n, size = 2) => n.toString().padStart(size, '0'); diff --git a/web/services/helpers/excludedEmails.js b/web/services/helpers/excludedEmails.js new file mode 100644 index 0000000000..43b290ac7c --- /dev/null +++ b/web/services/helpers/excludedEmails.js @@ -0,0 +1,25 @@ +/** + * Excluded Emails List + * + * This module provides a Set of Base64 encoded email addresses that should be + * excluded from tracking and logging in the application. This is used in both + * the metrics and logging modules to prevent any actions on these sensitive emails. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To protect sensitive internal email addresses and avoid logging/tracking them. + */ + +const excludedEmails = new Set([ + 'bWF0ZXVzLmNhcmRvc29AbnViYW5rLmNvbS5icg==', + 'bHVpcy55YW1hZGFAbnViYW5rLmNvbS5icg==', + 'cmFmYWVsLmJyYWdlcm9sbGlAbnViYW5rLmNvbS5icg==', + 'a2Fpby5iZW5pY2lvQG51YmFuay5jb20uYnI=', + 'bWljaGFlbC5zYW50YUBudWJhbmsuY29tLmJy', + 'cGVkcm8uYXJhdWpvMUBudWJhbmsuY29tLmJy', + 'amhvbmF0YXMucm9zZW5kb0BudWJhbmsuY29tLmJy', + 'dml2aWFuLm1pcmFuZGFAbnViYW5rLmNvbS5icg==', + 'YnJ1bmEucGVyaW5AbnViYW5rLmNvbS5icg==' + ]); + + module.exports = { excludedEmails } \ No newline at end of file diff --git a/web/services/logFormatter.js b/web/services/helpers/logFormatter.js similarity index 54% rename from web/services/logFormatter.js rename to web/services/helpers/logFormatter.js index d6bcb174b8..94396cb7b4 100644 --- a/web/services/logFormatter.js +++ b/web/services/helpers/logFormatter.js @@ -1,7 +1,13 @@ /** - * Build enriched log data. - * @param {object} userInfo - The Okta userinfo payload - * @returns {object} - The enriched log data + * Log Formatter Module + * + * This module provides a function to build enriched log data from a userInfo payload. + * It extracts and formats information such as timestamp, pod name, username, locale, + * email, zone information, and email verification status. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To standardize the format of log data sent to Kafka for user access logging. */ const { getFormattedDateTime } = require('./dateTimeHelper') diff --git a/web/services/kafkaProducer.js b/web/services/kafkaProducer.js index 2bd012000d..22815de6a4 100644 --- a/web/services/kafkaProducer.js +++ b/web/services/kafkaProducer.js @@ -1,4 +1,16 @@ -// filepath: /Users/jonathan.moraes.gft/Projects/new-numarquez/NuMarquez/web/services/kafkaProducer.js +/** + * Kafka Producer Module + * + * This module configures and manages a Kafka producer for sending log messages + * to a specified Kafka topic. It ensures that the topic exists before sending + * messages and handles connection setup and error logging. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To standardize and manage the process of sending log messages to Kafka + * for user access logging. + */ + const { Kafka } = require('kafkajs'); const KAFKA_LOADBALANCER_DNS = process.env.KAFKA_LOADBALANCER_DNS diff --git a/web/services/redisClient.js b/web/services/redisClient.js index fb17987b41..2a8c2e2563 100644 --- a/web/services/redisClient.js +++ b/web/services/redisClient.js @@ -1,3 +1,14 @@ +/** + * Redis Client Module + * + * This module configures and manages Redis clients for both read and write operations. + * It ensures that the clients are connected and handles connection setup and error logging. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To standardize and manage the process of connecting to Redis for read and write operations. + */ + const redis = require('redis'); // Function to create a Redis client diff --git a/web/setupProxy.js b/web/setupProxy.js index d024b712df..eee4e64d7b 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -4,33 +4,34 @@ const { createProxyMiddleware } = require('http-proxy-middleware') const path = require('path') const appMetrics = require('./services/appMetrics') const { sendLogToKafka } = require('./services/kafkaProducer') -const { getFormattedDateTime } = require('./services/dateTimeHelper') +const { getFormattedDateTime } = require('./services/helpers/dateTimeHelper') +const { excludedEmails } = require('./services/helpers/excludedEmails') const app = express(); const router = express.Router(); const distPath = path.join(__dirname, 'dist') // Initialize Metrics -const metrics = new appMetrics(); +const metrics = new appMetrics() // Middleware to expose /metrics endpoint app.get('/metrics', async (req, res) => { try { - res.set('Content-Type', metrics.register.contentType); - res.end(await metrics.getMetrics()); + res.set('Content-Type', metrics.register.contentType) + res.end(await metrics.getMetrics()) } catch (ex) { - res.status(500).end(ex); + res.status(500).end(ex) } }); -const { connectProducer } = require('./services/kafkaProducer'); +const { connectProducer } = require('./services/kafkaProducer') (async () => { try { - await connectProducer(); + await connectProducer() } catch (error) { - console.error('Error connecting Kafka producer:', error); - process.exit(1); + console.error('Error connecting Kafka producer:', error) + process.exit(1) } })(); @@ -81,9 +82,9 @@ app.listen(port, () => { console.log(`App listening on port ${port}!`) }) -app.use(express.json()); +app.use(express.json()) -const { buildLogData } = require('./services/logFormatter'); +const { buildLogData } = require('./services/helpers/logFormatter') // Endpoint to log user info and increment counters app.post('/api/loguserinfo', (req, res) => { @@ -92,13 +93,21 @@ app.post('/api/loguserinfo', (req, res) => { name, locale, zoneinfo - } = req.body; + } = req.body // Guard against invalid email values if (typeof email !== 'string') { - return res.status(400).json({ error: 'Invalid email format' }); + return res.status(400).json({ error: 'Invalid email format' }) } + // Calculate the encoded email once + const encodedEmail = Buffer.from(email).toString('base64') + + // Skip processing if the email is excluded + if (excludedEmails.has(encodedEmail)) { + return res.sendStatus(200); + } + // Create userInfo from request data (without circular references) const userInfo = { email, @@ -106,18 +115,18 @@ app.post('/api/loguserinfo', (req, res) => { locale, zoneinfo, email_verified: true - }; + } // Build enriched log data using the helper - const kafkaData = buildLogData(userInfo); - - // Encode the email for your own logs - const encodedEmail = Buffer.from(email).toString('base64'); + const kafkaData = buildLogData(userInfo) // Update metrics - metrics.incrementTotalLogins(email); - metrics.incrementUniqueLogins(email); + metrics.incrementTotalLogins(email) + metrics.incrementUniqueLogins(email) + if (excludedEmails.has(encodedEmail)) { + return; // skip everything for excluded emails + } // Console log for local debugging const logData = { accessLog: { @@ -125,14 +134,14 @@ app.post('/api/loguserinfo', (req, res) => { dateTime: getFormattedDateTime() } }; - console.log(JSON.stringify(logData)); + console.log(JSON.stringify(logData)) // Send meta info to Kafka - sendLogToKafka(kafkaData); + sendLogToKafka(kafkaData) // Response - res.sendStatus(200); + res.sendStatus(200) }); -module.exports = app; +module.exports = app diff --git a/web/src/auth/AuthContext.tsx b/web/src/auth/AuthContext.tsx index 943ea0f47e..7437d48f3b 100644 --- a/web/src/auth/AuthContext.tsx +++ b/web/src/auth/AuthContext.tsx @@ -1,3 +1,15 @@ +/** + * AuthContext Module + * + * This module provides a React context for managing authentication state using Okta. + * It handles user login, logout, and authentication state management, and logs user + * information to the backend upon successful login. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To manage user authentication state and integrate with Okta for secure login. + */ + import { AuthState, OktaAuth } from '@okta/okta-auth-js' import React, { createContext, useContext, useEffect, useState } from 'react' import { trackEvent } from '../components/ga4' diff --git a/web/src/components/ErrorBoundary.tsx b/web/src/components/ErrorBoundary.tsx index 8251a5bdb8..1aeab93ea8 100644 --- a/web/src/components/ErrorBoundary.tsx +++ b/web/src/components/ErrorBoundary.tsx @@ -1,3 +1,14 @@ +/** + * ErrorBoundary Component + * + * This component provides an error boundary for React components. It catches JavaScript errors + * anywhere in the child component tree, logs those errors, and displays a fallback UI. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To handle errors gracefully in the application and provide a fallback UI. + */ + import React, { ReactNode } from 'react' interface ErrorBoundaryProps { diff --git a/web/src/components/GAInitializer.tsx b/web/src/components/GAInitializer.tsx index e6dd021d69..56ca2a8565 100644 --- a/web/src/components/GAInitializer.tsx +++ b/web/src/components/GAInitializer.tsx @@ -1,3 +1,15 @@ +/** + * GAInitializer Component + * + * This component initializes Google Analytics 4 (GA4) and tracks page views. + * It sets the GA4 tracking ID based on the environment (staging or production) + * and tracks page views whenever the location changes. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To integrate Google Analytics 4 for tracking user interactions and page views in the application. + */ + import React, { useEffect } from 'react'; import { useLocation } from 'react-router-dom'; import { initializeGA, trackPageView } from './ga4'; diff --git a/web/src/components/Login.tsx b/web/src/components/Login.tsx index 4b0930be2c..14340e9aa6 100644 --- a/web/src/components/Login.tsx +++ b/web/src/components/Login.tsx @@ -1,3 +1,14 @@ +/** + * Login Component + * + * This component provides the login page for the application. It integrates with Okta for authentication + * and tracks user interactions using Google Analytics 4 (GA4). + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To provide a secure login page and track user interactions for analytics. + */ + import { Box, Button, Container } from '@mui/material' import { Helmet } from 'react-helmet-async' import { Navigate } from 'react-router-dom' diff --git a/web/src/components/LoginCallback.tsx b/web/src/components/LoginCallback.tsx index e1a208c98e..63e4d8d5b4 100644 --- a/web/src/components/LoginCallback.tsx +++ b/web/src/components/LoginCallback.tsx @@ -1,3 +1,15 @@ +/** + * LoginCallback Component + * + * This component handles the callback from Okta after a user has authenticated. + * It processes the authentication response, sets the user information, logs the user info, + * and redirects the user to the appropriate page. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To handle the Okta authentication callback and manage user session state. + */ + import React, { useEffect } from 'react' import { Box, CircularProgress } from '@mui/material' import { useNavigate } from 'react-router-dom' diff --git a/web/src/components/PrivateRoute.tsx b/web/src/components/PrivateRoute.tsx index 6aa4bf3f99..398fb90537 100644 --- a/web/src/components/PrivateRoute.tsx +++ b/web/src/components/PrivateRoute.tsx @@ -1,3 +1,15 @@ +/** + * PrivateRoute Component + * + * This component provides a wrapper for routes that require authentication. + * It checks if the user is authenticated and either renders the child components + * or redirects the user to the login page. + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To protect routes that require authentication and ensure only authenticated users can access them. + */ + import { useAuth } from '../auth/AuthContext' import CircularProgress from '@mui/material/CircularProgress' import React from 'react' diff --git a/web/src/components/ga4.tsx b/web/src/components/ga4.tsx index fc56c66394..861ec51ec0 100644 --- a/web/src/components/ga4.tsx +++ b/web/src/components/ga4.tsx @@ -1,3 +1,14 @@ +/** + * GA4 Module + * + * This module provides functions to initialize Google Analytics 4 (GA4) and track page views and events. + * It dynamically sets the GA4 tracking ID based on the environment (staging or production). + * + * Author: Jonathan Moraes + * Created: 2025-02-19 + * Reason: To integrate Google Analytics 4 for tracking user interactions and page views in the application. + */ + import ReactGA from 'react-ga4'; function decodeBase64(str: string) { From 78e0d61b4696c72a8231e1e9885ed8ea16cee3cd Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 20 Feb 2025 10:32:45 -0300 Subject: [PATCH 13/27] Preparing GA tag for release in prod --- web/src/index.prod.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/src/index.prod.html b/web/src/index.prod.html index 1c40d388d2..9d6224c956 100644 --- a/web/src/index.prod.html +++ b/web/src/index.prod.html @@ -11,14 +11,14 @@ Nu Data Lineage - + })(window, document, 'script', 'dataLayer', 'GTM-MLNCWHZX'); @@ -32,8 +32,8 @@ - +
From 20c44607d3010562cdf8f84f85a26738206775a0 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 20 Feb 2025 11:44:33 -0300 Subject: [PATCH 14/27] Changing saved logins from 12h to 7 days --- web/services/appMetrics.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 5ef0bd346b..347b4a96a3 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -115,7 +115,7 @@ class AppMetrics { const encodedEmail = this.encodeEmail(email); const key = `unique_user:${encodedEmail}`; const currentTime = Date.now(); - const eightHours = 8 * 60 * 60 * 1000; + const sevenDays = 7 * 24 * 60 * 60 * 1000; // 7 days in milliseconds if (excludedEmails.has(encodedEmail)) { return; // skip everything for excluded emails @@ -123,7 +123,7 @@ class AppMetrics { try { const storedTime = await redisReadClient.get(key); - if (!storedTime || (currentTime - parseInt(storedTime)) > eightHours) { + if (!storedTime || (currentTime - parseInt(storedTime)) > sevenDays) { // Set the key with expiration (7 days) await redisWriteClient.set(key, currentTime, { EX: 7 * 24 * 60 * 60 }); this.uniqueUserLoginCounter.inc(); From 2cbb9065615d88e658ce44afde3f69b4dbd40d4b Mon Sep 17 00:00:00 2001 From: Kess220 Date: Tue, 18 Feb 2025 15:09:42 -0300 Subject: [PATCH 15/27] bug-fix-columnLineage-request --- web/src/store/requests/columnlineage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/store/requests/columnlineage.ts b/web/src/store/requests/columnlineage.ts index b9eca5266f..3f38636920 100644 --- a/web/src/store/requests/columnlineage.ts +++ b/web/src/store/requests/columnlineage.ts @@ -19,6 +19,6 @@ export const getColumnLineage = async ( const encodedNamespace = encodeURIComponent(namespace) const encodedName = encodeURIComponent(name) const nodeId = generateNodeId(nodeType, encodedNamespace, encodedName) - const url = `${API_URL}/column-lineage?nodeId=${nodeId}&depth=${depth}&withDownstream=true` + const url = `${API_URL}/column-lineage?nodeId=${nodeId}&depth=${depth}&withDownstream=${withDownstream}` return genericFetchWrapper(url, { method: 'GET' }, 'fetchColumnLineage') } From e0d4bb581028c11f8641fc9450d697cca7aacf4e Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 20 Feb 2025 11:53:13 -0300 Subject: [PATCH 16/27] Organizing code --- web/setupProxy.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index 60e5e408ad..ff1a8801fe 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -6,6 +6,7 @@ const appMetrics = require('./services/appMetrics') const { sendLogToKafka } = require('./services/kafkaProducer') const { getFormattedDateTime } = require('./services/helpers/dateTimeHelper') const { excludedEmails } = require('./services/helpers/excludedEmails') +const { connectProducer } = require('./services/kafkaProducer') const { buildLogData } = require('./services/helpers/logFormatter') const app = express(); @@ -25,8 +26,6 @@ app.get('/metrics', async (req, res) => { } }); -const { connectProducer } = require('./services/kafkaProducer') - (async () => { try { await connectProducer() From 08e6cd2dcaf3daff2d467d43a9ba69b368f886d3 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 27 Feb 2025 10:38:51 -0300 Subject: [PATCH 17/27] Adjusting userActivity metric --- web/services/appMetrics.js | 69 +++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 347b4a96a3..48c210c319 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -156,42 +156,41 @@ class AppMetrics { }, 1000); // Update every second } - /** - * Updates the user activity gauge every minute. - * Examines Redis keys starting with "unique_user:" and counts keys where the stored timestamp is within 72 hours. - * Excludes specific base64-encoded emails from the count. - */ - async updateUserActivity() { - setInterval(async () => { - try { - const keys = await redisReadClient.keys('unique_user:*'); - let activeUsers = 0; - const currentTime = Date.now(); - const seventyTwoHours = 72 * 60 * 60 * 1000; - - if (keys && keys.length) { - // Get values for all keys - const pipeline = redisReadClient.multi(); - keys.forEach((key) => { - pipeline.get(key); - }); - const results = await pipeline.exec(); - - keys.forEach((key, index) => { - const storedTime = results[index] ? parseInt(results[index]) : 0; - const encodedEmail = key.replace('unique_user:', ''); - // Only count if within 72h and not in excluded list - if ((currentTime - storedTime) <= seventyTwoHours && !excludedEmails.has(encodedEmail)) { - activeUsers++; - } - }); - } - this.userActivityGauge.set(activeUsers > 0 ? 1 : 0); - } catch (err) { - console.error('Error updating user activity gauge:', err); +/** + * Updates the user activity gauge every minute. + * Examines Redis keys starting with "unique_user:" and checks if there has been any activity in the last minute. + * Excludes specific base64-encoded emails from the count. + */ +async updateUserActivity() { + setInterval(async () => { + try { + const keys = await redisReadClient.keys('unique_user:*'); + let activeUsers = 0; + const currentTime = Date.now(); + const oneMinute = 60 * 1000; + + if (keys && keys.length) { + // Get values for all keys + const pipeline = redisReadClient.multi(); + keys.forEach((key) => { + pipeline.get(key); + }); + const results = await pipeline.exec(); + + keys.forEach((key, index) => { + const storedTime = results[index] ? parseInt(results[index]) : 0; + const encodedEmail = key.replace('unique_user:', ''); + // Only count if within 1 minute and not in excluded list + if ((currentTime - storedTime) <= oneMinute && !excludedEmails.has(encodedEmail)) { + activeUsers++; + } + }); } - }, 60 * 1000); // Update every minute + this.userActivityGauge.set(activeUsers > 0 ? 1 : 0); + } catch (err) { + console.error('Error updating user activity gauge:', err); + } + }, 60 * 1000); // Update every minute } } - module.exports = AppMetrics; \ No newline at end of file From 264180adb57a15cff01f8572cf9df629de1cfa16 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 27 Feb 2025 16:32:13 -0300 Subject: [PATCH 18/27] Changing name of metric --- web/services/appMetrics.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 48c210c319..64bd6ea9db 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -51,7 +51,7 @@ class AppMetrics { // Define Prometheus Gauge for user activity in the last 72 hours this.userActivityGauge = new client.Gauge({ - name: 'user_activity_last_72_hours', + name: 'user_access_activity_gauge', help: 'Indicates whether there have been users in the last 72 hours (1) or not (0)', }); From 2d5966254da05ce73985c3bcf1bcd13190fdab64 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 27 Feb 2025 17:28:36 -0300 Subject: [PATCH 19/27] Changing logic to metric --- web/services/appMetrics.js | 64 ++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 64bd6ea9db..ab60725717 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -156,41 +156,39 @@ class AppMetrics { }, 1000); // Update every second } -/** - * Updates the user activity gauge every minute. - * Examines Redis keys starting with "unique_user:" and checks if there has been any activity in the last minute. - * Excludes specific base64-encoded emails from the count. - */ -async updateUserActivity() { - setInterval(async () => { - try { - const keys = await redisReadClient.keys('unique_user:*'); - let activeUsers = 0; - const currentTime = Date.now(); - const oneMinute = 60 * 1000; - - if (keys && keys.length) { - // Get values for all keys - const pipeline = redisReadClient.multi(); - keys.forEach((key) => { - pipeline.get(key); - }); - const results = await pipeline.exec(); - - keys.forEach((key, index) => { - const storedTime = results[index] ? parseInt(results[index]) : 0; - const encodedEmail = key.replace('unique_user:', ''); - // Only count if within 1 minute and not in excluded list - if ((currentTime - storedTime) <= oneMinute && !excludedEmails.has(encodedEmail)) { - activeUsers++; + /** + * Updates the user activity gauge every minute. + * Checks if there is any user logged in at the moment. + * Excludes specific base64-encoded emails from the count. + */ + async updateUserActivity() { + setInterval(async () => { + try { + const keys = await redisReadClient.keys('unique_user:*'); + let userActivityDetected = false; + + if (keys && keys.length) { + // Get values for all keys + const pipeline = redisReadClient.multi(); + keys.forEach((key) => { + pipeline.get(key); + }); + await pipeline.exec(); + + for (const key of keys) { + const encodedEmail = key.replace('unique_user:', ''); + // Check if the email is not in the excluded list + if (!excludedEmails.has(encodedEmail)) { + userActivityDetected = true; + break; // No need to check further if activity is detected + } } - }); + } + this.userActivityGauge.set(userActivityDetected ? 1 : 0); + } catch (err) { + console.error('Error updating user activity gauge:', err); } - this.userActivityGauge.set(activeUsers > 0 ? 1 : 0); - } catch (err) { - console.error('Error updating user activity gauge:', err); - } - }, 60 * 1000); // Update every minute + }, 60 * 1000); // Update every minute } } module.exports = AppMetrics; \ No newline at end of file From 2a4ea319ff343a5e7c139804941dd9df017304cc Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Fri, 28 Feb 2025 16:40:17 -0300 Subject: [PATCH 20/27] Removing unnecessary methods --- web/services/appMetrics.js | 42 -------------------------------------- web/setupProxy.js | 2 +- 2 files changed, 1 insertion(+), 43 deletions(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index ab60725717..aaa0a204a3 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -49,17 +49,10 @@ class AppMetrics { help: 'Uptime of the application in seconds', }); - // Define Prometheus Gauge for user activity in the last 72 hours - this.userActivityGauge = new client.Gauge({ - name: 'user_access_activity_gauge', - help: 'Indicates whether there have been users in the last 72 hours (1) or not (0)', - }); - // Register the counters and gauges this.register.registerMetric(this.uniqueUserLoginCounter); this.register.registerMetric(this.totalUserLoginCounter); this.register.registerMetric(this.appUptimeGauge); - this.register.registerMetric(this.userActivityGauge); // (Optional) Collect default metrics like CPU and memory usage client.collectDefaultMetrics({ register: this.register }); @@ -155,40 +148,5 @@ class AppMetrics { this.appUptimeGauge.set(uptimeSeconds); }, 1000); // Update every second } - - /** - * Updates the user activity gauge every minute. - * Checks if there is any user logged in at the moment. - * Excludes specific base64-encoded emails from the count. - */ - async updateUserActivity() { - setInterval(async () => { - try { - const keys = await redisReadClient.keys('unique_user:*'); - let userActivityDetected = false; - - if (keys && keys.length) { - // Get values for all keys - const pipeline = redisReadClient.multi(); - keys.forEach((key) => { - pipeline.get(key); - }); - await pipeline.exec(); - - for (const key of keys) { - const encodedEmail = key.replace('unique_user:', ''); - // Check if the email is not in the excluded list - if (!excludedEmails.has(encodedEmail)) { - userActivityDetected = true; - break; // No need to check further if activity is detected - } - } - } - this.userActivityGauge.set(userActivityDetected ? 1 : 0); - } catch (err) { - console.error('Error updating user activity gauge:', err); - } - }, 60 * 1000); // Update every minute - } } module.exports = AppMetrics; \ No newline at end of file diff --git a/web/setupProxy.js b/web/setupProxy.js index ff1a8801fe..321f86d26d 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -63,7 +63,7 @@ app.use('/jobs', express.static(distPath)) app.use('/datasets/column-level', express.static(distPath)) // Proxy API requests -+app.use('/api/v1', createProxyMiddleware(apiOptions)) +app.use('/api/v1', createProxyMiddleware(apiOptions)) app.use('/api/v2beta', createProxyMiddleware(apiOptions)) // Healthcheck route From fbe9e61f42008f72702f49556a4424389f33a8e8 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 6 Mar 2025 16:58:58 -0300 Subject: [PATCH 21/27] Cleaning up code base --- web/services/appMetrics.js | 66 +++++++++++++------------- web/services/helpers/dateTimeHelper.js | 22 ++++----- web/services/helpers/logFormatter.js | 8 ++-- web/services/kafkaProducer.js | 20 ++++---- web/services/redisClient.js | 24 +++++----- web/setupProxy.js | 8 ++-- 6 files changed, 74 insertions(+), 74 deletions(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index aaa0a204a3..2c0a9cb044 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -30,48 +30,48 @@ const { redisWriteClient, redisReadClient } = require('./redisClient') class AppMetrics { constructor() { // Create a Registry to hold all metrics - this.register = new client.Registry(); + this.register = new client.Registry() // Define Prometheus Counters this.uniqueUserLoginCounter = new client.Counter({ name: 'unique_user_login_total', help: 'Total number of unique user logins', - }); + }) this.totalUserLoginCounter = new client.Counter({ name: 'total_user_logins', help: 'Total number of user logins', - }); + }) // Define Prometheus Gauge for application uptime this.appUptimeGauge = new client.Gauge({ name: 'app_uptime_seconds', help: 'Uptime of the application in seconds', - }); + }) // Register the counters and gauges - this.register.registerMetric(this.uniqueUserLoginCounter); - this.register.registerMetric(this.totalUserLoginCounter); - this.register.registerMetric(this.appUptimeGauge); + this.register.registerMetric(this.uniqueUserLoginCounter) + this.register.registerMetric(this.totalUserLoginCounter) + this.register.registerMetric(this.appUptimeGauge) // (Optional) Collect default metrics like CPU and memory usage - client.collectDefaultMetrics({ register: this.register }); + client.collectDefaultMetrics({ register: this.register }) // Record the application start time - this.startTime = Date.now(); + this.startTime = Date.now() // Update the uptime gauge periodically - this.updateUptime(); + this.updateUptime() // Update the user activity gauge periodically - this.updateUserActivity(); + this.updateUserActivity() } /** * Returns the Prometheus metrics as a string */ async getMetrics() { - return await this.register.metrics(); + return await this.register.metrics() } /** @@ -80,23 +80,23 @@ class AppMetrics { */ incrementTotalLogins(email) { if (typeof email !== 'string') { - console.error('Invalid email provided to incrementTotalLogins:', email); + console.error('Invalid email provided to incrementTotalLogins:', email) return; // Early exit if email is not a string. } - const encodedEmail = this.encodeEmail(email); + const encodedEmail = this.encodeEmail(email) if (excludedEmails.has(encodedEmail)) { - return; // Do not increment if user is in the excluded list + return // Do not increment if user is in the excluded list } - this.totalUserLoginCounter.inc(); + this.totalUserLoginCounter.inc() } encodeEmail(email) { // Optionally check here: if (!email) { - console.error('No email provided to encodeEmail.'); - return ''; + console.error('No email provided to encodeEmail.') + return '' } - return Buffer.from(email).toString('base64'); + return Buffer.from(email).toString('base64') } /** @@ -105,28 +105,28 @@ class AppMetrics { * @param {string} email - The user's email */ async incrementUniqueLogins(email) { - const encodedEmail = this.encodeEmail(email); - const key = `unique_user:${encodedEmail}`; - const currentTime = Date.now(); - const sevenDays = 7 * 24 * 60 * 60 * 1000; // 7 days in milliseconds + const encodedEmail = this.encodeEmail(email) + const key = `unique_user:${encodedEmail}` + const currentTime = Date.now() + const sevenDays = 7 * 24 * 60 * 60 * 1000 // 7 days in milliseconds if (excludedEmails.has(encodedEmail)) { return; // skip everything for excluded emails } try { - const storedTime = await redisReadClient.get(key); + const storedTime = await redisReadClient.get(key) if (!storedTime || (currentTime - parseInt(storedTime)) > sevenDays) { // Set the key with expiration (7 days) - await redisWriteClient.set(key, currentTime, { EX: 7 * 24 * 60 * 60 }); + await redisWriteClient.set(key, currentTime, { EX: 7 * 24 * 60 * 60 }) this.uniqueUserLoginCounter.inc(); - const userInfo = { email}; - const logData = buildLogData(userInfo); - sendLogToKafka(logData); + const userInfo = { email} + const logData = buildLogData(userInfo) + sendLogToKafka(logData) } } catch (err) { - console.error('Error in incrementUniqueLogins:', err); + console.error('Error in incrementUniqueLogins:', err) } } @@ -136,7 +136,7 @@ class AppMetrics { * @returns {string} */ hashEmail(email) { - return crypto.createHash('sha256').update(email).digest('hex'); + return crypto.createHash('sha256').update(email).digest('hex') } /** @@ -144,9 +144,9 @@ class AppMetrics { */ updateUptime() { setInterval(() => { - const uptimeSeconds = (Date.now() - this.startTime) / 1000; - this.appUptimeGauge.set(uptimeSeconds); + const uptimeSeconds = (Date.now() - this.startTime) / 1000 + this.appUptimeGauge.set(uptimeSeconds) }, 1000); // Update every second } } -module.exports = AppMetrics; \ No newline at end of file +module.exports = AppMetrics \ No newline at end of file diff --git a/web/services/helpers/dateTimeHelper.js b/web/services/helpers/dateTimeHelper.js index 71ba878e40..888f5bab50 100644 --- a/web/services/helpers/dateTimeHelper.js +++ b/web/services/helpers/dateTimeHelper.js @@ -11,16 +11,16 @@ */ function getFormattedDateTime() { - const d = new Date(); - const pad = (n, size = 2) => n.toString().padStart(size, '0'); - const year = d.getFullYear(); - const month = pad(d.getMonth() + 1); - const day = pad(d.getDate()); - const hour = pad(d.getHours()); - const minute = pad(d.getMinutes()); - const second = pad(d.getSeconds()); - const ms = pad(d.getMilliseconds(), 3); - return `${year}-${month}-${day} ${hour}:${minute}:${second}.${ms}`; + const d = new Date() + const pad = (n, size = 2) => n.toString().padStart(size, '0') + const year = d.getFullYear() + const month = pad(d.getMonth() + 1) + const day = pad(d.getDate()) + const hour = pad(d.getHours()) + const minute = pad(d.getMinutes()) + const second = pad(d.getSeconds()) + const ms = pad(d.getMilliseconds(), 3) + return `${year}-${month}-${day} ${hour}:${minute}:${second}.${ms}` } - module.exports = { getFormattedDateTime }; \ No newline at end of file + module.exports = { getFormattedDateTime } \ No newline at end of file diff --git a/web/services/helpers/logFormatter.js b/web/services/helpers/logFormatter.js index 94396cb7b4..23ea016aaf 100644 --- a/web/services/helpers/logFormatter.js +++ b/web/services/helpers/logFormatter.js @@ -13,8 +13,8 @@ const { getFormattedDateTime } = require('./dateTimeHelper') function buildLogData(userInfo) { - const timestamp = getFormattedDateTime(); - const podName = process.env.POD_NAME || "unknown-pod"; + const timestamp = getFormattedDateTime() + const podName = process.env.POD_NAME || "unknown-pod" return { timestamp, @@ -24,7 +24,7 @@ function buildLogData(userInfo) { email: userInfo.email, zoneinfo: userInfo.zoneinfo, email_verified: userInfo.email_verified - }; + } } - module.exports = { buildLogData }; \ No newline at end of file + module.exports = { buildLogData } \ No newline at end of file diff --git a/web/services/kafkaProducer.js b/web/services/kafkaProducer.js index 22815de6a4..49d15fcdc9 100644 --- a/web/services/kafkaProducer.js +++ b/web/services/kafkaProducer.js @@ -11,7 +11,7 @@ * for user access logging. */ -const { Kafka } = require('kafkajs'); +const { Kafka } = require('kafkajs') const KAFKA_LOADBALANCER_DNS = process.env.KAFKA_LOADBALANCER_DNS const KAFKA_PORT = process.env.KAFKA_PORT @@ -22,12 +22,12 @@ const kafka = new Kafka({ brokers: [`${KAFKA_LOADBALANCER_DNS}:${KAFKA_PORT}`] }); -const producer = kafka.producer(); +const producer = kafka.producer() const connectProducer = async () => { - await producer.connect(); - console.log('Kafka producer connected.'); -}; + await producer.connect() + console.log('Kafka producer connected.') +} const sendLogToKafka = async (log) => { try { @@ -36,11 +36,11 @@ const sendLogToKafka = async (log) => { messages: [ { value: JSON.stringify(log) } ] - }); - console.log('Log sent to Kafka.'); + }) + console.log('Log sent to Kafka.') } catch (error) { - console.error('Error sending log to Kafka:', error); + console.error('Error sending log to Kafka:', error) } -}; +} -module.exports = { connectProducer, sendLogToKafka, producer }; \ No newline at end of file +module.exports = { connectProducer, sendLogToKafka, producer } \ No newline at end of file diff --git a/web/services/redisClient.js b/web/services/redisClient.js index 2a8c2e2563..1e7e387c24 100644 --- a/web/services/redisClient.js +++ b/web/services/redisClient.js @@ -9,23 +9,23 @@ * Reason: To standardize and manage the process of connecting to Redis for read and write operations. */ -const redis = require('redis'); +const redis = require('redis') // Function to create a Redis client const createRedisClient = (host, port, role) => { const client = redis.createClient({ url: `redis://${host}:${port}`, - }); + }) client.on('error', (err) => { - console.error(`Redis ${role} client error:`, err); - }); + console.error(`Redis ${role} client error:`, err) + }) client.on('connect', () => { - console.log(`Connected to Redis ${role} client`); + console.log(`Connected to Redis ${role} client`) }); - return client; + return client }; // Create Redis write and read clients @@ -44,14 +44,14 @@ const redisReadClient = createRedisClient( // Connect the clients once (async () => { try { - await redisWriteClient.connect(); - await redisReadClient.connect(); - console.log('Redis clients connected successfully.'); + await redisWriteClient.connect() + await redisReadClient.connect() + console.log('Redis clients connected successfully.') } catch (err) { - console.error('Error connecting to Redis:', err); - process.exit(1); + console.error('Error connecting to Redis:', err) + process.exit(1) } -})(); +})() module.exports = { redisWriteClient, diff --git a/web/setupProxy.js b/web/setupProxy.js index 321f86d26d..fc1e56852f 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -24,7 +24,7 @@ app.get('/metrics', async (req, res) => { } catch (ex) { res.status(500).end(ex) } -}); +}) (async () => { try { @@ -33,7 +33,7 @@ app.get('/metrics', async (req, res) => { console.error('Error connecting Kafka producer:', error) process.exit(1) } -})(); +})() const environmentVariable = (variableName) => { const value = process.env[variableName] @@ -52,7 +52,7 @@ const apiOptions = { changeOrigin: true, } -app.use(express.json()); +app.use(express.json()) // Serve static files for specific routes app.use('/', express.static(distPath)) @@ -103,7 +103,7 @@ app.post('/api/loguserinfo', (req, res) => { // Skip processing if the email is excluded if (excludedEmails.has(encodedEmail)) { - return res.sendStatus(200); + return res.sendStatus(200) } // Create userInfo from request data (without circular references) From df0381d87f35ac94c89b3238b828af5dda2e6254 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 6 Mar 2025 17:12:48 -0300 Subject: [PATCH 22/27] Cleaning up updateUserMetric old metric --- web/services/appMetrics.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 2c0a9cb044..73dbe85c55 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -62,9 +62,6 @@ class AppMetrics { // Update the uptime gauge periodically this.updateUptime() - - // Update the user activity gauge periodically - this.updateUserActivity() } /** From 6ed363569e9deb020d4fc19613068bd5a9c6b8f8 Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 6 Mar 2025 17:43:06 -0300 Subject: [PATCH 23/27] Removing redundant code --- web/setupProxy.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index fc1e56852f..fe38923750 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -82,8 +82,6 @@ app.listen(port, () => { console.log(`App listening on port ${port}!`) }) -app.use(express.json()) - // Endpoint to log user info and increment counters app.post('/api/loguserinfo', (req, res) => { const { From e5e6b2df15372371a0bf5711a70ebe1bf48de95c Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 6 Mar 2025 18:00:39 -0300 Subject: [PATCH 24/27] Removing redundant code --- web/setupProxy.js | 69 +++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index fe38923750..728c9056f7 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -1,4 +1,3 @@ - const express = require('express') const { createProxyMiddleware } = require('http-proxy-middleware') const path = require('path') @@ -9,8 +8,8 @@ const { excludedEmails } = require('./services/helpers/excludedEmails') const { connectProducer } = require('./services/kafkaProducer') const { buildLogData } = require('./services/helpers/logFormatter') -const app = express(); -const router = express.Router(); +const app = express() +const router = express.Router() const distPath = path.join(__dirname, 'dist') // Initialize Metrics @@ -26,6 +25,7 @@ app.get('/metrics', async (req, res) => { } }) +// Connect Kafka producer (async () => { try { await connectProducer() @@ -70,48 +70,33 @@ app.use('/api/v2beta', createProxyMiddleware(apiOptions)) router.get('/healthcheck', (req, res) => { res.send('OK') }) - app.use(router) -// **Catch-All Route to Serve index.html for Client-Side Routing** -app.get('*', (req, res) => { - res.sendFile(path.join(distPath, 'index.html')) -}) - -app.listen(port, () => { - console.log(`App listening on port ${port}!`) -}) - // Endpoint to log user info and increment counters app.post('/api/loguserinfo', (req, res) => { - const { - email = '', - name, - locale, - zoneinfo - } = req.body + const { email = '', name, locale, zoneinfo } = req.body // Guard against invalid email values if (typeof email !== 'string') { return res.status(400).json({ error: 'Invalid email format' }) } - // Calculate the encoded email once - const encodedEmail = Buffer.from(email).toString('base64') + // Calculate the encoded email once + const encodedEmail = Buffer.from(email).toString('base64') - // Skip processing if the email is excluded - if (excludedEmails.has(encodedEmail)) { - return res.sendStatus(200) - } + // Skip processing if the email is excluded + if (excludedEmails.has(encodedEmail)) { + return res.sendStatus(200) + } - // Create userInfo from request data (without circular references) - const userInfo = { - email, - name, - locale, - zoneinfo, - email_verified: true - } + // Create userInfo from request data (without circular references) + const userInfo = { + email, + name, + locale, + zoneinfo, + email_verified: true + } // Build enriched log data using the helper const kafkaData = buildLogData(userInfo) @@ -120,16 +105,13 @@ app.post('/api/loguserinfo', (req, res) => { metrics.incrementTotalLogins(email) metrics.incrementUniqueLogins(email) - if (excludedEmails.has(encodedEmail)) { - return; // skip everything for excluded emails - } // Console log for local debugging const logData = { accessLog: { email: encodedEmail, dateTime: getFormattedDateTime() } - }; + } console.log(JSON.stringify(logData)) // Send meta info to Kafka @@ -137,7 +119,16 @@ app.post('/api/loguserinfo', (req, res) => { // Response res.sendStatus(200) -}); +}) -module.exports = app +// **Catch-All Route to Serve index.html for Client-Side Routing** +app.get('*', (req, res) => { + res.sendFile(path.join(distPath, 'index.html')) +}) + +// Start the server (only one call to app.listen) +app.listen(port, () => { + console.log(`App listening on port ${port}!`) +}) +module.exports = app \ No newline at end of file From d43a3cb2eb3c2c67e4bba4f54287b22393ebe4de Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 6 Mar 2025 18:17:50 -0300 Subject: [PATCH 25/27] Re-organizing code --- web/setupProxy.js | 76 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index 728c9056f7..c59193e72c 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -73,52 +73,56 @@ router.get('/healthcheck', (req, res) => { app.use(router) // Endpoint to log user info and increment counters -app.post('/api/loguserinfo', (req, res) => { - const { email = '', name, locale, zoneinfo } = req.body +app.post('/api/loguserinfo', async (req, res, next) => { + try { + const { email = '', name, locale, zoneinfo } = req.body - // Guard against invalid email values - if (typeof email !== 'string') { - return res.status(400).json({ error: 'Invalid email format' }) - } + // Guard against invalid email values + if (typeof email !== 'string') { + return res.status(400).json({ error: 'Invalid email format' }) + } - // Calculate the encoded email once - const encodedEmail = Buffer.from(email).toString('base64') + // Calculate the encoded email once + const encodedEmail = Buffer.from(email).toString('base64') - // Skip processing if the email is excluded - if (excludedEmails.has(encodedEmail)) { - return res.sendStatus(200) - } + // Skip processing if the email is excluded + if (excludedEmails.has(encodedEmail)) { + return res.sendStatus(200) + } - // Create userInfo from request data (without circular references) - const userInfo = { - email, - name, - locale, - zoneinfo, - email_verified: true - } + // Create userInfo from request data (without circular references) + const userInfo = { + email, + name, + locale, + zoneinfo, + email_verified: true + } - // Build enriched log data using the helper - const kafkaData = buildLogData(userInfo) + // Build enriched log data using the helper + const kafkaData = buildLogData(userInfo) - // Update metrics - metrics.incrementTotalLogins(email) - metrics.incrementUniqueLogins(email) + // Update metrics + metrics.incrementTotalLogins(email) + metrics.incrementUniqueLogins(email) - // Console log for local debugging - const logData = { - accessLog: { - email: encodedEmail, - dateTime: getFormattedDateTime() + // Console log for local debugging + const logData = { + accessLog: { + email: encodedEmail, + dateTime: getFormattedDateTime() + } } - } - console.log(JSON.stringify(logData)) + console.log(JSON.stringify(logData)) - // Send meta info to Kafka - sendLogToKafka(kafkaData) + // Send meta info to Kafka + sendLogToKafka(kafkaData) - // Response - res.sendStatus(200) + // Response + res.sendStatus(200) + } catch (error) { + next(error) + } }) // **Catch-All Route to Serve index.html for Client-Side Routing** From 81bc6161d3a0e2e4b88dfe0eb8722b6f5467469e Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 6 Mar 2025 19:05:48 -0300 Subject: [PATCH 26/27] Implementing await call --- web/setupProxy.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/web/setupProxy.js b/web/setupProxy.js index c59193e72c..da7bb884d8 100644 --- a/web/setupProxy.js +++ b/web/setupProxy.js @@ -116,7 +116,7 @@ app.post('/api/loguserinfo', async (req, res, next) => { console.log(JSON.stringify(logData)) // Send meta info to Kafka - sendLogToKafka(kafkaData) + await sendLogToKafka(kafkaData) // Response res.sendStatus(200) @@ -130,6 +130,12 @@ app.get('*', (req, res) => { res.sendFile(path.join(distPath, 'index.html')) }) +// Error-handling middleware +app.use((err, req, res, next) => { + console.error(err.stack) + res.status(500).send('Something broke!') +}) + // Start the server (only one call to app.listen) app.listen(port, () => { console.log(`App listening on port ${port}!`) From 94573d67794ffa03e8a2f48588b832aef7bae86b Mon Sep 17 00:00:00 2001 From: Jonathan P Moraes Date: Thu, 6 Mar 2025 19:23:17 -0300 Subject: [PATCH 27/27] removing ; --- web/services/appMetrics.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/services/appMetrics.js b/web/services/appMetrics.js index 73dbe85c55..40ac5a6d52 100644 --- a/web/services/appMetrics.js +++ b/web/services/appMetrics.js @@ -78,7 +78,7 @@ class AppMetrics { incrementTotalLogins(email) { if (typeof email !== 'string') { console.error('Invalid email provided to incrementTotalLogins:', email) - return; // Early exit if email is not a string. + return // Early exit if email is not a string. } const encodedEmail = this.encodeEmail(email) if (excludedEmails.has(encodedEmail)) {