Skip to content

Commit

Permalink
Merge pull request #5939 from Countly/stats-metric
Browse files Browse the repository at this point in the history 
[SER-2272] Update data points calculation in server stats job
  • Loading branch information
@ArtursKadikis
ArtursKadikis authored Jan 28, 2025
2 parents c7b2197 + f364bdf commit 28fa8a2
Show file tree
Hide file tree
Showing 5 changed files with 226 additions and 135 deletions.
5 changes: 4 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
## Version 24.05.xx
## Version 24.05.XX
Fixes:
- [push] Using apns-id header as message result in debug mode
- [drill] [license] Update license loader to enable supplying db client
- [server-stats] Fix data point calculation in job
- [users] Format data points displayed in user sidebar

Dependencies:
- Bump fs-extra from 11.2.0 to 11.3.0
Expand Down
295 changes: 165 additions & 130 deletions plugins/server-stats/api/jobs/stats.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,20 @@
const job = require('../../../../api/parts/jobs/job.js'),
tracker = require('../../../../api/parts/mgmt/tracker.js'),
log = require('../../../../api/utils/log.js')('job:stats'),
config = require("../../../../frontend/express/config.js"),
config = require('../../../../frontend/express/config.js'),
pluginManager = require('../../../pluginManager.js'),
serverStats = require('../parts/stats.js'),
moment = require('moment-timezone'),
request = require('countly-request')(pluginManager.getConfig("security"));
request = require('countly-request')(pluginManager.getConfig('security'));

let drill;
try {
drill = require('../../../drill/api/parts/data/drill.js');
}
catch (ex) {
log.e(ex);
drill = null;
}

const promisedLoadConfigs = function(db) {
return new Promise((resolve) => {
Expand All @@ -27,146 +37,171 @@ class StatsJob extends job.Job {
super(name, data);
}

/**
* @param {Object} allData - All server stats data from the beginning of time
* @returns {Object} Sum of all data, average data per month, and last three month data
*/
static generateDataSummary(allData) {
const data = {};
data.all = 0;
data.month3 = [];
const utcMoment = moment.utc();
const months = {};
for (let i = 0; i < 3; i++) {
months[utcMoment.format('YYYY:M')] = true;
utcMoment.subtract(1, 'months');
}
for (const [key, value] of Object.entries(allData)) {
data.all += value;
if (months[key]) {
data.month3.push(key + ' - ' + (value));
}
}
data.avg = Math.round((data.all / Object.keys(allData).length) * 100) / 100;

return data;
}

/**
* @param {Object} allData - All server stats data from the beginning of time
* @returns {Object} Monthly data
*/
static generateDataMonthly(allData) {
const data = {};
const utcMoment = moment.utc();
const ids = {};
const ids6 = {};
const ids0 = {};
const order = [];

for (let i = 0; i < 12; i++) {
order.push(utcMoment.format('MMM YYYY'));
ids[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY');
if (i < 7) {
ids6[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY');
}
if (i === 0) {
ids0[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY');
}
utcMoment.subtract(1, 'months');
}

const DP = {};
data.DP = [];
let avg12monthDP = 0;
let avg6monthDP = 0;

let avg12 = 0;
let avg6 = 0;
for (const [key, value] of Object.entries(allData)) {
if (ids[key]) {
DP[ids[key]] = value;

if (!ids0[key]) {
avg12monthDP += DP[ids[key]];
avg12++;
}
if (ids6[key] && !ids0[key]) {
avg6monthDP += DP[ids[key]];
avg6++;
}
}
}

for (let i = 0; i < order.length; i++) {
data.DP.push((i < 9 ? '0' + (i + 1) : i + 1) + '. ' + order[i] + ': ' + ((DP[order[i]] || 0).toLocaleString()));
}
if (avg12) {
data['Last 12 months'] = Math.round(avg12monthDP / avg12).toLocaleString();
}
if (avg6) {
data['Last 6 months'] = Math.round(avg6monthDP / avg6).toLocaleString();
}

return data;
}

/**
* Returns a human readable name given application id.
* @param {Object} db - Database object, used for querying
* @param {function} done - Completion callback
* @returns {undefined} Returns nothing, only callback
**/
run(db, done) {
if (config.web.track !== "none") {
db.collection("members").find({global_admin: true}).toArray(function(err, members) {
if (config.web.track !== 'none') {
db.collection('members').find({global_admin: true}).toArray(async(err, members) => {
if (!err && members.length > 0) {
db.collection("server_stats_data_points").aggregate([
{
$group: {
_id: "$m",
e: { $sum: "$e"},
s: { $sum: "$s"}
}
let license = {};
if (drill) {
try {
license = await drill.loadLicense(undefined, db);
}
], { allowDiskUse: true }, async function(error, allData) {
if (!error) {
var data = {};
data.all = 0;
data.month3 = [];
var utcMoment = moment.utc();
var months = {};
for (let i = 0; i < 3; i++) {
months[utcMoment.format("YYYY:M")] = true;
utcMoment.subtract(1, 'months');
}
for (let i = 0; i < allData.length; i++) {
data.all += allData[i].e + allData[i].s;
if (months[allData[i]._id]) {
data.month3.push(allData[i]._id + " - " + (allData[i].e + allData[i].s));
}
}
data.avg = Math.round((data.all / allData.length) * 100) / 100;
var date = new Date();
var usersData = [];

await promisedLoadConfigs(db);

let domain = '';

try {
// try to extract hostname from full domain url
const urlObj = new URL(pluginManager.getConfig('api').domain);
domain = urlObj.hostname;
}
catch (_) {
// do nothing, domain from config will be used as is
}

usersData.push({
device_id: domain,
timestamp: Math.floor(date.getTime() / 1000),
hour: date.getHours(),
dow: date.getDay(),
user_details: JSON.stringify({
custom: {
dataPointsAll: data.all,
dataPointsMonthlyAvg: data.avg,
dataPointsLast3Months: data.month3
}
})
});
catch (error) {
log.e(error);
// do nothing, most likely there is no license
}
}

var formData = {
app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f",
requests: JSON.stringify(usersData)
};

request.post({
url: 'https://stats.count.ly/i/bulk',
body: formData
}, function(a) {
log.d('Done running stats job: %j', a);
done();
});
const options = {
monthlyBreakdown: true,
license_hosting: license.license_hosting,
};

serverStats.fetchDatapoints(db, {}, options, async(allData) => {
const dataSummary = StatsJob.generateDataSummary(allData);

if (tracker.isEnabled()) {
utcMoment = moment.utc();
data = {};
var ids = {};
var ids6 = {};
var ids0 = {};
var order = [];
var Countly = tracker.getSDK();
for (let i = 0; i < 12; i++) {
order.push(utcMoment.format("MMM YYYY"));
ids[utcMoment.format("YYYY:M")] = utcMoment.format("MMM YYYY");
if (i < 7) {
ids6[utcMoment.format("YYYY:M")] = utcMoment.format("MMM YYYY");
}
if (i === 0) {
ids0[utcMoment.format("YYYY:M")] = utcMoment.format("MMM YYYY");
}
utcMoment.subtract(1, 'months');
}

var DP = {};
data.DP = [];
var avg12monthDP = 0;
var avg6monthDP = 0;

var avg12 = 0;
var avg6 = 0;
for (let i = 0; i < allData.length; i++) {
if (ids[allData[i]._id]) {
var val = allData[i].e + allData[i].s;
DP[ids[allData[i]._id]] = val;
if (!ids0[allData[i]._id]) {
avg12monthDP += DP[ids[allData[i]._id]];
avg12++;
}
if (ids6[allData[i]._id] && !ids0[allData[i]._id]) {
avg6monthDP += DP[ids[allData[i]._id]];
avg6++;
}
}
}

for (let i = 0; i < order.length; i++) {
data.DP.push((i < 9 ? "0" + (i + 1) : i + 1) + ". " + order[i] + ": " + ((DP[order[i]] || 0).toLocaleString()));
}

if (avg12) {
data["Last 12 months"] = Math.round(avg12monthDP / avg12).toLocaleString();
}
if (avg6) {
data["Last 6 months"] = Math.round(avg6monthDP / avg6).toLocaleString();
}
Countly.user_details({
"custom": data
});

Countly.userData.save();
}
let date = new Date();
const usersData = [];

await promisedLoadConfigs(db);

let domain = '';

try {
// try to extract hostname from full domain url
const urlObj = new URL(pluginManager.getConfig('api').domain);
domain = urlObj.hostname;
}
else {
catch (_) {
// do nothing, domain from config will be used as is
}

usersData.push({
device_id: domain,
timestamp: Math.floor(date.getTime() / 1000),
hour: date.getHours(),
dow: date.getDay(),
user_details: JSON.stringify({
custom: {
dataPointsAll: dataSummary.all,
dataPointsMonthlyAvg: dataSummary.avg,
dataPointsLast3Months: dataSummary.month3,
},
}),
});

var formData = {
app_key: 'e70ec21cbe19e799472dfaee0adb9223516d238f',
requests: JSON.stringify(usersData)
};

request.post({
url: 'https://stats.count.ly/i/bulk',
json: formData
}, function(a) {
log.d('Done running stats job: %j', a);
done();
});

if (tracker.isEnabled()) {
const dataMonthly = StatsJob.generateDataMonthly(allData);

const Countly = tracker.getSDK();
Countly.user_details({
'custom': dataMonthly,
});

Countly.userData.save();
}
});
}
Expand All @@ -184,4 +219,4 @@ class StatsJob extends job.Job {
}
}

module.exports = StatsJob;
module.exports = StatsJob;
8 changes: 4 additions & 4 deletions plugins/server-stats/tests.js → plugins/server-stats/tests/api.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
var request = require('supertest');
var should = require('should');
var testUtils = require("../../test/testUtils");
const pluginManager = require('../pluginManager.js');
var statInternalEvents = require('../server-stats/api/parts/stats.js').internalEventsEnum;
var testUtils = require('../../../test/testUtils');
const pluginManager = require('../../pluginManager.js');
var statInternalEvents = require('../../server-stats/api/parts/stats.js').internalEventsEnum;
request = request.agent(testUtils.url);
const dataPointTimeout = 1000;

Expand Down Expand Up @@ -567,4 +567,4 @@ describe('Testing data points plugin', function() {
});
});
});
});
});
2 changes: 2 additions & 0 deletions plugins/server-stats/tests/index.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
require('./api.js');
require('./job.js');
Loading

0 comments on commit 28fa8a2

Please sign in to comment.