From f77380b01e9b57db0b5c782d6d512431229bcd79 Mon Sep 17 00:00:00 2001 From: Chunwai Li Date: Mon, 11 Nov 2024 11:36:20 -0600 Subject: [PATCH] feat: Refactor feature storages (#1241) Co-authored-by: ptang-nr --- .github/workflows/wdio-single-browser.yml | 5 + src/common/aggregate/aggregator.js | 54 ++--- src/common/aggregate/event-aggregator.js | 76 +++++++ src/common/harvest/harvest-scheduler.js | 2 +- src/common/harvest/harvest.js | 6 +- src/common/harvest/types.js | 1 - src/features/ajax/aggregate/chunk.js | 52 ----- src/features/ajax/aggregate/index.js | 127 ++++++----- .../generic_events/aggregate/index.js | 53 ++--- src/features/jserrors/aggregate/index.js | 98 ++------- src/features/logging/aggregate/index.js | 106 ++++----- src/features/metrics/aggregate/index.js | 10 +- .../page_view_timing/aggregate/index.js | 39 +--- .../session_replay/aggregate/index.js | 24 +-- .../session_replay/shared/recorder-events.js | 4 +- src/features/session_trace/aggregate/index.js | 137 ++++++------ .../session_trace/aggregate/trace/storage.js | 39 ++-- .../soft_navigations/aggregate/index.js | 33 +-- src/features/spa/aggregate/index.js | 39 ++-- src/features/utils/aggregate-base.js | 39 ++++ src/features/utils/event-buffer.js | 119 ++++------- src/features/utils/instrument-base.js | 6 +- src/loaders/features/features.js | 13 ++ tests/components/ajax/aggregate.test.js | 74 ++----- .../generic_events/aggregate/index.test.js | 20 +- tests/components/logging/aggregate.test.js | 14 +- tests/components/metrics/aggregate.test.js | 12 +- .../page_view_timing/aggregate.test.js | 12 +- .../components/page_view_timing/index.test.js | 12 +- .../session_trace/aggregate.test.js | 42 ++-- tests/components/setup-agent.js | 6 +- .../soft_navigations/aggregate.test.js | 49 ++--- tests/components/soft_navigations/api.test.js | 26 ++- tests/specs/session-trace/modes.e2e.js | 4 +- tests/specs/session-trace/trace-nodes.e2e.js | 4 +- .../unit/common/aggregate/aggregator.test.js | 26 +-- .../common/harvest/harvest-scheduler.test.js | 4 +- tests/unit/common/harvest/harvest.test.js | 4 +- .../page_view_timing/aggregate/index.test.js | 17 +- .../unit/features/utils/event-buffer.test.js | 202 ++++++------------ .../features/utils/instrument-base.test.js | 12 +- 41 files changed, 697 insertions(+), 925 deletions(-) create mode 100644 src/common/aggregate/event-aggregator.js delete mode 100644 src/features/ajax/aggregate/chunk.js diff --git a/.github/workflows/wdio-single-browser.yml b/.github/workflows/wdio-single-browser.yml index 343a84cc8..32780a107 100644 --- a/.github/workflows/wdio-single-browser.yml +++ b/.github/workflows/wdio-single-browser.yml @@ -16,6 +16,11 @@ on: required: false type: boolean default: false + concurrency: + description: 'The number of test runner threads to spawn for a run' + required: false + type: number + default: 10 workflow_call: inputs: browser-target: diff --git a/src/common/aggregate/aggregator.js b/src/common/aggregate/aggregator.js index 41bd187ce..d84316074 100644 --- a/src/common/aggregate/aggregator.js +++ b/src/common/aggregate/aggregator.js @@ -2,12 +2,8 @@ * Copyright 2020 New Relic Corporation. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ - -import { SharedContext } from '../context/shared-context' - -export class Aggregator extends SharedContext { - constructor (parent) { - super(parent) +export class Aggregator { + constructor () { this.aggregatedData = {} } @@ -16,13 +12,14 @@ export class Aggregator extends SharedContext { // metrics are the numeric values to be aggregated store (type, name, params, newMetrics, customParams) { - var bucket = this.getBucket(type, name, params, customParams) + var bucket = this.#getBucket(type, name, params, customParams) bucket.metrics = aggregateMetrics(newMetrics, bucket.metrics) return bucket } - merge (type, name, metrics, params, customParams) { - var bucket = this.getBucket(type, name, params, customParams) + merge (type, name, metrics, params, customParams, overwriteParams = false) { + var bucket = this.#getBucket(type, name, params, customParams) + if (overwriteParams) bucket.params = params // replace current params with incoming params obj if (!bucket.metrics) { bucket.metrics = metrics @@ -50,32 +47,13 @@ export class Aggregator extends SharedContext { } storeMetric (type, name, params, value) { - var bucket = this.getBucket(type, name, params) + var bucket = this.#getBucket(type, name, params) bucket.stats = updateMetric(value, bucket.stats) return bucket } - getBucket (type, name, params, customParams) { - if (!this.aggregatedData[type]) this.aggregatedData[type] = {} - var bucket = this.aggregatedData[type][name] - if (!bucket) { - bucket = this.aggregatedData[type][name] = { params: params || {} } - if (customParams) { - bucket.custom = customParams - } - } - return bucket - } - - get (type, name) { - // if name is passed, get a single bucket - if (name) return this.aggregatedData[type] && this.aggregatedData[type][name] - // else, get all buckets of that type - return this.aggregatedData[type] - } - - // Like get, but for many types and it deletes the retrieved content from the aggregatedData - take (types) { + // Get all listed types buckets and it deletes the retrieved content from the aggregatedData + take (types, deleteWhenRetrieved = true) { var results = {} var type = '' var hasData = false @@ -84,10 +62,22 @@ export class Aggregator extends SharedContext { results[type] = Object.values(this.aggregatedData[type] || {}) if (results[type].length) hasData = true - delete this.aggregatedData[type] + if (deleteWhenRetrieved) delete this.aggregatedData[type] } return hasData ? results : null } + + #getBucket (type, name, params, customParams) { + if (!this.aggregatedData[type]) this.aggregatedData[type] = {} + var bucket = this.aggregatedData[type][name] + if (!bucket) { + bucket = this.aggregatedData[type][name] = { params: params || {} } + if (customParams) { + bucket.custom = customParams + } + } + return bucket + } } function aggregateMetrics (newMetrics, oldMetrics) { diff --git a/src/common/aggregate/event-aggregator.js b/src/common/aggregate/event-aggregator.js new file mode 100644 index 000000000..942192e81 --- /dev/null +++ b/src/common/aggregate/event-aggregator.js @@ -0,0 +1,76 @@ +import { Aggregator } from './aggregator' + +/** + * An extension of the Aggregator class that provides an interface similar to that of EventBuffer class. + * This typecasting allow features that uses Aggregator as their event handler to share the same AggregateBase.events utilization by those features. + */ +export class EventAggregator { + #aggregator = new Aggregator() + #savedNamesToBuckets = {} + + isEmpty ({ aggregatorTypes }) { + if (!aggregatorTypes) return Object.keys(this.#aggregator.aggregatedData).length === 0 + return aggregatorTypes.every(type => !this.#aggregator.aggregatedData[type]) // no bucket exist for any of the types we're looking for + } + + add (type, name, params, newMetrics, customParams) { + // Do we need to track byte size here like EventBuffer? + this.#aggregator.store(type, name, params, newMetrics, customParams) + return true + } + + addMetric (type, name, params, value) { + this.#aggregator.storeMetric(type, name, params, value) + return true + } + + save ({ aggregatorTypes }) { + const key = aggregatorTypes.toString() // the stringified types serve as the key to each save call, e.g. ['err', 'ierr', 'xhr'] => 'err,ierr,xhr' + const backupAggregatedDataSubset = {} + aggregatorTypes.forEach(type => (backupAggregatedDataSubset[type] = this.#aggregator.aggregatedData[type])) // make a subset of the aggregatedData for each of the types we want to save + this.#savedNamesToBuckets[key] = backupAggregatedDataSubset + /* + { 'err,ierr,xhr': { + 'err': { + : { metrics: { count: 1, time, ... }, params: {}, custom: {} }, + : { metrics: { count: 1, ... }, ... } + }, + 'ierr': { ... }, + 'xhr': { ... } + } + } + */ + } + + get (opts) { + const aggregatorTypes = Array.isArray(opts) ? opts : opts.aggregatorTypes + return this.#aggregator.take(aggregatorTypes, false) + } + + clear ({ aggregatorTypes } = {}) { + if (!aggregatorTypes) { + this.#aggregator.aggregatedData = {} + return + } + aggregatorTypes.forEach(type => delete this.#aggregator.aggregatedData[type]) + } + + reloadSave ({ aggregatorTypes }) { + const key = aggregatorTypes.toString() + const backupAggregatedDataSubset = this.#savedNamesToBuckets[key] + // Grabs the previously stored subset and merge it back into aggregatedData. + aggregatorTypes.forEach(type => { + Object.keys(backupAggregatedDataSubset[type] || {}).forEach(name => { + const bucket = backupAggregatedDataSubset[type][name] + // The older aka saved params take effect over the newer one. This is especially important when merging back for a failed harvest retry if, for example, + // the first-ever occurrence of an error is in the retry: it contains the params.stack_trace whereas the newer or current bucket.params would not. + this.#aggregator.merge(type, name, bucket.metrics, bucket.params, bucket.custom, true) + }) + }) + } + + clearSave ({ aggregatorTypes }) { + const key = aggregatorTypes.toString() + delete this.#savedNamesToBuckets[key] + } +} diff --git a/src/common/harvest/harvest-scheduler.js b/src/common/harvest/harvest-scheduler.js index d570291b8..1cd13732e 100644 --- a/src/common/harvest/harvest-scheduler.js +++ b/src/common/harvest/harvest-scheduler.js @@ -98,7 +98,7 @@ export class HarvestScheduler extends SharedContext { let payload if (this.opts.getPayload) { - // Ajax & PVT & SR features provide a callback function to get data for harvesting + // Ajax, PVT, Softnav, Logging, SR & ST features provide a single callback function to get data for harvesting submitMethod = submitData.getSubmitMethod({ isFinalHarvest: opts?.unload }) if (!submitMethod) return false diff --git a/src/common/harvest/harvest.js b/src/common/harvest/harvest.js index 64a923e6d..1ddcb5881 100644 --- a/src/common/harvest/harvest.js +++ b/src/common/harvest/harvest.js @@ -106,11 +106,7 @@ export class Harvest extends SharedContext { const gzip = !!qs?.attributes?.includes('gzip') if (!gzip) { - if (endpoint === 'events') { - body = body.e - } else { - body = stringify(body) - } + if (endpoint !== 'events') body = stringify(body) // all features going to /events/ endpoint should already be serialized & stringified /** Warn --once per endpoint-- if the agent tries to send large payloads */ if (body.length > 750000 && (warnings[endpoint] = (warnings?.[endpoint] || 0) + 1) === 1) warn(28, endpoint) } diff --git a/src/common/harvest/types.js b/src/common/harvest/types.js index 76c6d99b2..3d6a06bcf 100644 --- a/src/common/harvest/types.js +++ b/src/common/harvest/types.js @@ -12,7 +12,6 @@ * @typedef {object} HarvestPayload * @property {object} qs Map of values that should be sent as part of the request query string. * @property {object} body Map of values that should be sent as the body of the request. - * @property {string} body.e Special case of body used for browser interactions. */ /** diff --git a/src/features/ajax/aggregate/chunk.js b/src/features/ajax/aggregate/chunk.js deleted file mode 100644 index 3c7486581..000000000 --- a/src/features/ajax/aggregate/chunk.js +++ /dev/null @@ -1,52 +0,0 @@ -import { nullable, numeric, getAddStringContext, addCustomAttributes } from '../../../common/serialize/bel-serializer' -import { getInfo } from '../../../common/config/info' -import { MAX_PAYLOAD_SIZE } from '../../../common/constants/agent-constants' - -export default class Chunk { - constructor (events, aggregateInstance) { - this.addString = getAddStringContext(aggregateInstance.agentIdentifier) // pass agentIdentifier here - this.events = events - this.payload = 'bel.7;' - - for (let i = 0; i < events.length; i++) { - const event = events[i] - const fields = [ - numeric(event.startTime), - numeric(event.endTime - event.startTime), - numeric(0), // callbackEnd - numeric(0), // no callbackDuration for non-SPA events - this.addString(event.method), - numeric(event.status), - this.addString(event.domain), - this.addString(event.path), - numeric(event.requestSize), - numeric(event.responseSize), - event.type === 'fetch' ? 1 : '', - this.addString(0), // nodeId - nullable(event.spanId, this.addString, true) + // guid - nullable(event.traceId, this.addString, true) + // traceId - nullable(event.spanTimestamp, numeric, false) // timestamp - ] - - let insert = '2,' - - // Since configuration objects (like info) are created new each time they are set, we have to grab the current pointer to the attr object here. - const jsAttributes = getInfo(aggregateInstance.agentIdentifier).jsAttributes - - // add custom attributes - // gql decorators are added as custom attributes to alleviate need for new BEL schema - const attrParts = addCustomAttributes({ ...(jsAttributes || {}), ...(event.gql || {}) }, this.addString) - fields.unshift(numeric(attrParts.length)) - - insert += fields.join(',') - if (attrParts && attrParts.length > 0) { - insert += ';' + attrParts.join(';') - } - if ((i + 1) < events.length) insert += ';' - - this.payload += insert - } - - this.tooBig = this.payload.length * 2 > MAX_PAYLOAD_SIZE - } -} diff --git a/src/features/ajax/aggregate/index.js b/src/features/ajax/aggregate/index.js index c1aa02350..885ff78e0 100644 --- a/src/features/ajax/aggregate/index.js +++ b/src/features/ajax/aggregate/index.js @@ -8,13 +8,11 @@ import { handle } from '../../../common/event-emitter/handle' import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler' import { setDenyList, shouldCollectEvent } from '../../../common/deny-list/deny-list' import { FEATURE_NAME } from '../constants' -import { FEATURE_NAMES } from '../../../loaders/features/features' +import { FEATURE_NAMES, FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { SUPPORTABILITY_METRIC_CHANNEL } from '../../metrics/constants' import { AggregateBase } from '../../utils/aggregate-base' import { parseGQL } from './gql' -import { getNREUMInitializedAgent } from '../../../common/window/nreum' -import Chunk from './chunk' -import { EventBuffer } from '../../utils/event-buffer' +import { nullable, numeric, getAddStringContext, addCustomAttributes } from '../../../common/serialize/bel-serializer' export class Aggregate extends AggregateBase { static featureName = FEATURE_NAME @@ -25,31 +23,30 @@ export class Aggregate extends AggregateBase { const harvestTimeSeconds = agentRef.init.ajax.harvestTimeSeconds || 10 setDenyList(agentRef.runtime.denyList) - this.ajaxEvents = new EventBuffer() - this.spaAjaxEvents = {} + this.underSpaEvents = {} const classThis = this // --- v Used by old spa feature this.ee.on('interactionDone', (interaction, wasSaved) => { - if (!this.spaAjaxEvents[interaction.id]?.hasData) return + if (!this.underSpaEvents[interaction.id]) return if (!wasSaved) { // if the ixn was saved, then its ajax reqs are part of the payload whereas if it was discarded, it should still be harvested in the ajax feature itself - this.ajaxEvents.merge(this.spaAjaxEvents[interaction.id]) + this.underSpaEvents[interaction.id].forEach((item) => this.events.add(item)) } - delete this.spaAjaxEvents[interaction.id] + delete this.underSpaEvents[interaction.id] }) // --- ^ // --- v Used by new soft nav - registerHandler('returnAjax', event => this.ajaxEvents.add(event), this.featureName, this.ee) + registerHandler('returnAjax', event => this.events.add(event), this.featureName, this.ee) // --- ^ registerHandler('xhr', function () { // the EE-drain system not only switches "this" but also passes a new EventContext with info. Should consider platform refactor to another system which passes a mutable context around separately and predictably to avoid problems like this. classThis.storeXhr(...arguments, this) // this switches the context back to the class instance while passing the NR context as an argument -- see "ctx" in storeXhr }, this.featureName, this.ee) this.waitForFlags(([])).then(() => { - const scheduler = new HarvestScheduler('events', { - onFinished: this.onEventsHarvestFinished.bind(this), - getPayload: this.prepareHarvest.bind(this) + const scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry), + getPayload: (options) => this.makeHarvestPayload(options.retry) }, this) scheduler.startTimer(harvestTimeSeconds) this.drain() @@ -69,10 +66,11 @@ export class Aggregate extends AggregateBase { const shouldCollect = shouldCollectEvent(params) const shouldOmitAjaxMetrics = this.agentRef.init.feature_flags?.includes('ajax_metrics_deny_list') + const jserrorsInUse = Boolean(this.agentRef.features?.[FEATURE_NAMES.jserrors]) - // store for timeslice metric (harvested by jserrors feature) - if (shouldCollect || !shouldOmitAjaxMetrics) { - this.agentRef.sharedAggregator.store('xhr', hash, params, metrics) + // Report ajax timeslice metric (to be harvested by jserrors feature, but only if it's running). + if (jserrorsInUse && (shouldCollect || !shouldOmitAjaxMetrics)) { + this.agentRef.sharedAggregator.add('xhr', hash, params, metrics) } if (!shouldCollect) { @@ -119,66 +117,61 @@ export class Aggregate extends AggregateBase { }) if (event.gql) handle(SUPPORTABILITY_METRIC_CHANNEL, ['Ajax/Events/GraphQL/Bytes-Added', stringify(event.gql).length], undefined, FEATURE_NAMES.metrics, this.ee) - const softNavInUse = Boolean(getNREUMInitializedAgent(this.agentIdentifier)?.features?.[FEATURE_NAMES.softNav]) + const softNavInUse = Boolean(this.agentRef.features?.[FEATURE_NAMES.softNav]) if (softNavInUse) { // For newer soft nav (when running), pass the event to it for evaluation -- either part of an interaction or is given back handle('ajax', [event], undefined, FEATURE_NAMES.softNav, this.ee) } else if (ctx.spaNode) { // For old spa (when running), if the ajax happened inside an interaction, hold it until the interaction finishes const interactionId = ctx.spaNode.interaction.id - this.spaAjaxEvents[interactionId] ??= new EventBuffer() - this.spaAjaxEvents[interactionId].add(event) + this.underSpaEvents[interactionId] ??= [] + this.underSpaEvents[interactionId].push(event) } else { - this.ajaxEvents.add(event) + this.events.add(event) } } - prepareHarvest (options) { - options = options || {} - if (this.ajaxEvents.buffer.length === 0) return null - - const payload = this.#getPayload(this.ajaxEvents.buffer) - const payloadObjs = [] - - for (let i = 0; i < payload.length; i++) payloadObjs.push({ body: { e: payload[i] } }) - - if (options.retry) this.ajaxEvents.hold() - else this.ajaxEvents.clear() - - return payloadObjs - } - - onEventsHarvestFinished (result) { - if (result.retry && this.ajaxEvents.held.hasData) this.ajaxEvents.unhold() - else this.ajaxEvents.held.clear() - } - - #getPayload (events, numberOfChunks) { - numberOfChunks = numberOfChunks || 1 - const payload = [] - const chunkSize = events.length / numberOfChunks - const eventChunks = splitChunks.call(this, events, chunkSize) - let tooBig = false - for (let i = 0; i < eventChunks.length; i++) { - const currentChunk = eventChunks[i] - if (currentChunk.tooBig) { - if (currentChunk.events.length > 1) { - tooBig = true - break // if the payload is too big BUT is made of more than 1 event, we can split it down again - } - // Otherwise, if it consists of one sole event, we do not send it (discarded) since we cannot break it apart any further. - } else { - payload.push(currentChunk.payload) - } - } - // Check if the current payload string is too big, if so then run getPayload again with more buckets. - return tooBig ? this.#getPayload(events, ++numberOfChunks) : payload - - function splitChunks (arr, chunkSize) { - chunkSize = chunkSize || arr.length - const chunks = [] - for (let i = 0, len = arr.length; i < len; i += chunkSize) { - chunks.push(new Chunk(arr.slice(i, i + chunkSize), this)) + serializer (eventBuffer) { + const addString = getAddStringContext(this.agentIdentifier) + let payload = 'bel.7;' + + for (let i = 0; i < eventBuffer.length; i++) { + const event = eventBuffer[i] + const fields = [ + numeric(event.startTime), + numeric(event.endTime - event.startTime), + numeric(0), // callbackEnd + numeric(0), // no callbackDuration for non-SPA events + addString(event.method), + numeric(event.status), + addString(event.domain), + addString(event.path), + numeric(event.requestSize), + numeric(event.responseSize), + event.type === 'fetch' ? 1 : '', + addString(0), // nodeId + nullable(event.spanId, addString, true) + // guid + nullable(event.traceId, addString, true) + // traceId + nullable(event.spanTimestamp, numeric, false) // timestamp + ] + + let insert = '2,' + + // Since configuration objects (like info) are created new each time they are set, we have to grab the current pointer to the attr object here. + const jsAttributes = this.agentRef.info.jsAttributes + + // add custom attributes + // gql decorators are added as custom attributes to alleviate need for new BEL schema + const attrParts = addCustomAttributes({ ...(jsAttributes || {}), ...(event.gql || {}) }, addString) + fields.unshift(numeric(attrParts.length)) + + insert += fields.join(',') + if (attrParts && attrParts.length > 0) { + insert += ';' + attrParts.join(';') } - return chunks + if ((i + 1) < eventBuffer.length) insert += ';' + + payload += insert } + + return payload } } diff --git a/src/features/generic_events/aggregate/index.js b/src/features/generic_events/aggregate/index.js index 8b4e282e4..2bdf2c82a 100644 --- a/src/features/generic_events/aggregate/index.js +++ b/src/features/generic_events/aggregate/index.js @@ -12,9 +12,9 @@ import { warn } from '../../../common/util/console' import { now } from '../../../common/timing/now' import { registerHandler } from '../../../common/event-emitter/register-handler' import { SUPPORTABILITY_METRIC_CHANNEL } from '../../metrics/constants' -import { EventBuffer } from '../../utils/event-buffer' import { applyFnToProps } from '../../../common/util/traverse' import { IDEAL_PAYLOAD_SIZE } from '../../../common/constants/agent-constants' +import { FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { UserActionsAggregator } from './user-actions/user-actions-aggregator' import { isIFrameWindow } from '../../../common/dom/iframe' @@ -27,7 +27,6 @@ export class Aggregate extends AggregateBase { this.harvestTimeSeconds = agentRef.init.generic_events.harvestTimeSeconds this.referrerUrl = (isBrowserScope && document.referrer) ? cleanURL(document.referrer) : undefined - this.events = new EventBuffer() this.waitForFlags(['ins']).then(([ins]) => { if (!ins) { @@ -36,8 +35,6 @@ export class Aggregate extends AggregateBase { return } - const preHarvestMethods = [] - if (agentRef.init.page_action.enabled) { registerHandler('api-addPageAction', (timestamp, name, attributes) => { this.addEvent({ @@ -55,10 +52,11 @@ export class Aggregate extends AggregateBase { }, this.featureName, this.ee) } + let addUserAction if (isBrowserScope && agentRef.init.user_actions.enabled) { this.userActionAggregator = new UserActionsAggregator() - this.addUserAction = (aggregatedUserAction) => { + addUserAction = (aggregatedUserAction) => { try { /** The aggregator process only returns an event when it is "done" aggregating - * so we still need to validate that an event was given to this method before we try to add */ @@ -87,14 +85,8 @@ export class Aggregate extends AggregateBase { registerHandler('ua', (evt) => { /** the processor will return the previously aggregated event if it has been completed by processing the current event */ - this.addUserAction(this.userActionAggregator.process(evt)) + addUserAction(this.userActionAggregator.process(evt)) }, this.featureName, this.ee) - - preHarvestMethods.push((options = {}) => { - /** send whatever UserActions have been aggregated up to this point - * if we are in a final harvest. By accessing the aggregationEvent, the aggregation is then force-cleared */ - if (options.isFinalHarvest) this.addUserAction(this.userActionAggregator.aggregationEvent) - }) } /** @@ -132,11 +124,11 @@ export class Aggregate extends AggregateBase { } } - this.harvestScheduler = new HarvestScheduler('ins', { onFinished: (...args) => this.onHarvestFinished(...args) }, this) - this.harvestScheduler.harvest.on('ins', (...args) => { - preHarvestMethods.forEach(fn => fn(...args)) - return this.onHarvestStarted(...args) - }) + this.harvestScheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry), + onUnload: () => addUserAction?.(this.userActionAggregator.aggregationEvent) + }, this) + this.harvestScheduler.harvest.on(FEATURE_TO_ENDPOINT[this.featureName], (options) => this.makeHarvestPayload(options.retry)) this.harvestScheduler.startTimer(this.harvestTimeSeconds, 0) this.drain() @@ -190,34 +182,17 @@ export class Aggregate extends AggregateBase { this.checkEventLimits() } - onHarvestStarted (options) { - const { userAttributes, atts } = this.agentRef.info - if (!this.events.hasData) return - var payload = ({ - qs: { - ua: userAttributes, - at: atts - }, - body: applyFnToProps( - { ins: this.events.buffer }, - this.obfuscator.obfuscateString.bind(this.obfuscator), 'string' - ) - }) - - if (options.retry) this.events.hold() - else this.events.clear() - - return payload + serializer (eventBuffer) { + return applyFnToProps({ ins: eventBuffer }, this.obfuscator.obfuscateString.bind(this.obfuscator), 'string') } - onHarvestFinished (result) { - if (result && result?.sent && result?.retry && this.events.held.hasData) this.events.unhold() - else this.events.held.clear() + queryStringsBuilder () { + return { ua: this.agentRef.info.userAttributes, at: this.agentRef.info.atts } } checkEventLimits () { // check if we've reached any harvest limits... - if (this.events.bytes > IDEAL_PAYLOAD_SIZE) { + if (this.events.byteSize() > IDEAL_PAYLOAD_SIZE) { this.ee.emit(SUPPORTABILITY_METRIC_CHANNEL, ['GenericEvents/Harvest/Max/Seen']) this.harvestScheduler.runHarvest() } diff --git a/src/features/jserrors/aggregate/index.js b/src/features/jserrors/aggregate/index.js index 25383d03b..aabbac7a0 100644 --- a/src/features/jserrors/aggregate/index.js +++ b/src/features/jserrors/aggregate/index.js @@ -15,9 +15,8 @@ import { handle } from '../../../common/event-emitter/handle' import { globalScope } from '../../../common/constants/runtime' import { FEATURE_NAME } from '../constants' -import { FEATURE_NAMES } from '../../../loaders/features/features' +import { FEATURE_NAMES, FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { AggregateBase } from '../../utils/aggregate-base' -import { getNREUMInitializedAgent } from '../../../common/window/nreum' import { now } from '../../../common/timing/now' import { applyFnToProps } from '../../../common/util/traverse' import { evaluateInternalError } from './internal-errors' @@ -36,7 +35,6 @@ export class Aggregate extends AggregateBase { this.observedAt = {} this.pageviewReported = {} this.bufferedErrorsUnderSpa = {} - this.currentBody = undefined this.errorOnPage = false // this will need to change to match whatever ee we use in the instrument @@ -48,12 +46,15 @@ export class Aggregate extends AggregateBase { this.onSoftNavNotification(interactionId, wasFinished, softNavAttrs), this.featureName, this.ee) // when an ixn is done or cancelled const harvestTimeSeconds = agentRef.init.jserrors.harvestTimeSeconds || 10 + const aggregatorTypes = ['err', 'ierr', 'xhr'] // the types in EventAggregator this feature cares about // 0 == off, 1 == on this.waitForFlags(['err']).then(([errFlag]) => { if (errFlag) { - const scheduler = new HarvestScheduler('jserrors', { onFinished: (...args) => this.onHarvestFinished(...args) }, this) - scheduler.harvest.on('jserrors', (...args) => this.onHarvestStarted(...args)) + const scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry, { aggregatorTypes }) + }, this) + scheduler.harvest.on(FEATURE_TO_ENDPOINT[this.featureName], (options) => this.makeHarvestPayload(options.retry, { aggregatorTypes })) scheduler.startTimer(harvestTimeSeconds) this.drain() } else { @@ -63,58 +64,24 @@ export class Aggregate extends AggregateBase { }) } - onHarvestStarted (options) { - // this gets rid of dependency in AJAX module - var body = applyFnToProps( - this.agentRef.sharedAggregator.take(['err', 'ierr', 'xhr']), - this.obfuscator.obfuscateString.bind(this.obfuscator), 'string' - ) - - if (options.retry) { - this.currentBody = body - } - - var payload = { body, qs: {} } - var releaseIds = stringify(this.agentRef.runtime.releaseIds) + serializer (aggregatorTypeToBucketsMap) { + return applyFnToProps(aggregatorTypeToBucketsMap, this.obfuscator.obfuscateString.bind(this.obfuscator), 'string') + } - if (releaseIds !== '{}') { - payload.qs.ri = releaseIds - } + queryStringsBuilder (aggregatorTakeReturnedData) { + const qs = {} + const releaseIds = stringify(this.agentRef.runtime.releaseIds) + if (releaseIds !== '{}') qs.ri = releaseIds - if (body && body.err && body.err.length) { - this.#runCrossFeatureChecks(body.err) + if (aggregatorTakeReturnedData?.err?.length) { if (!this.errorOnPage) { - payload.qs.pve = '1' + qs.pve = '1' this.errorOnPage = true } + // For assurance, erase any `hasReplay` flag from all errors if replay is not recording, not-yet imported, or not running at all. + if (!this.agentRef.features?.[FEATURE_NAMES.sessionReplay]?.featAggregate?.replayIsActive()) aggregatorTakeReturnedData.err.forEach(error => delete error.params.hasReplay) } - - return payload - } - - onHarvestFinished (result) { - if (result.retry && this.currentBody) { - Object.entries(this.currentBody || {}).forEach(([key, value]) => { - for (var i = 0; i < value.length; i++) { - var bucket = value[i] - var name = this.getBucketName(key, bucket.params, bucket.custom) - this.agentRef.sharedAggregator.merge(key, name, bucket.metrics, bucket.params, bucket.custom) - } - }) - this.currentBody = null - } - } - - nameHash (params) { - return stringHashCode(`${params.exceptionClass}_${params.message}_${params.stack_trace || params.browser_stack_hash}`) - } - - getBucketName (objType, params, customParams) { - if (objType === 'xhr') { - return stringHashCode(stringify(params)) + ':' + stringHashCode(stringify(customParams)) - } - - return this.nameHash(params) + ':' + stringHashCode(stringify(customParams)) + return qs } /** @@ -230,7 +197,7 @@ export class Aggregate extends AggregateBase { params._interactionNodeId = err.__newrelic[this.agentIdentifier].interactionNodeId } - const softNavInUse = Boolean(getNREUMInitializedAgent(this.agentIdentifier)?.features[FEATURE_NAMES.softNav]) + const softNavInUse = Boolean(this.agentRef.features?.[FEATURE_NAMES.softNav]) // Note: the following are subject to potential race cond wherein if the other feature aren't fully initialized, it'll be treated as there being no associated interaction. // They each will also tack on their respective properties to the params object as part of the decision flow. if (softNavInUse) handle('jserror', [params, time], undefined, FEATURE_NAMES.softNav, this.ee) @@ -267,7 +234,7 @@ export class Aggregate extends AggregateBase { const jsAttributesHash = stringHashCode(stringify(allCustomAttrs)) const aggregateHash = bucketHash + ':' + jsAttributesHash - this.agentRef.sharedAggregator.store(type, aggregateHash, params, newMetrics, allCustomAttrs) + this.events.add(type, aggregateHash, params, newMetrics, allCustomAttrs) function setCustom (key, val) { allCustomAttrs[key] = (val && typeof val === 'object' ? stringify(val) : val) @@ -297,7 +264,7 @@ export class Aggregate extends AggregateBase { var jsAttributesHash = stringHashCode(stringify(allCustomAttrs)) var aggregateHash = hash + ':' + jsAttributesHash - this.agentRef.sharedAggregator.store(item[0], aggregateHash, params, item[3], allCustomAttrs) + this.events.add(item[0], aggregateHash, params, item[3], allCustomAttrs) function setCustom ([key, val]) { allCustomAttrs[key] = (val && typeof val === 'object' ? stringify(val) : val) @@ -314,27 +281,4 @@ export class Aggregate extends AggregateBase { ) delete this.bufferedErrorsUnderSpa[interactionId] // wipe the list of jserrors so they aren't duplicated by another call to the same id } - - /** - * Dispatches a cross-feature communication event to allow other - * features to provide flags and data that can be used to mutation - * to the payload and to allow features to know about a feature - * harvest happening. - * @param {any[]} errors Array of errors from the payload body - */ - #runCrossFeatureChecks (errors) { - const errorHashes = errors.map(error => error.params.stackHash) - const crossFeatureData = { - errorHashes - } - this.ee.emit(`cfc.${this.featureName}`, [crossFeatureData]) - - let hasReplayFlag = errors.find(err => err.params.hasReplay) - if (hasReplayFlag && !crossFeatureData.hasReplay) { - // Some errors have `hasReplay` and a replay is not being recorded - errors.forEach(error => { - delete error.params.hasReplay - }) - } - } } diff --git a/src/features/logging/aggregate/index.js b/src/features/logging/aggregate/index.js index dd60f7408..09678149a 100644 --- a/src/features/logging/aggregate/index.js +++ b/src/features/logging/aggregate/index.js @@ -10,23 +10,19 @@ import { Log } from '../shared/log' import { isValidLogLevel } from '../shared/utils' import { applyFnToProps } from '../../../common/util/traverse' import { MAX_PAYLOAD_SIZE } from '../../../common/constants/agent-constants' -import { EventBuffer } from '../../utils/event-buffer' +import { FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' export class Aggregate extends AggregateBase { static featureName = FEATURE_NAME constructor (agentRef) { super(agentRef, FEATURE_NAME) - - /** held logs before sending */ - this.bufferedLogs = new EventBuffer() - this.harvestTimeSeconds = agentRef.init.logging.harvestTimeSeconds this.waitForFlags([]).then(() => { - this.scheduler = new HarvestScheduler('browser/logs', { - onFinished: this.onHarvestFinished.bind(this), + this.scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry), retryDelay: this.harvestTimeSeconds, - getPayload: this.prepareHarvest.bind(this), + getPayload: (options) => this.makeHarvestPayload(options.retry), raw: true }, this) /** emitted by instrument class (wrapped loggers) or the api methods directly */ @@ -69,65 +65,55 @@ export class Aggregate extends AggregateBase { ) const logBytes = log.message.length + stringify(log.attributes).length + log.level.length + 10 // timestamp == 10 chars - if (!this.bufferedLogs.canMerge(logBytes)) { - if (this.bufferedLogs.hasData) { - handle(SUPPORTABILITY_METRIC_CHANNEL, ['Logging/Harvest/Early/Seen', this.bufferedLogs.bytes + logBytes]) - this.scheduler.runHarvest({}) - if (logBytes < MAX_PAYLOAD_SIZE) this.bufferedLogs.add(log) - } else { - handle(SUPPORTABILITY_METRIC_CHANNEL, ['Logging/Harvest/Failed/Seen', logBytes]) - warn(31, log.message.slice(0, 25) + '...') - } + const failToHarvestMessage = 'Logging/Harvest/Failed/Seen' + if (logBytes > MAX_PAYLOAD_SIZE) { // cannot possibly send this, even with an empty buffer + handle(SUPPORTABILITY_METRIC_CHANNEL, [failToHarvestMessage, logBytes]) + warn(31, log.message.slice(0, 25) + '...') return } - this.bufferedLogs.add(log) - } - - prepareHarvest (options = {}) { - if (this.blocked || !this.bufferedLogs.hasData) return - /** These attributes are evaluated and dropped at ingest processing time and do not get stored on NRDB */ - const unbilledAttributes = { - 'instrumentation.provider': 'browser', - 'instrumentation.version': this.agentRef.runtime.version, - 'instrumentation.name': this.agentRef.runtime.loaderType - } - /** see https://source.datanerd.us/agents/rum-specs/blob/main/browser/Log for logging spec */ - const payload = { - qs: { - browser_monitoring_key: this.agentRef.info.licenseKey - }, - body: [{ - common: { - /** Attributes in the `common` section are added to `all` logs generated in the payload */ - attributes: { - 'entity.guid': this.agentRef.runtime.appMetadata?.agents?.[0]?.entityGuid, // browser entity guid as provided from RUM response - session: this.agentRef.runtime.session?.state.value || '0', // The session ID that we generate and keep across page loads - hasReplay: this.agentRef.runtime.session?.state.sessionReplayMode === 1, // True if a session replay recording is running - hasTrace: this.agentRef.runtime.session?.state.sessionTraceMode === 1, // True if a session trace recording is running - ptid: this.agentRef.runtime.ptid, // page trace id - appId: this.agentRef.info.applicationID, // Application ID from info object, - standalone: Boolean(this.agentRef.info.sa), // copy paste (true) vs APM (false) - agentVersion: this.agentRef.runtime.version, // browser agent version - ...unbilledAttributes - } - }, - /** logs section contains individual unique log entries */ - logs: applyFnToProps( - this.bufferedLogs.buffer, - this.obfuscator.obfuscateString.bind(this.obfuscator), 'string' - ) - }] + if (this.events.wouldExceedMaxSize(logBytes)) { + handle(SUPPORTABILITY_METRIC_CHANNEL, ['Logging/Harvest/Early/Seen', this.events.bytes + logBytes]) + this.scheduler.runHarvest() // force a harvest to try adding again } - if (options.retry) this.bufferedLogs.hold() - else this.bufferedLogs.clear() + if (!this.events.add(log)) { // still failed after a harvest attempt despite not being too large would mean harvest failed with options.retry + handle(SUPPORTABILITY_METRIC_CHANNEL, [failToHarvestMessage, logBytes]) + warn(31, log.message.slice(0, 25) + '...') + } + } - return payload + serializer (eventBuffer) { + const sessionEntity = this.agentRef.runtime.session + return [{ + common: { + /** Attributes in the `common` section are added to `all` logs generated in the payload */ + attributes: { + 'entity.guid': this.agentRef.runtime.appMetadata?.agents?.[0]?.entityGuid, // browser entity guid as provided from RUM response + ...(sessionEntity && { + session: sessionEntity.state.value || '0', // The session ID that we generate and keep across page loads + hasReplay: sessionEntity.state.sessionReplayMode === 1, // True if a session replay recording is running + hasTrace: sessionEntity.state.sessionTraceMode === 1 // True if a session trace recording is running + }), + ptid: this.agentRef.runtime.ptid, // page trace id + appId: this.agentRef.info.applicationID, // Application ID from info object, + standalone: Boolean(this.agentRef.info.sa), // copy paste (true) vs APM (false) + agentVersion: this.agentRef.runtime.version, // browser agent version + // The following 3 attributes are evaluated and dropped at ingest processing time and do not get stored on NRDB: + 'instrumentation.provider': 'browser', + 'instrumentation.version': this.agentRef.runtime.version, + 'instrumentation.name': this.agentRef.runtime.loaderType + } + }, + /** logs section contains individual unique log entries */ + logs: applyFnToProps( + eventBuffer, + this.obfuscator.obfuscateString.bind(this.obfuscator), 'string' + ) + }] } - onHarvestFinished (result) { - if (result.retry) this.bufferedLogs.unhold() - else this.bufferedLogs.held.clear() + queryStringsBuilder () { + return { browser_monitoring_key: this.agentRef.info.licenseKey } } } diff --git a/src/features/metrics/aggregate/index.js b/src/features/metrics/aggregate/index.js index 9796ec892..1f0647e1d 100644 --- a/src/features/metrics/aggregate/index.js +++ b/src/features/metrics/aggregate/index.js @@ -7,6 +7,7 @@ import { onDOMContentLoaded } from '../../../common/window/load' import { windowAddEventListener } from '../../../common/event-listener/event-listener-opts' import { isBrowserScope, isWorkerScope } from '../../../common/constants/runtime' import { AggregateBase } from '../../utils/aggregate-base' +import { FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { isIFrameWindow } from '../../../common/dom/iframe' // import { WEBSOCKET_TAG } from '../../../common/wrap/wrap-websocket' // import { handleWebsocketEvents } from './websocket-detection' @@ -15,13 +16,14 @@ export class Aggregate extends AggregateBase { static featureName = FEATURE_NAME constructor (agentRef) { super(agentRef, FEATURE_NAME) + const aggregatorTypes = ['cm', 'sm'] // the types in EventAggregator this feature cares about this.waitForFlags(['err']).then(([errFlag]) => { if (errFlag) { // *cli, Mar 23 - Per NR-94597, this feature should only harvest ONCE at the (potential) EoL time of the page. - const scheduler = new HarvestScheduler('jserrors', { onUnload: () => this.unload() }, this) + const scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { onUnload: () => this.unload() }, this) // this is needed to ensure EoL is "on" and sent - scheduler.harvest.on('jserrors', () => ({ body: this.agentRef.sharedAggregator.take(['cm', 'sm']) })) + scheduler.harvest.on(FEATURE_TO_ENDPOINT[this.featureName], () => this.makeHarvestPayload(undefined, { aggregatorTypes })) this.drain() } else { this.blocked = true // if rum response determines that customer lacks entitlements for spa endpoint, this feature shouldn't harvest @@ -41,14 +43,14 @@ export class Aggregate extends AggregateBase { if (this.blocked) return const type = SUPPORTABILITY_METRIC const params = { name } - this.agentRef.sharedAggregator.storeMetric(type, name, params, value) + this.events.addMetric(type, name, params, value) } storeEventMetrics (name, metrics) { if (this.blocked) return const type = CUSTOM_METRIC const params = { name } - this.agentRef.sharedAggregator.store(type, name, params, metrics) + this.events.add(type, name, params, metrics) } singleChecks () { diff --git a/src/features/page_view_timing/aggregate/index.js b/src/features/page_view_timing/aggregate/index.js index 936e99410..694051130 100644 --- a/src/features/page_view_timing/aggregate/index.js +++ b/src/features/page_view_timing/aggregate/index.js @@ -8,7 +8,7 @@ import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler' import { registerHandler } from '../../../common/event-emitter/register-handler' import { handle } from '../../../common/event-emitter/handle' import { FEATURE_NAME } from '../constants' -import { FEATURE_NAMES } from '../../../loaders/features/features' +import { FEATURE_NAMES, FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { AggregateBase } from '../../utils/aggregate-base' import { cumulativeLayoutShift } from '../../../common/vitals/cumulative-layout-shift' import { firstContentfulPaint } from '../../../common/vitals/first-contentful-paint' @@ -19,7 +19,6 @@ import { largestContentfulPaint } from '../../../common/vitals/largest-contentfu import { timeToFirstByte } from '../../../common/vitals/time-to-first-byte' import { subscribeToVisibilityChange } from '../../../common/window/page-visibility' import { VITAL_NAMES } from '../../../common/vitals/constants' -import { EventBuffer } from '../../utils/event-buffer' export class Aggregate extends AggregateBase { static featureName = FEATURE_NAME @@ -30,8 +29,6 @@ export class Aggregate extends AggregateBase { constructor (agentRef) { super(agentRef, FEATURE_NAME) - - this.timings = new EventBuffer() this.curSessEndRecorded = false registerHandler('docHidden', msTimestamp => this.endCurrentSession(msTimestamp), this.featureName, this.ee) @@ -60,9 +57,9 @@ export class Aggregate extends AggregateBase { this.addTiming(name, value * 1000, attrs) }, true) // CLS node should only reports on vis change rather than on every change - const scheduler = new HarvestScheduler('events', { - onFinished: (...args) => this.onHarvestFinished(...args), - getPayload: (...args) => this.prepareHarvest(...args) + const scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry), + getPayload: (options) => this.makeHarvestPayload(options.retry) }, this) scheduler.startTimer(harvestTimeSeconds) @@ -97,7 +94,7 @@ export class Aggregate extends AggregateBase { attrs.cls = cumulativeLayoutShift.current.value } - this.timings.add({ + this.events.add({ name, value, attrs @@ -106,11 +103,6 @@ export class Aggregate extends AggregateBase { handle('pvtAdded', [name, value, attrs], undefined, FEATURE_NAMES.sessionTrace, this.ee) } - onHarvestFinished (result) { - if (result.retry && this.timings.held.hasData) this.timings.unhold() - else this.timings.held.clear() - } - appendGlobalCustomAttributes (timing) { var timingAttributes = timing.attrs || {} @@ -123,27 +115,14 @@ export class Aggregate extends AggregateBase { }) } - // serialize and return current timing data, clear and save current data for retry - prepareHarvest (options) { - if (!this.timings.hasData) return - - var payload = this.getPayload(this.timings.buffer) - if (options.retry) this.timings.hold() - else this.timings.clear() - - return { - body: { e: payload } - } - } - // serialize array of timing data - getPayload (data) { + serializer (eventBuffer) { var addString = getAddStringContext(this.agentIdentifier) var payload = 'bel.6;' - for (var i = 0; i < data.length; i++) { - var timing = data[i] + for (var i = 0; i < eventBuffer.length; i++) { + var timing = eventBuffer[i] payload += 'e,' payload += addString(timing.name) + ',' @@ -156,7 +135,7 @@ export class Aggregate extends AggregateBase { payload += numeric(attrParts.length) + ';' + attrParts.join(';') } - if ((i + 1) < data.length) payload += ';' + if ((i + 1) < eventBuffer.length) payload += ';' } return payload diff --git a/src/features/session_replay/aggregate/index.js b/src/features/session_replay/aggregate/index.js index afce5b642..c824a5390 100644 --- a/src/features/session_replay/aggregate/index.js +++ b/src/features/session_replay/aggregate/index.js @@ -16,7 +16,7 @@ import { warn } from '../../../common/util/console' import { globalScope } from '../../../common/constants/runtime' import { SUPPORTABILITY_METRIC_CHANNEL } from '../../metrics/constants' import { handle } from '../../../common/event-emitter/handle' -import { FEATURE_NAMES } from '../../../loaders/features/features' +import { FEATURE_NAMES, FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { RRWEB_VERSION } from '../../../common/constants/env' import { MODE, SESSION_EVENTS, SESSION_EVENT_TYPES } from '../../../common/session/constants' import { stringify } from '../../../common/util/stringify' @@ -54,14 +54,6 @@ export class Aggregate extends AggregateBase { handle(SUPPORTABILITY_METRIC_CHANNEL, ['Config/SessionReplay/Enabled'], undefined, FEATURE_NAMES.metrics, this.ee) - this.ee.on(`cfc.${FEATURE_NAMES.jserrors}`, (crossFeatureData) => { - crossFeatureData.hasReplay = !!(this.scheduler?.started && - this.recorder && - this.mode === MODE.FULL && - !this.blocked && - this.entitled) - }) - // The SessionEntity class can emit a message indicating the session was cleared and reset (expiry, inactivity). This feature must abort and never resume if that occurs. this.ee.on(SESSION_EVENTS.RESET, () => { this.abort(ABORT_REASONS.RESET) @@ -85,10 +77,10 @@ export class Aggregate extends AggregateBase { }) // Bespoke logic for blobs endpoint. - this.scheduler = new HarvestScheduler('browser/blobs', { - onFinished: this.onHarvestFinished.bind(this), + this.scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result), retryDelay: this.harvestTimeSeconds, - getPayload: this.prepareHarvest.bind(this), + getPayload: ({ retry, ...opts }) => this.makeHarvestPayload(retry, opts), raw: true }, this) @@ -134,6 +126,10 @@ export class Aggregate extends AggregateBase { handle(SUPPORTABILITY_METRIC_CHANNEL, ['Config/SessionReplay/ErrorSamplingRate/Value', error_sampling_rate], undefined, FEATURE_NAMES.metrics, this.ee) } + replayIsActive () { + return Boolean(this.scheduler?.started && this.recorder && this.mode === MODE.FULL && !this.blocked && this.entitled) + } + handleError (e) { if (this.recorder) this.recorder.currentBufferTarget.hasError = true // run once @@ -236,7 +232,7 @@ export class Aggregate extends AggregateBase { } } - prepareHarvest ({ opts } = {}) { + makeHarvestPayload (shouldRetryOnFail, opts) { if (!this.recorder || !this.timeKeeper?.ready || !this.recorder.hasSeenSnapshot) return const recorderEvents = this.recorder.getEvents() // get the event type and use that to trigger another harvest if needed @@ -365,7 +361,7 @@ export class Aggregate extends AggregateBase { } } - onHarvestFinished (result) { + postHarvestCleanup (result) { // The mutual decision for now is to stop recording and clear buffers if ingest is experiencing 429 rate limiting if (result.status === 429) { this.abort(ABORT_REASONS.TOO_MANY) diff --git a/src/features/session_replay/shared/recorder-events.js b/src/features/session_replay/shared/recorder-events.js index 664dd8f1f..6f69c1e20 100644 --- a/src/features/session_replay/shared/recorder-events.js +++ b/src/features/session_replay/shared/recorder-events.js @@ -24,11 +24,11 @@ export class RecorderEvents { } get events () { - return this.#events.buffer + return this.#events.get() } /** A value which increments with every new mutation node reported. Resets after a harvest is sent */ get payloadBytesEstimation () { - return this.#events.bytes + return this.#events.byteSize() } } diff --git a/src/features/session_trace/aggregate/index.js b/src/features/session_trace/aggregate/index.js index 9d5875628..0b8f67fc7 100644 --- a/src/features/session_trace/aggregate/index.js +++ b/src/features/session_trace/aggregate/index.js @@ -7,6 +7,7 @@ import { obj as encodeObj } from '../../../common/url/encode' import { globalScope } from '../../../common/constants/runtime' import { MODE, SESSION_EVENTS } from '../../../common/session/constants' import { applyFnToProps } from '../../../common/util/traverse' +import { FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { cleanURL } from '../../../common/url/clean-url' const ERROR_MODE_SECONDS_WINDOW = 30 * 1000 // sliding window of nodes to track when simply monitoring (but not harvesting) in error mode @@ -18,8 +19,6 @@ export class Aggregate extends AggregateBase { constructor (agentRef) { super(agentRef, FEATURE_NAME) - /** A buffer to hold on to harvested traces in the case that a retry must be made later */ - this.sentTrace = null this.harvestTimeSeconds = agentRef.init.session_trace.harvestTimeSeconds || 30 /** Tied to the entitlement flag response from BCS. Will short circuit operations of the agg if false */ this.entitled = undefined @@ -28,7 +27,7 @@ export class Aggregate extends AggregateBase { /** If the harvest module is harvesting */ this.harvesting = false /** TraceStorage is the mechanism that holds, normalizes and aggregates ST nodes. It will be accessed and purged when harvests occur */ - this.traceStorage = new TraceStorage(this) + this.events = new TraceStorage(this) /** This agg needs information about sampling (sts) and entitlements (st) to make the appropriate decisions on running */ this.waitForFlags(['sts', 'st']) .then(([stMode, stEntitled]) => this.initialize(stMode, stEntitled)) @@ -60,9 +59,9 @@ export class Aggregate extends AggregateBase { }) if (typeof PerformanceNavigationTiming !== 'undefined') { - this.traceStorage.storeTiming(globalScope.performance?.getEntriesByType?.('navigation')[0]) + this.events.storeTiming(globalScope.performance?.getEntriesByType?.('navigation')[0]) } else { - this.traceStorage.storeTiming(globalScope.performance?.timing, true) + this.events.storeTiming(globalScope.performance?.timing, true) } } @@ -77,21 +76,21 @@ export class Aggregate extends AggregateBase { this.timeKeeper ??= this.agentRef.runtime.timeKeeper - this.scheduler = new HarvestScheduler('browser/blobs', { - onFinished: this.onHarvestFinished.bind(this), + this.scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry), retryDelay: this.harvestTimeSeconds, - getPayload: this.prepareHarvest.bind(this), + getPayload: (options) => this.makeHarvestPayload(options.retry), raw: true }, this) /** The handlers set up by the Inst file */ - registerHandler('bst', (...args) => this.traceStorage.storeEvent(...args), this.featureName, this.ee) - registerHandler('bstResource', (...args) => this.traceStorage.storeResources(...args), this.featureName, this.ee) - registerHandler('bstHist', (...args) => this.traceStorage.storeHist(...args), this.featureName, this.ee) - registerHandler('bstXhrAgg', (...args) => this.traceStorage.storeXhrAgg(...args), this.featureName, this.ee) - registerHandler('bstApi', (...args) => this.traceStorage.storeSTN(...args), this.featureName, this.ee) - registerHandler('trace-jserror', (...args) => this.traceStorage.storeErrorAgg(...args), this.featureName, this.ee) - registerHandler('pvtAdded', (...args) => this.traceStorage.processPVT(...args), this.featureName, this.ee) + registerHandler('bst', (...args) => this.events.storeEvent(...args), this.featureName, this.ee) + registerHandler('bstResource', (...args) => this.events.storeResources(...args), this.featureName, this.ee) + registerHandler('bstHist', (...args) => this.events.storeHist(...args), this.featureName, this.ee) + registerHandler('bstXhrAgg', (...args) => this.events.storeXhrAgg(...args), this.featureName, this.ee) + registerHandler('bstApi', (...args) => this.events.storeSTN(...args), this.featureName, this.ee) + registerHandler('trace-jserror', (...args) => this.events.storeErrorAgg(...args), this.featureName, this.ee) + registerHandler('pvtAdded', (...args) => this.events.processPVT(...args), this.featureName, this.ee) /** Only start actually harvesting if running in full mode at init time */ if (this.mode === MODE.FULL) this.startHarvesting() @@ -112,28 +111,32 @@ export class Aggregate extends AggregateBase { this.scheduler.startTimer(this.harvestTimeSeconds) } - /** Called by the harvest scheduler at harvest time to retrieve the payload. This will only actually return a payload if running in full mode */ - prepareHarvest (options = {}) { - this.traceStorage.prevStoredEvents.clear() // release references to past events for GC + preHarvestChecks () { + if (this.mode !== MODE.FULL) return // only allow harvest if running in full mode if (!this.timeKeeper?.ready) return // this should likely never happen, but just to be safe, we should never harvest if we cant correct time - if (this.blocked || this.mode !== MODE.FULL || this.traceStorage.nodeCount === 0) return - if (this.sessionId !== this.agentRef.runtime.session?.state.value || this.ptid !== this.agentRef.runtime.ptid) return this.abort(3) // if something unexpected happened and we somehow still got to the point of harvesting after a session identifier changed, we should force-exit instead of harvesting - /** Get the ST nodes from the traceStorage buffer. This also returns helpful metadata about the payload. */ - const { stns, earliestTimeStamp, latestTimeStamp } = this.traceStorage.takeSTNs() - if (!stns) return // there are no trace nodes - if (options.retry) { - this.sentTrace = stns + if (!this.agentRef.runtime.session) return // session entity is required for trace to run and continue running + if (this.sessionId !== this.agentRef.runtime.session.state.value || this.ptid !== this.agentRef.runtime.ptid) { + // If something unexpected happened and we somehow still got to harvesting after a session identifier changed, we should force-exit instead of harvesting: + this.abort(3) + return } + return true + } + + serializer ({ stns }) { + if (!stns.length) return // there are no processed nodes + this.everHarvested = true + return applyFnToProps(stns, this.obfuscator.obfuscateString.bind(this.obfuscator), 'string') + } + queryStringsBuilder ({ stns, earliestTimeStamp, latestTimeStamp }) { const firstSessionHarvest = !this.agentRef.runtime.session.state.traceHarvestStarted if (firstSessionHarvest) this.agentRef.runtime.session.write({ traceHarvestStarted: true }) + const hasReplay = this.agentRef.runtime.session.state.sessionReplayMode === 1 + const endUserId = this.agentRef.info.jsAttributes['enduser.id'] + const entityGuid = this.agentRef.runtime.appMetadata.agents?.[0]?.entityGuid - const hasReplay = this.agentRef.runtime.session?.state.sessionReplayMode === 1 - const endUserId = this.agentRef.info?.jsAttributes?.['enduser.id'] - - this.everHarvested = true - - /** The blob consumer expects the following and will reject if not supplied: + /* The blob consumer expects the following and will reject if not supplied: * browser_monitoring_key * type * app_id @@ -142,47 +145,34 @@ export class Aggregate extends AggregateBase { * * For data that does not fit the schema of the above, it should be url-encoded and placed into `attributes` */ - const agentMetadata = this.agentRef.runtime.appMetadata?.agents?.[0] || {} - return { - qs: { - browser_monitoring_key: this.agentRef.info.licenseKey, - type: 'BrowserSessionChunk', - app_id: this.agentRef.info.applicationID, - protocol_version: '0', - timestamp: Math.floor(this.timeKeeper.correctRelativeTimestamp(earliestTimeStamp)), - attributes: encodeObj({ - ...(agentMetadata.entityGuid && { entityGuid: agentMetadata.entityGuid }), - harvestId: `${this.agentRef.runtime.session?.state.value}_${this.agentRef.runtime.ptid}_${this.agentRef.runtime.harvestCount}`, - // this section of attributes must be controllable and stay below the query param padding limit -- see QUERY_PARAM_PADDING - // if not, data could be lost to truncation at time of sending, potentially breaking parsing / API behavior in NR1 - // trace payload metadata - 'trace.firstTimestamp': Math.floor(this.timeKeeper.correctRelativeTimestamp(earliestTimeStamp)), - 'trace.lastTimestamp': Math.floor(this.timeKeeper.correctRelativeTimestamp(latestTimeStamp)), - 'trace.nodes': stns.length, - 'trace.originTimestamp': this.timeKeeper.correctedOriginTime, - // other payload metadata - agentVersion: this.agentRef.runtime.version, - ...(firstSessionHarvest && { firstSessionHarvest }), - ...(hasReplay && { hasReplay }), - ptid: `${this.ptid}`, - session: `${this.sessionId}`, - // customer-defined data should go last so that if it exceeds the query param padding limit it will be truncated instead of important attrs - ...(endUserId && { 'enduser.id': this.obfuscator.obfuscateString(endUserId) }), - currentUrl: this.obfuscator.obfuscateString(cleanURL('' + location)) - // The Query Param is being arbitrarily limited in length here. It is also applied when estimating the size of the payload in getPayloadSize() - }, QUERY_PARAM_PADDING).substring(1) // remove the leading '&' - }, - body: applyFnToProps(stns, this.obfuscator.obfuscateString.bind(this.obfuscator), 'string') - } - } - /** When the harvest scheduler finishes, this callback is executed. It's main purpose is to determine if the payload needs to be retried - * and if so, it will take all data from the temporary buffer and place it back into the traceStorage module - */ - onHarvestFinished (result) { - if (result.sent && result.retry && this.sentTrace) { // merge previous trace back into buffer to retry for next harvest - Object.entries(this.sentTrace).forEach(([name, listOfSTNodes]) => { this.traceStorage.restoreNode(name, listOfSTNodes) }) - this.sentTrace = null + return { + browser_monitoring_key: this.agentRef.info.licenseKey, + type: 'BrowserSessionChunk', + app_id: this.agentRef.info.applicationID, + protocol_version: '0', + timestamp: Math.floor(this.timeKeeper.correctRelativeTimestamp(earliestTimeStamp)), + attributes: encodeObj({ + ...(entityGuid && { entityGuid }), + harvestId: `${this.agentRef.runtime.session.state.value}_${this.agentRef.runtime.ptid}_${this.agentRef.runtime.harvestCount}`, + // this section of attributes must be controllable and stay below the query param padding limit -- see QUERY_PARAM_PADDING + // if not, data could be lost to truncation at time of sending, potentially breaking parsing / API behavior in NR1 + // trace payload metadata + 'trace.firstTimestamp': Math.floor(this.timeKeeper.correctRelativeTimestamp(earliestTimeStamp)), + 'trace.lastTimestamp': Math.floor(this.timeKeeper.correctRelativeTimestamp(latestTimeStamp)), + 'trace.nodes': stns.length, + 'trace.originTimestamp': this.timeKeeper.correctedOriginTime, + // other payload metadata + agentVersion: this.agentRef.runtime.version, + ...(firstSessionHarvest && { firstSessionHarvest }), + ...(hasReplay && { hasReplay }), + ptid: `${this.ptid}`, + session: `${this.sessionId}`, + // customer-defined data should go last so that if it exceeds the query param padding limit it will be truncated instead of important attrs + ...(endUserId && { 'enduser.id': this.obfuscator.obfuscateString(endUserId) }), + currentUrl: this.obfuscator.obfuscateString(cleanURL('' + location)) + // The Query Param is being arbitrarily limited in length here. It is also applied when estimating the size of the payload in getPayloadSize() + }, QUERY_PARAM_PADDING).substring(1) // remove the leading '&' } } @@ -194,16 +184,17 @@ export class Aggregate extends AggregateBase { this.agentRef.runtime.session.write({ sessionTraceMode: this.mode }) if (prevMode === MODE.OFF || !this.initialized) return this.initialize(this.mode, this.entitled) if (this.initialized) { - this.traceStorage.trimSTNs(ERROR_MODE_SECONDS_WINDOW) // up until now, Trace would've been just buffering nodes up to max, which needs to be trimmed to last X seconds + this.events.trimSTNs(ERROR_MODE_SECONDS_WINDOW) // up until now, Trace would've been just buffering nodes up to max, which needs to be trimmed to last X seconds } this.startHarvesting() } /** Stop running for the remainder of the page lifecycle */ - abort (reason) { + abort () { this.blocked = true this.mode = MODE.OFF this.agentRef.runtime.session.write({ sessionTraceMode: this.mode }) this.scheduler?.stopTimer() + this.events.clear() } } diff --git a/src/features/session_trace/aggregate/trace/storage.js b/src/features/session_trace/aggregate/trace/storage.js index 27f7a2969..294996cb1 100644 --- a/src/features/session_trace/aggregate/trace/storage.js +++ b/src/features/session_trace/aggregate/trace/storage.js @@ -29,8 +29,8 @@ export class TraceStorage { trace = {} earliestTimeStamp = Infinity latestTimeStamp = 0 - tempStorage = [] prevStoredEvents = new Set() + #backupTrace constructor (parent) { this.parent = parent @@ -44,9 +44,6 @@ export class TraceStorage { const openedSpace = this.trimSTNs(ERROR_MODE_SECONDS_WINDOW) // but maybe we could make some space by discarding irrelevant nodes if we're in sessioned Error mode if (openedSpace === 0) return } - while (this.tempStorage.length) { - this.storeSTN(this.tempStorage.shift()) - } if (this.trace[stn.n]) this.trace[stn.n].push(stn) else this.trace[stn.n] = [stn] @@ -96,15 +93,9 @@ export class TraceStorage { const partitionListByOriginMap = listOfSTNodes.sort((a, b) => a.s - b.s).reduce(reindexByOriginFn, {}) return Object.values(partitionListByOriginMap).flat() // join the partitions back into 1-D, now ordered by origin then start time }, this) - if (stns.length === 0) return {} - this.trace = {} - this.nodeCount = 0 const earliestTimeStamp = this.earliestTimeStamp - this.earliestTimeStamp = Infinity const latestTimeStamp = this.latestTimeStamp - this.latestTimeStamp = 0 - return { stns, earliestTimeStamp, latestTimeStamp } } @@ -282,10 +273,30 @@ export class TraceStorage { this.storeSTN(new TraceNode('Ajax', metrics.time, metrics.time + metrics.duration, `${params.status} ${params.method}: ${params.host}${params.pathname}`, 'ajax')) } - restoreNode (name, listOfSTNodes) { - if (this.nodeCount >= MAX_NODES_PER_HARVEST) return + /* Below are the interface expected & required of whatever storage is used across all features on an individual basis. This allows a common `.events` property on Trace. */ + isEmpty () { + return this.nodeCount === 0 + } + + save () { + this.#backupTrace = this.trace + } + + get = this.takeSTNs + + clear () { + this.trace = {} + this.nodeCount = 0 + this.prevStoredEvents.clear() // release references to past events for GC + this.earliestTimeStamp = Infinity + this.latestTimeStamp = 0 + } + + reloadSave () { + Object.values(this.#backupTrace).forEach(stnsArray => stnsArray.forEach(stn => this.storeSTN(stn))) + } - this.nodeCount += listOfSTNodes.length - this.trace[name] = this.trace[name] ? listOfSTNodes.concat(this.trace[name]) : listOfSTNodes + clearSave () { + this.#backupTrace = undefined } } diff --git a/src/features/soft_navigations/aggregate/index.js b/src/features/soft_navigations/aggregate/index.js index c7f050244..3f2baa2e2 100644 --- a/src/features/soft_navigations/aggregate/index.js +++ b/src/features/soft_navigations/aggregate/index.js @@ -3,10 +3,9 @@ import { registerHandler } from '../../../common/event-emitter/register-handler' import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler' import { single } from '../../../common/util/invoke' import { timeToFirstByte } from '../../../common/vitals/time-to-first-byte' -import { FEATURE_NAMES } from '../../../loaders/features/features' +import { FEATURE_NAMES, FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { SUPPORTABILITY_METRIC_CHANNEL } from '../../metrics/constants' import { AggregateBase } from '../../utils/aggregate-base' -import { EventBuffer } from '../../utils/event-buffer' import { API_TRIGGER_NAME, FEATURE_NAME, INTERACTION_STATUS } from '../constants' import { AjaxNode } from './ajax-node' import { InitialPageLoadInteraction } from './initial-page-load-interaction' @@ -18,7 +17,7 @@ export class Aggregate extends AggregateBase { super(agentRef, FEATURE_NAME) const harvestTimeSeconds = agentRef.init.soft_navigations.harvestTimeSeconds || 10 - this.interactionsToHarvest = new EventBuffer() + this.interactionsToHarvest = this.events this.domObserver = domObserver this.initialPageLoadInteraction = new InitialPageLoadInteraction(agentRef.agentIdentifier) @@ -39,12 +38,12 @@ export class Aggregate extends AggregateBase { this.waitForFlags(['spa']).then(([spaOn]) => { if (spaOn) { this.drain() - const scheduler = new HarvestScheduler('events', { - onFinished: this.onHarvestFinished.bind(this), + const scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry), + getPayload: (options) => this.makeHarvestPayload(options.retry), retryDelay: harvestTimeSeconds, onUnload: () => this.interactionInProgress?.done() // return any held ajax or jserr events so they can be sent with EoL harvest }, this) - scheduler.harvest.on('events', this.onHarvestStarted.bind(this)) scheduler.startTimer(harvestTimeSeconds, 0) } else { this.blocked = true // if rum response determines that customer lacks entitlements for spa endpoint, this feature shouldn't harvest @@ -66,27 +65,16 @@ export class Aggregate extends AggregateBase { registerHandler('jserror', this.#handleJserror.bind(this), this.featureName, this.ee) } - onHarvestStarted (options) { - if (!this.interactionsToHarvest.hasData || this.blocked) return + serializer (eventBuffer) { // The payload depacker takes the first ixn of a payload (if there are multiple ixns) and positively offset the subsequent ixns timestamps by that amount. // In order to accurately portray the real start & end times of the 2nd & onward ixns, we hence need to negatively offset their start timestamps with that of the 1st ixn. let firstIxnStartTime = 0 // the very 1st ixn does not require any offsetting const serializedIxnList = [] - for (const interaction of this.interactionsToHarvest.buffer) { + for (const interaction of eventBuffer) { serializedIxnList.push(interaction.serialize(firstIxnStartTime)) if (!firstIxnStartTime) firstIxnStartTime = Math.floor(interaction.start) } - const payload = `bel.7;${serializedIxnList.join(';')}` - - if (options.retry) this.interactionsToHarvest.hold() - else this.interactionsToHarvest.clear() - - return { body: { e: payload } } - } - - onHarvestFinished (result) { - if (result.sent && result.retry && this.interactionsToHarvest.held.hasData) this.interactionsToHarvest.unhold() - else this.interactionsToHarvest.held.clear() + return `bel.7;${serializedIxnList.join(';')}` } startUIInteraction (eventName, startedAt, sourceElem) { // this is throttled by instrumentation so that it isn't excessively called @@ -139,8 +127,9 @@ export class Aggregate extends AggregateBase { */ if (this.interactionInProgress?.isActiveDuring(timestamp)) return this.interactionInProgress let saveIxn - for (let idx = this.interactionsToHarvest.buffer.length - 1; idx >= 0; idx--) { // reverse search for the latest completed interaction for efficiency - const finishedInteraction = this.interactionsToHarvest.buffer[idx] + const interactionsBuffer = this.interactionsToHarvest.get() + for (let idx = interactionsBuffer.length - 1; idx >= 0; idx--) { // reverse search for the latest completed interaction for efficiency + const finishedInteraction = interactionsBuffer[idx] if (finishedInteraction.isActiveDuring(timestamp)) { if (finishedInteraction.trigger !== 'initialPageLoad') return finishedInteraction // It's possible that a complete interaction occurs before page is fully loaded, so we need to consider if a route-change ixn may have overlapped this iPL diff --git a/src/features/spa/aggregate/index.js b/src/features/spa/aggregate/index.js index ce1f9e433..403c49a6e 100644 --- a/src/features/spa/aggregate/index.js +++ b/src/features/spa/aggregate/index.js @@ -14,7 +14,7 @@ import { HarvestScheduler } from '../../../common/harvest/harvest-scheduler' import { Serializer } from './serializer' import { ee } from '../../../common/event-emitter/contextual-ee' import * as CONSTANTS from '../constants' -import { FEATURE_NAMES } from '../../../loaders/features/features' +import { FEATURE_NAMES, FEATURE_TO_ENDPOINT } from '../../../loaders/features/features' import { AggregateBase } from '../../utils/aggregate-base' import { firstContentfulPaint } from '../../../common/vitals/first-contentful-paint' import { firstPaint } from '../../../common/vitals/first-paint' @@ -23,7 +23,6 @@ import { initialLocation, loadedAsDeferredBrowserScript } from '../../../common/ import { handle } from '../../../common/event-emitter/handle' import { SUPPORTABILITY_METRIC_CHANNEL } from '../../metrics/constants' import { warn } from '../../../common/util/console' -import { EventBuffer } from '../../utils/event-buffer' const { FEATURE_NAME, INTERACTION_EVENTS, MAX_TIMER_BUDGET, FN_START, FN_END, CB_START, INTERACTION_API, REMAINING, @@ -34,7 +33,7 @@ export class Aggregate extends AggregateBase { constructor (agentRef) { super(agentRef, FEATURE_NAME) - this.state = { + const state = this.state = { initialPageURL: initialLocation, lastSeenUrl: initialLocation, lastSeenRouteName: null, @@ -48,15 +47,13 @@ export class Aggregate extends AggregateBase { childTime: 0, depth: 0, harvestTimeSeconds: agentRef.init.spa.harvestTimeSeconds || 10, - interactionsToHarvest: new EventBuffer(), // The below feature flag is used to disable the SPA ajax fix for specific customers, see https://new-relic.atlassian.net/browse/NR-172169 disableSpaFix: (agentRef.init.feature_flags || []).indexOf('disable-spa-fix') > -1 } + this.spaSerializerClass = new Serializer(this) + const classThis = this let scheduler - this.serializer = new Serializer(this) - - const { state, serializer } = this const baseEE = ee.get(agentRef.agentIdentifier) // <-- parent baseEE const mutationEE = baseEE.get('mutation') @@ -103,11 +100,11 @@ export class Aggregate extends AggregateBase { this.waitForFlags((['spa'])).then(([spaFlag]) => { if (spaFlag) { - scheduler = new HarvestScheduler('events', { - onFinished: onHarvestFinished, + scheduler = new HarvestScheduler(FEATURE_TO_ENDPOINT[this.featureName], { + onFinished: (result) => this.postHarvestCleanup(result.sent && result.retry), + getPayload: (options) => this.makeHarvestPayload(options.retry), retryDelay: state.harvestTimeSeconds }, this) - scheduler.harvest.on('events', onHarvestStarted) this.drain() } else { this.blocked = true @@ -670,22 +667,6 @@ export class Aggregate extends AggregateBase { setCurrentNode(null) } - const classThis = this - function onHarvestStarted (options) { - if (!state.interactionsToHarvest.hasData || classThis.blocked) return {} - var payload = serializer.serializeMultiple(state.interactionsToHarvest.buffer, 0, navTiming) - - if (options.retry) state.interactionsToHarvest.hold() - else state.interactionsToHarvest.clear() - - return { body: { e: payload } } - } - - function onHarvestFinished (result) { - if (result.sent && result.retry && state.interactionsToHarvest.held.hasData) state.interactionsToHarvest.unhold() - else state.interactionsToHarvest.held.clear() - } - baseEE.on('spa-jserror', function (type, name, params, metrics) { if (!state.currentNode) return params._interactionId = state.currentNode.interaction.id @@ -736,7 +717,7 @@ export class Aggregate extends AggregateBase { interaction.root.attrs.firstContentfulPaint = firstContentfulPaint.current.value } baseEE.emit('interactionDone', [interaction, true]) - state.interactionsToHarvest.add(interaction) + classThis.events.add(interaction) let smCategory if (interaction.root?.attrs?.trigger === 'initialPageLoad') smCategory = 'InitialPageLoad' @@ -748,4 +729,8 @@ export class Aggregate extends AggregateBase { if (!scheduler) warn(19) } } + + serializer (eventBuffer) { + return this.spaSerializerClass.serializeMultiple(eventBuffer, 0, navTiming) + } } diff --git a/src/features/utils/aggregate-base.js b/src/features/utils/aggregate-base.js index 43e0ab3eb..873b64b61 100644 --- a/src/features/utils/aggregate-base.js +++ b/src/features/utils/aggregate-base.js @@ -5,11 +5,17 @@ import { gosCDN } from '../../common/window/nreum' import { drain } from '../../common/drain/drain' import { activatedFeatures } from '../../common/util/feature-flags' import { Obfuscator } from '../../common/util/obfuscate' +import { EventBuffer } from './event-buffer' +import { FEATURE_NAMES } from '../../loaders/features/features' export class AggregateBase extends FeatureBase { constructor (agentRef, featureName) { super(agentRef.agentIdentifier, featureName) this.agentRef = agentRef + // Jserror and Metric features uses a singleton EventAggregator instead of a regular EventBuffer. + if ([FEATURE_NAMES.jserrors, FEATURE_NAMES.metrics].includes(this.featureName)) this.events = agentRef.sharedAggregator + // PVE has no need for eventBuffer, and SessionTrace has its own storage mechanism. + else if (![FEATURE_NAMES.pageViewEvent, FEATURE_NAMES.sessionTrace].includes(this.featureName)) this.events = new EventBuffer() this.checkConfiguration(agentRef) this.obfuscator = agentRef.runtime.obfuscator } @@ -47,6 +53,39 @@ export class AggregateBase extends FeatureBase { this.drained = true } + /** + * Return harvest payload. A "serializer" function can be defined on a derived class to format the payload. + * @param {Boolean} shouldRetryOnFail - harvester flag to backup payload for retry later if harvest request fails; this should be moved to harvester logic + * @returns final payload, or undefined if there are no pending events + */ + makeHarvestPayload (shouldRetryOnFail = false, opts = {}) { + if (this.events.isEmpty(opts)) return + // Other conditions and things to do when preparing harvest that is required. + if (this.preHarvestChecks && !this.preHarvestChecks()) return + + if (shouldRetryOnFail) this.events.save(opts) + const returnedData = this.events.get(opts) + // A serializer or formatter assists in creating the payload `body` from stored events on harvest when defined by derived feature class. + const body = this.serializer ? this.serializer(returnedData) : returnedData + this.events.clear(opts) + + const payload = { + body + } + // Constructs the payload `qs` for relevant features on harvest. + if (this.queryStringsBuilder) payload.qs = this.queryStringsBuilder(returnedData) + return payload + } + + /** + * Cleanup task after a harvest. + * @param {Boolean} harvestFailed - harvester flag to restore events in main buffer for retry later if request failed + */ + postHarvestCleanup (harvestFailed = false, opts = {}) { + if (harvestFailed) this.events.reloadSave(opts) + this.events.clearSave(opts) + } + /** * Checks for additional `jsAttributes` items to support backward compatibility with implementations of the agent where * loader configurations may appear after the loader code is executed. diff --git a/src/features/utils/event-buffer.js b/src/features/utils/event-buffer.js index 7ca5167f8..86fef5ef9 100644 --- a/src/features/utils/event-buffer.js +++ b/src/features/utils/event-buffer.js @@ -1,127 +1,80 @@ import { stringify } from '../../common/util/stringify' import { MAX_PAYLOAD_SIZE } from '../../common/constants/agent-constants' -/** - * A container that keeps an event buffer and size with helper methods - * @typedef {Object} EventBuffer - * @property {number} size - * @property {*[]} buffer - */ - -/** - * A container that holds, evaluates, and merges event objects for harvesting - */ export class EventBuffer { - /** @type {Object[]} */ #buffer = [] - /** @type {number} */ - #bytes = 0 - /** @type {EventBuffer} */ - #held + #rawBytes = 0 + #bufferBackup + #rawBytesBackup /** - * - * @param {number=} maxPayloadSize + * @param {number} maxPayloadSize */ constructor (maxPayloadSize = MAX_PAYLOAD_SIZE) { this.maxPayloadSize = maxPayloadSize } - /** - * buffer is read only, use the helper methods to add or clear buffer data - */ - get buffer () { - return this.#buffer + isEmpty () { + return this.#buffer.length === 0 } - /** - * bytes is read only, use the helper methods to add or clear buffer data - */ - get bytes () { - return this.#bytes + get () { + return this.#buffer } - /** - * held is another event buffer - */ - get held () { - if (!this.#held) this.#held = new EventBuffer(this.maxPayloadSize) - return this.#held + byteSize () { + return this.#rawBytes } - /** - * Returns a boolean indicating whether the current size and buffer contain valid data - * @returns {boolean} - */ - get hasData () { - return this.buffer.length > 0 && this.bytes > 0 + wouldExceedMaxSize (incomingSize) { + return this.#rawBytes + incomingSize > this.maxPayloadSize } /** - * Adds an event object to the buffer while tallying size. Only adds the event if it is valid - * and would not make the event buffer exceed the maxPayloadSize. - * @param {Object} event the event object to add to the buffer - * @returns {EventBuffer} returns the event buffer for chaining + * Add feature-processed event to our buffer. If this event would cause our total raw size to exceed the set max payload size, it is dropped. + * @param {any} event - any primitive type or object + * @returns {Boolean} true if successfully added; false otherwise */ add (event) { - const size = stringify(event).length - if (!this.canMerge(size)) return this + const addSize = stringify(event)?.length || 0 // (estimate) # of bytes a directly stringified event it would take to send + if (this.#rawBytes + addSize > this.maxPayloadSize) return false this.#buffer.push(event) - this.#bytes += size - return this + this.#rawBytes += addSize + return true } /** - * clear the buffer data - * @returns {EventBuffer} + * Wipes the main buffer */ clear () { - this.#bytes = 0 this.#buffer = [] - return this - } - - /** - * Hold the buffer data in a new (child) EventBuffer (.held) to unblock the main buffer. - * This action clears the main buffer - * @returns {EventBuffer} - */ - hold () { - this.held.merge(this) - this.clear() - return this + this.#rawBytes = 0 } /** - * Prepend the held EventBuffer (.held) back into the main buffer - * This action clears the held buffer - * @returns {EventBuffer} + * Backup the buffered data and clear the main buffer + * @returns {Array} the events being backed up */ - unhold () { - this.merge(this.held, true) - this.held.clear() - return this + save () { + this.#bufferBackup = this.#buffer + this.#rawBytesBackup = this.#rawBytes } /** - * Merges an EventBuffer into this EventBuffer - * @param {EventBuffer} events an EventBuffer intended to merge with this EventBuffer - * @param {boolean} prepend if true, the supplied events will be prepended before the events of this class - * @returns {EventBuffer} returns the event buffer for chaining + * Wipes the backup buffer */ - merge (eventBuffer, prepend = false) { - if (!this.canMerge(eventBuffer.bytes)) return this - this.#buffer = prepend ? [...eventBuffer.buffer, ...this.#buffer] : [...this.#buffer, ...eventBuffer.buffer] - this.#bytes += eventBuffer.#bytes - return this + clearSave () { + this.#bufferBackup = undefined + this.#rawBytesBackup = undefined } /** - * Returns a boolean indicating the resulting size of a merge would be valid. Compares against the maxPayloadSize provided at initialization time. - * @param {number} size - * @returns {boolean} + * Prepend the backup buffer back into the main buffer */ - canMerge (size) { - return this.bytes + (size || Infinity) < this.maxPayloadSize + reloadSave () { + if (!this.#bufferBackup) return + if (this.#rawBytesBackup + this.#rawBytes > this.maxPayloadSize) return + this.#buffer = [...this.#bufferBackup, ...this.#buffer] + this.#rawBytes = this.#rawBytesBackup + this.#rawBytes } } diff --git a/src/features/utils/instrument-base.js b/src/features/utils/instrument-base.js index d7b570cee..23aa5ab85 100644 --- a/src/features/utils/instrument-base.js +++ b/src/features/utils/instrument-base.js @@ -97,9 +97,9 @@ export class InstrumentBase extends FeatureBase { try { // Create a single Aggregator for this agent if DNE yet; to be used by jserror endpoint features. if (!agentRef.sharedAggregator) { - agentRef.sharedAggregator = import(/* webpackChunkName: "shared-aggregator" */ '../../common/aggregate/aggregator') - const { Aggregator } = await agentRef.sharedAggregator - agentRef.sharedAggregator = new Aggregator() + agentRef.sharedAggregator = import(/* webpackChunkName: "shared-aggregator" */ '../../common/aggregate/event-aggregator') + const { EventAggregator } = await agentRef.sharedAggregator + agentRef.sharedAggregator = new EventAggregator() } else await agentRef.sharedAggregator // if another feature is already importing the aggregator, wait for it to finish if (!this.#shouldImportAgg(this.featureName, session)) { diff --git a/src/loaders/features/features.js b/src/loaders/features/features.js index 21e8a581e..444c885f9 100644 --- a/src/loaders/features/features.js +++ b/src/loaders/features/features.js @@ -33,3 +33,16 @@ export const featurePriority = { [FEATURE_NAMES.logging]: 10, [FEATURE_NAMES.genericEvents]: 11 } + +export const FEATURE_TO_ENDPOINT = { + [FEATURE_NAMES.pageViewTiming]: 'events', + [FEATURE_NAMES.ajax]: 'events', + [FEATURE_NAMES.spa]: 'events', + [FEATURE_NAMES.softNav]: 'events', + [FEATURE_NAMES.metrics]: 'jserrors', + [FEATURE_NAMES.jserrors]: 'jserrors', + [FEATURE_NAMES.sessionTrace]: 'browser/blobs', + [FEATURE_NAMES.sessionReplay]: 'browser/blobs', + [FEATURE_NAMES.logging]: 'browser/logs', + [FEATURE_NAMES.genericEvents]: 'ins' +} diff --git a/tests/components/ajax/aggregate.test.js b/tests/components/ajax/aggregate.test.js index 9f52a7b8e..653954e31 100644 --- a/tests/components/ajax/aggregate.test.js +++ b/tests/components/ajax/aggregate.test.js @@ -6,7 +6,6 @@ import { Instrument as Ajax } from '../../../src/features/ajax/instrument' import { resetAgent, setupAgent } from '../setup-agent' import { EventContext } from '../../../src/common/event-emitter/event-context' import { getInfo } from '../../../src/common/config/info' -import * as agentConstants from '../../../src/common/constants/agent-constants' const ajaxArguments = [ { // params @@ -61,8 +60,8 @@ test('on interactionDiscarded, saved (old) SPA events are put back in ajaxEvents ajaxAggregate.ee.emit('xhr', ajaxArguments, context) ajaxAggregate.ee.emit('interactionDone', [interaction, false]) - expect(ajaxAggregate.spaAjaxEvents[interaction.id]).toBeUndefined() // no interactions in SPA under interaction 0 - expect(ajaxAggregate.ajaxEvents.buffer.length).toEqual(1) + expect(ajaxAggregate.underSpaEvents[interaction.id]).toBeUndefined() // no interactions in SPA under interaction 0 + expect(ajaxAggregate.events.get().length).toEqual(1) }) test('on returnAjax from soft nav, event is re-routed back into ajaxEvents', () => { @@ -74,32 +73,32 @@ test('on returnAjax from soft nav, event is re-routed back into ajaxEvents', () const event = jest.mocked(handleModule.handle).mock.lastCall[1][0] ajaxAggregate.ee.emit('returnAjax', [event], context) - expect(ajaxAggregate.ajaxEvents.buffer.length).toEqual(1) - expect(ajaxAggregate.ajaxEvents.buffer[0]).toEqual(expect.objectContaining({ startTime: 0, path: '/pathname' })) + expect(ajaxAggregate.events.get().length).toEqual(1) + expect(ajaxAggregate.events.get()[0]).toEqual(expect.objectContaining({ startTime: 0, path: '/pathname' })) }) describe('storeXhr', () => { test('for a plain ajax request buffers in ajaxEvents', () => { ajaxAggregate.ee.emit('xhr', ajaxArguments, context) - expect(ajaxAggregate.ajaxEvents.buffer.length).toEqual(1) // non-SPA ajax requests are buffered in ajaxEvents - expect(Object.keys(ajaxAggregate.spaAjaxEvents).length).toEqual(0) + expect(ajaxAggregate.events.get().length).toEqual(1) // non-SPA ajax requests are buffered in ajaxEvents + expect(Object.keys(ajaxAggregate.underSpaEvents).length).toEqual(0) - const ajaxEvent = ajaxAggregate.ajaxEvents.buffer[0] + const ajaxEvent = ajaxAggregate.events.get()[0] expect(ajaxEvent).toEqual(expect.objectContaining({ startTime: 0, path: '/pathname' })) }) - test('for a (old) SPA ajax request buffers in spaAjaxEvents', () => { + test('for a (old) SPA ajax request buffers in underSpaEvents', () => { const interaction = { id: 0 } context.spaNode = { interaction } ajaxAggregate.ee.emit('xhr', ajaxArguments, context) - const interactionAjaxEvents = ajaxAggregate.spaAjaxEvents[interaction.id] - expect(interactionAjaxEvents.buffer.length).toEqual(1) // SPA ajax requests are buffered in spaAjaxEvents and under its interaction id - expect(ajaxAggregate.ajaxEvents.buffer.length).toEqual(0) + const interactionAjaxEvents = ajaxAggregate.underSpaEvents[interaction.id] + expect(interactionAjaxEvents.length).toEqual(1) // SPA ajax requests are buffered in underSpaEvents and under its interaction id + expect(ajaxAggregate.events.get().length).toEqual(0) - const spaAjaxEvent = interactionAjaxEvents.buffer[0] + const spaAjaxEvent = interactionAjaxEvents[0] expect(spaAjaxEvent).toEqual(expect.objectContaining({ startTime: 0, path: '/pathname' })) }) @@ -110,8 +109,8 @@ describe('storeXhr', () => { ajaxAggregate.ee.emit('xhr', ajaxArguments, context) - expect(ajaxAggregate.ajaxEvents.buffer.length).toEqual(0) - expect(Object.keys(ajaxAggregate.spaAjaxEvents).length).toEqual(0) + expect(ajaxAggregate.events.get().length).toEqual(0) + expect(Object.keys(ajaxAggregate.underSpaEvents).length).toEqual(0) expect(handleModule.handle).toHaveBeenLastCalledWith( 'ajax', [expect.objectContaining({ startTime: 0, path: '/pathname' })], @@ -153,52 +152,25 @@ describe('prepareHarvest', () => { } getInfo(agentSetup.agentIdentifier).jsAttributes = expectedCustomAttributes - const serializedPayload = ajaxAggregate.prepareHarvest({ retry: false }) + const serializedPayload = ajaxAggregate.makeHarvestPayload(false) // serializedPayload from ajax comes back as an array of bodies now, so we just need to decode each one and flatten // this decoding does not happen elsewhere in the app so this only needs to happen here in this specific test - const decodedEvents = serializedPayload.map(sp => qp.decode(sp.body.e)) + const decodedEvents = qp.decode(serializedPayload.body) - decodedEvents.forEach(payload => { - payload.forEach(event => { - validateCustomAttributeValues(expectedCustomAttributes, event.children) - delete event.children + decodedEvents.forEach(event => { + validateCustomAttributeValues(expectedCustomAttributes, event.children) + delete event.children - expect(event).toEqual(expected) // event attributes serialized correctly - }) - }) - }) - - test('correctly serializes a very large AjaxRequest events payload', () => { - for (let callNo = 0; callNo < 10; callNo++) ajaxAggregate.ee.emit('xhr', ajaxArguments, context) - - const expectedCustomAttributes = { - customStringAttribute: 'customStringAttribute', - customNumAttribute: 2, - customBooleanAttribute: true, - nullCustomAttribute: null - } - getInfo(agentSetup.agentIdentifier).jsAttributes = expectedCustomAttributes - - jest.replaceProperty(agentConstants, 'MAX_PAYLOAD_SIZE', 500) - const serializedPayload = ajaxAggregate.prepareHarvest({ retry: false }) - - expect(serializedPayload.length).toBeGreaterThan(1) // large payload of AJAX Events are broken into multiple chunks - expect(serializedPayload.every(sp => sp.body.e.length < 500)).toBeTruthy() // each chunks is less than the maxPayloadSize - - const decodedEvents = serializedPayload.map(sp => qp.decode(sp.body.e)) - decodedEvents.forEach(payload => { - payload.forEach(event => { - validateCustomAttributeValues(expectedCustomAttributes, event.children) // Custom attributes are accounted for in chunked AJAX payloads - }) + expect(event).toEqual(expected) // event attributes serialized correctly }) }) test('correctly exits if maxPayload is too small', () => { + ajaxAggregate.events.maxPayloadSize = 10 // this is too small for any AJAX payload to fit in for (let callNo = 0; callNo < 10; callNo++) ajaxAggregate.ee.emit('xhr', ajaxArguments, context) - jest.replaceProperty(agentConstants, 'MAX_PAYLOAD_SIZE', 10) // this is too small for any AJAX payload to fit in - const serializedPayload = ajaxAggregate.prepareHarvest({ retry: false }) - expect(serializedPayload.length).toEqual(0) // payload that are each too small for limit will be dropped + const serializedPayload = ajaxAggregate.makeHarvestPayload(false) + expect(serializedPayload).toBeUndefined() // payload that are each too small for limit will be dropped }) }) diff --git a/tests/components/generic_events/aggregate/index.test.js b/tests/components/generic_events/aggregate/index.test.js index 789f15386..e5e3294c3 100644 --- a/tests/components/generic_events/aggregate/index.test.js +++ b/tests/components/generic_events/aggregate/index.test.js @@ -28,9 +28,9 @@ test('should use default values', () => { expect(genericEventsAggregate).toMatchObject({ eventsPerHarvest: 1000, harvestTimeSeconds: 30, - referrerUrl: 'https://test.com', - events: new EventBuffer() + referrerUrl: 'https://test.com' }) + expect(genericEventsAggregate.events instanceof EventBuffer).toBeTruthy() }) test('should wait for flags - 1', async () => { @@ -89,7 +89,7 @@ describe('sub-features', () => { genericEventsAggregate.ee.emit('api-addPageAction', [relativeTimestamp, name, { foo: 'bar' }]) - expect(genericEventsAggregate.events.buffer[0]).toMatchObject({ + expect(genericEventsAggregate.events.get()[0]).toMatchObject({ eventType: 'PageAction', timestamp: Math.floor(timeKeeper.correctAbsoluteTimestamp( timeKeeper.convertRelativeTimestamp(relativeTimestamp) @@ -113,7 +113,7 @@ describe('sub-features', () => { genericEventsAggregate.ee.emit('api-addPageAction', [relativeTimestamp, name, { eventType: 'BetterPageAction', timestamp: 'BetterTimestamp' }]) - expect(genericEventsAggregate.events.buffer[0]).toMatchObject({ + expect(genericEventsAggregate.events.get()[0]).toMatchObject({ eventType: 'PageAction', timestamp: expect.any(Number) }) @@ -127,7 +127,7 @@ describe('sub-features', () => { genericEventsAggregate.ee.emit('api-addPageAction', [relativeTimestamp, name, {}]) - expect(genericEventsAggregate.events.buffer[0]).toMatchObject({ + expect(genericEventsAggregate.events.get()[0]).toMatchObject({ eventType: 'PageAction', timestamp: expect.any(Number) }) @@ -153,7 +153,7 @@ describe('sub-features', () => { // blur event to trigger aggregation to stop and add to harvest buffer genericEventsAggregate.ee.emit('ua', [{ timeStamp: 234567, type: 'blur', target: window }]) - const harvest = genericEventsAggregate.onHarvestStarted({ isFinalHarvest: true }) // force it to put the aggregation into the event buffer + const harvest = genericEventsAggregate.makeHarvestPayload() // force it to put the aggregation into the event buffer expect(harvest.body.ins[0]).toMatchObject({ eventType: 'UserAction', timestamp: expect.any(Number), @@ -180,7 +180,7 @@ describe('sub-features', () => { // blur event to trigger aggregation to stop and add to harvest buffer genericEventsAggregate.ee.emit('ua', [{ timeStamp: 234567, type: 'blur', target: window }]) - const harvest = genericEventsAggregate.onHarvestStarted({ isFinalHarvest: true }) // force it to put the aggregation into the event buffer + const harvest = genericEventsAggregate.makeHarvestPayload() // force it to put the aggregation into the event buffer expect(harvest.body.ins[0]).toMatchObject({ eventType: 'UserAction', timestamp: expect.any(Number), @@ -207,7 +207,7 @@ describe('sub-features', () => { // blur event to trigger aggregation to stop and add to harvest buffer genericEventsAggregate.ee.emit('ua', [{ timeStamp: 234567, type: 'blur', target: window }]) - const harvest = genericEventsAggregate.onHarvestStarted({ isFinalHarvest: true }) // force it to put the aggregation into the event buffer + const harvest = genericEventsAggregate.makeHarvestPayload() // force it to put the aggregation into the event buffer expect(harvest.body.ins[0]).toMatchObject({ eventType: 'UserAction', timestamp: expect.any(Number), @@ -262,7 +262,7 @@ describe('sub-features', () => { genericEventsAggregate.ee.emit('rumresp', [{ ins: 1 }]) await new Promise(process.nextTick) - expect(genericEventsAggregate.events.buffer[0]).toMatchObject({ + expect(genericEventsAggregate.events.get()[0]).toMatchObject({ eventType: 'BrowserPerformance', timestamp: expect.any(Number), entryName: 'test', @@ -302,7 +302,7 @@ describe('sub-features', () => { genericEventsAggregate.ee.emit('rumresp', [{ ins: 1 }]) await new Promise(process.nextTick) - expect(genericEventsAggregate.events.buffer[0]).toMatchObject({ + expect(genericEventsAggregate.events.get()[0]).toMatchObject({ eventType: 'BrowserPerformance', timestamp: expect.any(Number), entryName: 'test', diff --git a/tests/components/logging/aggregate.test.js b/tests/components/logging/aggregate.test.js index 6759df6d0..5488afbac 100644 --- a/tests/components/logging/aggregate.test.js +++ b/tests/components/logging/aggregate.test.js @@ -41,7 +41,7 @@ describe('class setup', () => { 'ee', 'featureName', 'blocked', - 'bufferedLogs', + 'events', 'harvestTimeSeconds' ])) }) @@ -72,9 +72,9 @@ describe('payloads', () => { { myAttributes: 1 }, 'error' ) - expect(loggingAggregate.bufferedLogs.buffer[0]).toEqual(expectedLog) + expect(loggingAggregate.events.get()[0]).toEqual(expectedLog) - expect(loggingAggregate.prepareHarvest()).toEqual({ + expect(loggingAggregate.makeHarvestPayload()).toEqual({ qs: { browser_monitoring_key: info.licenseKey }, body: [{ common: { @@ -100,7 +100,7 @@ describe('payloads', () => { test('prepares payload as expected', async () => { loggingAggregate.ee.emit(LOGGING_EVENT_EMITTER_CHANNEL, [1234, 'test message', { myAttributes: 1 }, 'error']) - expect(loggingAggregate.bufferedLogs.buffer[0]).toEqual(new Log( + expect(loggingAggregate.events.get()[0]).toEqual(new Log( Math.floor(runtime.timeKeeper.correctAbsoluteTimestamp( runtime.timeKeeper.convertRelativeTimestamp(1234) )), @@ -145,7 +145,7 @@ describe('payloads', () => { 'error' ) - const logs = loggingAggregate.bufferedLogs.buffer + const logs = loggingAggregate.events.get() loggingAggregate.ee.emit(LOGGING_EVENT_EMITTER_CHANNEL, [1234, 'test message', [], 'ERROR']) expect(logs.pop()).toEqual(expected) @@ -171,11 +171,11 @@ describe('payloads', () => { ) loggingAggregate.ee.emit(LOGGING_EVENT_EMITTER_CHANNEL, [1234, 'test message', {}, 'ErRoR']) - expect(loggingAggregate.bufferedLogs.buffer[0]).toEqual(expected) + expect(loggingAggregate.events.get()[0]).toEqual(expected) }) test('should buffer logs with non-stringify-able message', async () => { - const logs = loggingAggregate.bufferedLogs.buffer + const logs = loggingAggregate.events.get() loggingAggregate.ee.emit(LOGGING_EVENT_EMITTER_CHANNEL, [1234, new Error('test'), {}, 'error']) expect(logs.pop().message).toEqual('Error: test') diff --git a/tests/components/metrics/aggregate.test.js b/tests/components/metrics/aggregate.test.js index bb5ea1760..43330e9a2 100644 --- a/tests/components/metrics/aggregate.test.js +++ b/tests/components/metrics/aggregate.test.js @@ -30,7 +30,7 @@ afterEach(() => { test(`${name} with no value creates a metric with just a count`, () => { createAndStoreMetric(undefined, isSupportability) - const records = mainAgent.sharedAggregator.take([type])[type] + const records = metricsAggregate.events.get([type])[type] .filter(x => x?.params?.name === metricName) expect(records.length).toEqual(1) @@ -47,7 +47,7 @@ afterEach(() => { createAndStoreMetric(undefined, isSupportability) createAndStoreMetric(undefined, isSupportability) - const records = mainAgent.sharedAggregator.take([type])[type] + const records = metricsAggregate.events.get([type])[type] .filter(x => x?.params?.name === metricName) expect(records.length).toEqual(1) @@ -62,7 +62,7 @@ afterEach(() => { test(`${name} with a value ${auxDescription}`, () => { createAndStoreMetric(isSupportability ? 500 : { time: 500 }, isSupportability) - const records = mainAgent.sharedAggregator.take([type])[type] + const records = metricsAggregate.events.get([type])[type] .filter(x => x?.params?.name === metricName) expect(records.length).toEqual(1) @@ -80,7 +80,7 @@ afterEach(() => { .fill(null).map(() => faker.number.int({ min: 100, max: 1000 })) values.forEach(v => createAndStoreMetric(isSupportability ? v : { time: v }, isSupportability)) - const records = mainAgent.sharedAggregator.take([type])[type] + const records = metricsAggregate.events.get([type])[type] .filter(x => x?.params?.name === metricName) expect(records.length).toEqual(1) @@ -99,7 +99,7 @@ afterEach(() => { test(`${name} does not create a ${otherType} item`, () => { createAndStoreMetric(faker.number.float(), isSupportability) - const records = mainAgent.sharedAggregator.take([otherType])?.[otherType] + const records = metricsAggregate.events.get([otherType])?.[otherType] ?.filter(x => x?.params?.name === metricName) || [] expect(records).toEqual([]) }) @@ -108,7 +108,7 @@ afterEach(() => { test('storeEvent (custom) with an invalid value type does not create a named metric object in metrics section', () => { createAndStoreMetric(faker.number.float(), false) - const records = mainAgent.sharedAggregator.take([CUSTOM_METRIC])[CUSTOM_METRIC] + const records = metricsAggregate.events.get([CUSTOM_METRIC])[CUSTOM_METRIC] .filter(x => x?.params?.name === metricName) expect(records.length).toEqual(1) diff --git a/tests/components/page_view_timing/aggregate.test.js b/tests/components/page_view_timing/aggregate.test.js index 2b5c50c92..b97de0556 100644 --- a/tests/components/page_view_timing/aggregate.test.js +++ b/tests/components/page_view_timing/aggregate.test.js @@ -69,7 +69,7 @@ const expectedNetworkInfo = { } test('LCP event with CLS attribute', () => { - const timing = find(timingsAggregate.timings.buffer, function (t) { + const timing = find(timingsAggregate.events.get(), function (t) { return t.name === 'lcp' }) @@ -91,26 +91,26 @@ test('LCP event with CLS attribute', () => { }) test('sends expected FI attributes when available', () => { - expect(timingsAggregate.timings.buffer.length).toBeGreaterThanOrEqual(1) - const fiPayload = timingsAggregate.timings.buffer.find(x => x.name === 'fi') + expect(timingsAggregate.events.get().length).toBeGreaterThanOrEqual(1) + const fiPayload = timingsAggregate.events.get().find(x => x.name === 'fi') expect(fiPayload.value).toEqual(5) expect(fiPayload.attrs).toEqual(expect.objectContaining({ type: 'pointerdown', fid: 1234, cls: 0.1119, ...expectedNetworkInfo })) }) test('sends CLS node with right val on vis change', () => { - let clsNode = timingsAggregate.timings.buffer.find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) + let clsNode = timingsAggregate.events.get().find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) expect(clsNode).toBeUndefined() pageVisibilityModule.subscribeToVisibilityChange.mock.calls[1][0]() - clsNode = timingsAggregate.timings.buffer.find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) + clsNode = timingsAggregate.events.get().find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) expect(clsNode).toBeTruthy() expect(clsNode.value).toEqual(111.9) // since cls multiply decimal by 1000 to offset consumer division by 1000 expect(clsNode.attrs.cls).toBeUndefined() // cls node doesn't need cls property }) test('sends INP node with right val', () => { - let inpNode = timingsAggregate.timings.buffer.find(tn => tn.name === VITAL_NAMES.INTERACTION_TO_NEXT_PAINT) + let inpNode = timingsAggregate.events.get().find(tn => tn.name === VITAL_NAMES.INTERACTION_TO_NEXT_PAINT) expect(inpNode).toBeTruthy() expect(inpNode.value).toEqual(8) expect(inpNode.attrs.cls).toEqual(0.1119) diff --git a/tests/components/page_view_timing/index.test.js b/tests/components/page_view_timing/index.test.js index c2b8fe553..e0c46b0c5 100644 --- a/tests/components/page_view_timing/index.test.js +++ b/tests/components/page_view_timing/index.test.js @@ -69,7 +69,7 @@ describe('pvt aggregate tests', () => { pvtAgg.prepareHarvest = jest.fn(() => ({})) }) test('LCP event with CLS attribute', () => { - const timing = find(pvtAgg.timings.buffer, function (t) { + const timing = find(pvtAgg.events.get(), function (t) { return t.name === 'lcp' }) @@ -91,24 +91,24 @@ describe('pvt aggregate tests', () => { }) test('sends expected FI attributes when available', () => { - expect(pvtAgg.timings.hasData).toEqual(true) - const fiPayload = pvtAgg.timings.buffer.find(x => x.name === 'fi') + expect(pvtAgg.events.get().length).toBeTruthy() + const fiPayload = pvtAgg.events.get().find(x => x.name === 'fi') expect(fiPayload.value).toEqual(5) expect(fiPayload.attrs).toEqual(expect.objectContaining({ type: 'pointerdown', fid: 1234, cls: 0.1119, ...expectedNetworkInfo })) }) test('sends CLS node with right val on vis change', () => { - let clsNode = pvtAgg.timings.buffer.find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) + let clsNode = pvtAgg.events.get().find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) expect(clsNode).toBeUndefined() triggerVisChange() - clsNode = pvtAgg.timings.buffer.find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) + clsNode = pvtAgg.events.get().find(tn => tn.name === VITAL_NAMES.CUMULATIVE_LAYOUT_SHIFT) expect(clsNode).toBeTruthy() expect(clsNode.value).toEqual(111.9) // since cls multiply decimal by 1000 to offset consumer division by 1000 expect(clsNode.attrs.cls).toBeUndefined() // cls node doesn't need cls property }) test('sends INP node with right val', () => { - let inpNode = pvtAgg.timings.buffer.find(tn => tn.name === VITAL_NAMES.INTERACTION_TO_NEXT_PAINT) + let inpNode = pvtAgg.events.get().find(tn => tn.name === VITAL_NAMES.INTERACTION_TO_NEXT_PAINT) expect(inpNode).toBeTruthy() expect(inpNode.value).toEqual(8) expect(inpNode.attrs.cls).toEqual(0.1119) diff --git a/tests/components/session_trace/aggregate.test.js b/tests/components/session_trace/aggregate.test.js index 7b9192cc1..c623e8cc5 100644 --- a/tests/components/session_trace/aggregate.test.js +++ b/tests/components/session_trace/aggregate.test.js @@ -29,10 +29,10 @@ test('creates right nodes', async () => { document.dispatchEvent(new CustomEvent('DOMContentLoaded')) // simulate natural browser event window.dispatchEvent(new CustomEvent('load')) // load is actually ignored by Trace as it should be passed by the PVT feature, so it should not be in payload - sessionTraceAggregate.traceStorage.storeXhrAgg('xhr', '[200,null,null]', { method: 'GET', status: 200 }, { rxSize: 770, duration: 99, cbTime: 0, time: 217 }) // fake ajax data - sessionTraceAggregate.traceStorage.processPVT('fi', 30, { fid: 8 }) // fake pvt data + sessionTraceAggregate.events.storeXhrAgg('xhr', '[200,null,null]', { method: 'GET', status: 200 }, { rxSize: 770, duration: 99, cbTime: 0, time: 217 }) // fake ajax data + sessionTraceAggregate.events.processPVT('fi', 30, { fid: 8 }) // fake pvt data - const payload = sessionTraceAggregate.prepareHarvest() + const payload = sessionTraceAggregate.makeHarvestPayload() let res = payload.body let node = res.filter(node => node.n === 'DOMContentLoaded')[0] @@ -69,15 +69,15 @@ test('creates right nodes', async () => { }) test('prepareHarvest returns undefined if there are no trace nodes', () => { - jest.spyOn(sessionTraceAggregate.traceStorage, 'takeSTNs') + jest.spyOn(sessionTraceAggregate.events, 'takeSTNs') - let payload = sessionTraceAggregate.prepareHarvest() + let payload = sessionTraceAggregate.makeHarvestPayload() expect(payload).toBeUndefined() - expect(sessionTraceAggregate.traceStorage.takeSTNs).not.toHaveBeenCalled() + expect(sessionTraceAggregate.events.takeSTNs).not.toHaveBeenCalled() }) test('initialize only ever stores timings once', () => { - const storeTimingSpy = jest.spyOn(sessionTraceAggregate.traceStorage, 'storeTiming') + const storeTimingSpy = jest.spyOn(sessionTraceAggregate.events, 'storeTiming') /** initialize was already called in setup, so we should not see a new call */ sessionTraceAggregate.initialize() expect(storeTimingSpy).toHaveBeenCalledTimes(0) @@ -89,37 +89,37 @@ test('tracks previously stored events and processes them once per occurrence', d document.addEventListener('visibilitychange', () => 3) // additional listeners should not generate additional nodes document.dispatchEvent(new Event('visibilitychange')) - expect(sessionTraceAggregate.traceStorage.trace.visibilitychange[0]).toEqual(expect.objectContaining({ + expect(sessionTraceAggregate.events.trace.visibilitychange[0]).toEqual(expect.objectContaining({ n: 'visibilitychange', t: 'event', o: 'document' })) - expect(sessionTraceAggregate.traceStorage.prevStoredEvents.size).toEqual(1) + expect(sessionTraceAggregate.events.prevStoredEvents.size).toEqual(1) setTimeout(() => { // some time gap document.dispatchEvent(new Event('visibilitychange')) - expect(sessionTraceAggregate.traceStorage.trace.visibilitychange.length).toEqual(2) - expect(sessionTraceAggregate.traceStorage.trace.visibilitychange[0].s).not.toEqual(sessionTraceAggregate.traceStorage.trace.visibilitychange[1].s) // should not have same start times - expect(sessionTraceAggregate.traceStorage.prevStoredEvents.size).toEqual(2) + expect(sessionTraceAggregate.events.trace.visibilitychange.length).toEqual(2) + expect(sessionTraceAggregate.events.trace.visibilitychange[0].s).not.toEqual(sessionTraceAggregate.events.trace.visibilitychange[1].s) // should not have same start times + expect(sessionTraceAggregate.events.prevStoredEvents.size).toEqual(2) done() }, 100) }) test('when max nodes per harvest is reached, no node is further added in FULL mode', () => { - sessionTraceAggregate.traceStorage.nodeCount = MAX_NODES_PER_HARVEST + sessionTraceAggregate.events.nodeCount = MAX_NODES_PER_HARVEST sessionTraceAggregate.mode = MODE.FULL - sessionTraceAggregate.traceStorage.storeSTN({ n: 'someNode', s: 123 }) - expect(sessionTraceAggregate.traceStorage.nodeCount).toEqual(MAX_NODES_PER_HARVEST) - expect(Object.keys(sessionTraceAggregate.traceStorage.trace).length).toEqual(0) + sessionTraceAggregate.events.storeSTN({ n: 'someNode', s: 123 }) + expect(sessionTraceAggregate.events.nodeCount).toEqual(MAX_NODES_PER_HARVEST) + expect(Object.keys(sessionTraceAggregate.events.trace).length).toEqual(0) }) test('when max nodes per harvest is reached, node is still added in ERROR mode', () => { - sessionTraceAggregate.traceStorage.nodeCount = MAX_NODES_PER_HARVEST + sessionTraceAggregate.events.nodeCount = MAX_NODES_PER_HARVEST sessionTraceAggregate.mode = MODE.ERROR - jest.spyOn(sessionTraceAggregate.traceStorage, 'trimSTNs').mockReturnValue(MAX_NODES_PER_HARVEST) + jest.spyOn(sessionTraceAggregate.events, 'trimSTNs').mockReturnValue(MAX_NODES_PER_HARVEST) - sessionTraceAggregate.traceStorage.storeSTN({ n: 'someNode', s: 123 }) - expect(sessionTraceAggregate.traceStorage.nodeCount).toEqual(MAX_NODES_PER_HARVEST + 1) - expect(Object.keys(sessionTraceAggregate.traceStorage.trace).length).toEqual(1) + sessionTraceAggregate.events.storeSTN({ n: 'someNode', s: 123 }) + expect(sessionTraceAggregate.events.nodeCount).toEqual(MAX_NODES_PER_HARVEST + 1) + expect(Object.keys(sessionTraceAggregate.events.trace).length).toEqual(1) }) diff --git a/tests/components/setup-agent.js b/tests/components/setup-agent.js index da709988d..f3de67ff9 100644 --- a/tests/components/setup-agent.js +++ b/tests/components/setup-agent.js @@ -2,10 +2,10 @@ import { faker } from '@faker-js/faker' import { getNREUMInitializedAgent, setNREUMInitializedAgent } from '../../src/common/window/nreum' import { configure } from '../../src/loaders/configure/configure' import { ee } from '../../src/common/event-emitter/contextual-ee' -import { Aggregator } from '../../src/common/aggregate/aggregator' import { TimeKeeper } from '../../src/common/timing/time-keeper' import { getRuntime } from '../../src/common/config/runtime' import { setupAgentSession } from '../../src/features/utils/agent-session' +import { EventAggregator } from '../../src/common/aggregate/event-aggregator' /** * Sets up a new agent for component testing. This should be called only @@ -38,7 +38,7 @@ export function setupAgent ({ agentOverrides = {}, info = {}, init = {}, loaderC const fakeAgent = { agentIdentifier, ee: eventEmitter, - sharedAggregator: new Aggregator(), + sharedAggregator: new EventAggregator(), ...agentOverrides } setNREUMInitializedAgent(agentIdentifier, fakeAgent) @@ -79,7 +79,7 @@ export function resetAgentEventEmitter (agentIdentifier) { export function resetAggregator (agentIdentifier) { const agent = getNREUMInitializedAgent(agentIdentifier) - agent.sharedAggregator.take(Object.keys(agent.sharedAggregator.aggregatedData)) + agent.sharedAggregator.clear() } export function resetSession (agentIdentifier) { diff --git a/tests/components/soft_navigations/aggregate.test.js b/tests/components/soft_navigations/aggregate.test.js index 79c48412a..5d317c567 100644 --- a/tests/components/soft_navigations/aggregate.test.js +++ b/tests/components/soft_navigations/aggregate.test.js @@ -41,7 +41,7 @@ test('processes regular interactions', () => { const ttfbSubscriber = jest.mocked(ttfbModule.timeToFirstByte.subscribe).mock.calls[0][0] ttfbSubscriber({ attrs: { navigationEntry: { loadEventEnd: 123 } } }) expect(softNavAggregate.initialPageLoadInteraction).toBeNull() - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(1) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) softNavAggregate.ee.emit('newURL', [234, '' + window.location]) softNavAggregate.ee.emit('newDom', [235]) @@ -57,8 +57,8 @@ test('processes regular interactions', () => { softNavAggregate.ee.emit('newDom', [348.5]) expect(softNavAggregate.interactionInProgress).toBeNull() expect(softNavAggregate.domObserver.cb).toBeUndefined() // observer should be disconnected after ixn done - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(2) - expect(softNavAggregate.interactionsToHarvest.buffer[1].end).toEqual(348.5) // check end time for the ixn is as expected + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(2) + expect(softNavAggregate.interactionsToHarvest.get()[1].end).toEqual(348.5) // check end time for the ixn is as expected }) test('regular interactions have applicable timeouts', async () => { @@ -71,7 +71,7 @@ test('regular interactions have applicable timeouts', async () => { jest.runAllTimers() expect(softNavAggregate.interactionInProgress).toBeNull() - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(0) // since initialPageLoad ixn hasn't closed, and we expect that UI ixn to have been cancelled + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(0) // since initialPageLoad ixn hasn't closed, and we expect that UI ixn to have been cancelled jest.useRealTimers() }) @@ -103,11 +103,11 @@ test('getInteractionFor grabs the right active interaction for a timestamp', () softNavAggregate.interactionInProgress.forceSave = true expect(softNavAggregate.interactionInProgress.done()).toEqual(true) // this would mark the ixn as finished and queued for harvest - expect(softNavAggregate.getInteractionFor(currentTime)).toBe(softNavAggregate.interactionsToHarvest.buffer[0]) // queued+completed UI interaction is STILL chosen over initialPageLoad + expect(softNavAggregate.getInteractionFor(currentTime)).toBe(softNavAggregate.interactionsToHarvest.get()[0]) // queued+completed UI interaction is STILL chosen over initialPageLoad - softNavAggregate.interactionsToHarvest.buffer[0].status = 'cancelled' + softNavAggregate.interactionsToHarvest.get()[0].status = 'cancelled' expect(softNavAggregate.getInteractionFor(currentTime)).toBe(softNavAggregate.initialPageLoadInteraction) // cancelled ixn not considered (even if queued--not possible atm) - const holdIxn = softNavAggregate.interactionsToHarvest.buffer[softNavAggregate.interactionsToHarvest.buffer.length - 1] + const holdIxn = softNavAggregate.interactionsToHarvest.get()[softNavAggregate.interactionsToHarvest.get().length - 1] expect(softNavAggregate.getInteractionFor(currentTime)).toBe(softNavAggregate.initialPageLoadInteraction) // cancelled (untracked) ixn not considered; falls back to iPL const ttfbSubscriber = jest.mocked(ttfbModule.timeToFirstByte.subscribe).mock.calls[0][0] @@ -116,37 +116,38 @@ test('getInteractionFor grabs the right active interaction for a timestamp', () holdIxn.status = 'finished' // now we have an array of 2: [completed route-change, completed iPL] wherein the route-change duration is wholly within the iPL duration - expect(softNavAggregate.getInteractionFor(currentTime)).toBe(softNavAggregate.interactionsToHarvest.buffer[0]) + expect(softNavAggregate.getInteractionFor(currentTime)).toBe(softNavAggregate.interactionsToHarvest.get()[0]) }) test('interactions are backed up when pre harvesting', () => { const ttfbSubscriber = jest.mocked(ttfbModule.timeToFirstByte.subscribe).mock.calls[0][0] ttfbSubscriber({ attrs: { navigationEntry: { loadEventEnd: performance.now() } } }) - softNavAggregate.onHarvestStarted({ retry: true }) // this flag is on during typical interval harvests except for unload + softNavAggregate.makeHarvestPayload(true) // this flag is on during typical interval harvests except for unload - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(0) - expect(softNavAggregate.interactionsToHarvest.held.buffer.length).toEqual(1) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(0) + softNavAggregate.interactionsToHarvest.reloadSave() + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) }) describe('back up buffer is cleared when', () => { // prevent mem leak beforeEach(() => { const ttfbSubscriber = jest.mocked(ttfbModule.timeToFirstByte.subscribe).mock.calls[0][0] ttfbSubscriber({ attrs: { navigationEntry: { loadEventEnd: performance.now() } } }) - softNavAggregate.onHarvestStarted({ retry: true }) + softNavAggregate.makeHarvestPayload(true) }) - test('harvest was blocked', () => { - softNavAggregate.onHarvestFinished({ sent: false }) // when HTTP status returns 0 - expect(softNavAggregate.interactionsToHarvest.held.buffer.length).toEqual(0) - }) - test('harvest is sent but got a retry response', () => { - softNavAggregate.onHarvestFinished({ sent: true, retry: true }) - expect(softNavAggregate.interactionsToHarvest.held.buffer.length).toEqual(0) - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(1) // ixn goes from backup back into the main pending buffer + test('harvest was blocked, or sent successfully without retry response', () => { + softNavAggregate.postHarvestCleanup(false) // when HTTP status returns 0 + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(0) + + softNavAggregate.interactionsToHarvest.reloadSave() // backup buffer does not hold onto stale data + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(0) }) - test('harvest is sent fully and successfully', () => { - softNavAggregate.onHarvestFinished({ sent: true }) - expect(softNavAggregate.interactionsToHarvest.held.buffer.length).toEqual(0) - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(0) + test('harvest is sent and got a retry response', () => { + softNavAggregate.postHarvestCleanup(true) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) + + softNavAggregate.interactionsToHarvest.reloadSave() // backup buffer does not duplicate retrying data + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) }) }) diff --git a/tests/components/soft_navigations/api.test.js b/tests/components/soft_navigations/api.test.js index 3df00658b..99d66c7ee 100644 --- a/tests/components/soft_navigations/api.test.js +++ b/tests/components/soft_navigations/api.test.js @@ -1,5 +1,4 @@ import { Instrument as SoftNav } from '../../../src/features/soft_navigations/instrument' -import { EventBuffer } from '../../../src/features/utils/event-buffer' import { resetAgent, setupAgent } from '../setup-agent' /** @@ -35,7 +34,6 @@ beforeEach(async () => { softNavAggregate.initialPageLoadInteraction = null softNavAggregate.interactionInProgress = null - softNavAggregate.interactionsToHarvest = new EventBuffer() delete softNavAggregate.latestRouteSetByApi }) @@ -147,19 +145,19 @@ test('.save forcibly harvest any would-be cancelled ixns', async () => { let ixn = newrelic.interaction().save() let ixnContext = getIxnContext(ixn) softNavAggregate.ee.emit(`${INTERACTION_API}-end`, [100], ixnContext) - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(1) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) expect(ixnContext.associatedInteraction.end).toEqual(100) softNavAggregate.ee.emit('newUIEvent', [{ type: 'keydown', timeStamp: 200 }]) ixn = newrelic.interaction().save() ixnContext = getIxnContext(ixn) softNavAggregate.ee.emit('newUIEvent', [{ type: 'keydown', timeStamp: 210 }]) - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(2) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(2) expect(ixnContext.associatedInteraction.end).toBeGreaterThan(ixnContext.associatedInteraction.start) // thisCtx is still referencing the first keydown ixn newrelic.interaction().save().end() await new Promise(process.nextTick) - expect(softNavAggregate.interactionsToHarvest.buffer.length + softNavAggregate.interactionsToHarvest.held.buffer.length).toEqual(3) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(3) }) test('.interaction gets ixn retroactively too when processed late after ee buffer drain', async () => { @@ -168,7 +166,7 @@ test('.interaction gets ixn retroactively too when processed late after ee buffe await new Promise(resolve => setTimeout(resolve, 100)) newrelic.interaction().save().end() - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(1) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) const ixn = newrelic.interaction({ customIxnCreationTime: timeInBtwn }) expect(getIxnContext(ixn).associatedInteraction.trigger).toBe('submit') }) @@ -179,11 +177,11 @@ test('.ignore forcibly discard any would-be harvested ixns', () => { softNavAggregate.ee.emit('newURL', [23, 'example.com']) softNavAggregate.ee.emit('newDom', [34]) expect(softNavAggregate.interactionInProgress).toBeNull() - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(0) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(0) const ixn = newrelic.interaction({ waitForEnd: true }).ignore().save() // ignore ought to override this ixn.end() - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(0) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(0) expect(getIxnContext(ixn).associatedInteraction.status).toEqual('cancelled') }) @@ -218,7 +216,7 @@ test('.onEnd queues callbacks for right before ixn is done', async () => { ixn1.save() // should be able to force save this would-be discarded ixn }).end() expect(hasRan).toEqual(true) - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(1) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) hasRan = false const ixn2 = newrelic.interaction().save() @@ -227,7 +225,7 @@ test('.onEnd queues callbacks for right before ixn is done', async () => { ixn2.ignore() }).end() expect(hasRan).toEqual(true) - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(1) // ixn was discarded + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(1) // ixn was discarded }) test('.setCurrentRouteName updates the targetRouteName of current ixn and is tracked for new ixn', () => { @@ -301,9 +299,9 @@ test('multiple finished ixns retain the correct start/end timestamps in payload' ixnContext.associatedInteraction.forceSave = true softNavAggregate.ee.emit(`${INTERACTION_API}-end`, [1000], ixnContext) - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(3) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(3) // WARN: Double check decoded output & behavior or any introduced bugs before changing the follow line's static string. - expect(softNavAggregate.onHarvestStarted({}).body.e).toEqual("bel.7;1,,2s,2s,,,'api,'http://localhost/,1,1,,2,!!!!'some_id,'1,!!;;1,,5k,5k,,,'api,'http://localhost/,1,1,,2,!!!!'some_other_id,'2,!!;;1,,go,8c,,,'api,'http://localhost/,1,1,,2,!!!!'some_another_id,'3,!!;") + expect(softNavAggregate.makeHarvestPayload().body).toEqual("bel.7;1,,2s,2s,,,'api,'http://localhost/,1,1,,2,!!!!'some_id,'1,!!;;1,,5k,5k,,,'api,'http://localhost/,1,1,,2,!!!!'some_other_id,'2,!!;;1,,go,8c,,,'api,'http://localhost/,1,1,,2,!!!!'some_another_id,'3,!!;") }) // This isn't just an API test; it double serves as data validation on the querypack payload output. @@ -332,9 +330,9 @@ test('multiple finished ixns with ajax have correct start/end timestamps (in aja softNavAggregate.ee.emit('ajax', [{ startTime: 12, endTime: 13 }]) ixnContext.associatedInteraction.children[1].nodeId = 6 - expect(softNavAggregate.interactionsToHarvest.buffer.length).toEqual(2) + expect(softNavAggregate.interactionsToHarvest.get().length).toEqual(2) // WARN: Double check decoded output & behavior or any introduced bugs before changing the follow line's static string. - expect(softNavAggregate.onHarvestStarted({}).body.e).toEqual("bel.7;1,2,1,3,,,'api,'http://localhost/,1,1,,2,!!!!'some_id,'1,!!;2,,1,3,,,,,,,,,,'2,!!!;2,,2,3,,,,,,,,,,'3,!!!;;1,2,9,4,,,'api,'http://localhost/,1,1,,2,!!!!'some_other_id,'4,!!;2,,a,1,,,,,,,,,,'5,!!!;2,,b,1,,,,,,,,,,'6,!!!;") + expect(softNavAggregate.makeHarvestPayload().body).toEqual("bel.7;1,2,1,3,,,'api,'http://localhost/,1,1,,2,!!!!'some_id,'1,!!;2,,1,3,,,,,,,,,,'2,!!!;2,,2,3,,,,,,,,,,'3,!!!;;1,2,9,4,,,'api,'http://localhost/,1,1,,2,!!!!'some_other_id,'4,!!;2,,a,1,,,,,,,,,,'5,!!!;2,,b,1,,,,,,,,,,'6,!!!;") }) function getIxnContext (ixn) { diff --git a/tests/specs/session-trace/modes.e2e.js b/tests/specs/session-trace/modes.e2e.js index bd19a78d0..67224cda4 100644 --- a/tests/specs/session-trace/modes.e2e.js +++ b/tests/specs/session-trace/modes.e2e.js @@ -130,7 +130,7 @@ describe('respects feature flags', () => { const nodeCount = await browser.execute(function () { try { - return Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.traceStorage.nodeCount + return Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.events.nodeCount } catch (err) { return 0 } @@ -166,7 +166,7 @@ describe('respects feature flags', () => { const nodeCount = await browser.execute(function () { try { - return Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.traceStorage.nodeCount + return Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.events.nodeCount } catch (err) { return 0 } diff --git a/tests/specs/session-trace/trace-nodes.e2e.js b/tests/specs/session-trace/trace-nodes.e2e.js index 0f6aa5492..b7699693e 100644 --- a/tests/specs/session-trace/trace-nodes.e2e.js +++ b/tests/specs/session-trace/trace-nodes.e2e.js @@ -26,7 +26,7 @@ describe('Trace nodes', () => { const [sessionTraceHarvests] = await Promise.all([ sessionTraceCapture.waitForResult({ timeout: 10000 }), browser.execute(function () { - const storedEvents = Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.traceStorage.prevStoredEvents + const storedEvents = Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.events.prevStoredEvents for (let i = 0; i < 10; i++) storedEvents.add(i) // artificially add "events" since the counter is otherwise unreliable }).then(() => $('#btn1').click() // since the agent has multiple listeners on vischange, this is a good example of often duplicated event @@ -129,5 +129,5 @@ describe('Trace nodes', () => { }) function getEventsSetSize () { - return Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.traceStorage.prevStoredEvents.size + return Object.values(newrelic.initializedAgents)[0].features.session_trace.featAggregate.events.prevStoredEvents.size } diff --git a/tests/unit/common/aggregate/aggregator.test.js b/tests/unit/common/aggregate/aggregator.test.js index 8779f2c7b..f1ff9d0ab 100644 --- a/tests/unit/common/aggregate/aggregator.test.js +++ b/tests/unit/common/aggregate/aggregator.test.js @@ -2,37 +2,37 @@ import { Aggregator } from '../../../../src/common/aggregate/aggregator.js' let aggregator beforeEach(() => { - aggregator = new Aggregator({ agentIdentifier: 'blah' }) + aggregator = new Aggregator() }) test('storing and getting buckets', () => { let bucket = aggregator.store('abc', '123') const expectedBucketPattern = { params: {}, metrics: { count: 1 } } expect(bucket).toEqual(expectedBucketPattern) - expect(aggregator.get('abc', '123')).toEqual(expectedBucketPattern) + expect(aggregator.aggregatedData.abc['123']).toEqual(expectedBucketPattern) aggregator.store('abc', '456') // check we can get all buckets under the same type - expect(aggregator.get('abc')).toEqual({ 123: expectedBucketPattern, 456: expectedBucketPattern }) + expect(aggregator.aggregatedData.abc).toEqual({ 123: expectedBucketPattern, 456: expectedBucketPattern }) }) describe('storing the same bucket again', () => { test('increments the count', () => { aggregator.store('abc', '123') aggregator.store('abc', '123') - expect(aggregator.get('abc', '123')).toEqual({ params: {}, metrics: { count: 2 } }) + expect(aggregator.aggregatedData.abc['123']).toEqual({ params: {}, metrics: { count: 2 } }) }) test('does not overwrite params', () => { aggregator.store('def', '123', { someParam: true }) aggregator.store('def', '123', { anotherParam: true }) - expect(aggregator.get('def', '123')).toEqual({ params: { someParam: true }, metrics: { count: 2 } }) + expect(aggregator.aggregatedData.def['123']).toEqual({ params: { someParam: true }, metrics: { count: 2 } }) }) test('does not overwrite custom params either', () => { aggregator.store('ghi', '123', undefined, undefined, { someCustomParam: true }) aggregator.store('ghi', '123', undefined, undefined, { someCustomParam: false }) - expect(aggregator.get('ghi', '123')).toEqual({ params: {}, custom: { someCustomParam: true }, metrics: { count: 2 } }) + expect(aggregator.aggregatedData.ghi['123']).toEqual({ params: {}, custom: { someCustomParam: true }, metrics: { count: 2 } }) }) }) @@ -48,7 +48,7 @@ describe('metrics are properly updated', () => { } aggregator.store('abc', '123', undefined, { met1: 1, met2: 3 }) aggregator.store('abc', '123', undefined, { met1: 2, met2: 4 }) - expect(aggregator.get('abc', '123')).toEqual(expectedBucketPattern) + expect(aggregator.aggregatedData.abc['123']).toEqual(expectedBucketPattern) }) test('when using storeMetric fn', () => { @@ -59,7 +59,7 @@ describe('metrics are properly updated', () => { aggregator.storeMetric('abc', 'metric', undefined, 2) aggregator.storeMetric('abc', 'metric', undefined, 1) aggregator.storeMetric('abc', 'metric', undefined, 3) - expect(aggregator.get('abc', 'metric')).toEqual(expectedBucketPattern) + expect(aggregator.aggregatedData.abc.metric).toEqual(expectedBucketPattern) }) test('when using merge fn', () => { @@ -74,7 +74,7 @@ describe('metrics are properly updated', () => { aggregator.store('abc', 'metric', { other: 'blah' }, { met1: 1, met2: 3 }) aggregator.merge('abc', 'metric', { count: 2, met1: { t: 2 }, met2: { t: 4 } }) aggregator.merge('abc', 'metric', { count: 2, met1: { t: 5, min: 3, max: 6, c: 2, sos: 30 }, met2: { t: 7 } }) - expect(aggregator.get('abc', 'metric')).toEqual(expectedBucketPattern) + expect(aggregator.aggregatedData.abc.metric).toEqual(expectedBucketPattern) }) }) @@ -88,8 +88,8 @@ test('take fn gets and deletes correctly', () => { let obj = aggregator.take(['type1', 'type2']) expect(obj.type1.length).toEqual(2) expect(obj.type2.length).toEqual(1) - expect(aggregator.get('type1', 'a')).toBeUndefined() // should be gone now - expect(aggregator.get('type3', 'a')).toEqual(expect.any(Object)) + expect(aggregator.aggregatedData.type1?.a).toBeUndefined() // should be gone now + expect(aggregator.aggregatedData.type3?.a).toEqual(expect.any(Object)) }) test('merge fn combines metrics correctly', () => { @@ -103,7 +103,7 @@ test('merge fn combines metrics correctly', () => { expect(bucket.metrics).toEqual(expectedMetrics) aggregator.merge('abc', '456', bucket.metrics) - expect(aggregator.get('abc', '456').metrics).toEqual(expectedMetrics) + expect(aggregator.aggregatedData.abc['456'].metrics).toEqual(expectedMetrics) aggregator.merge('abc', '123', { count: 4, @@ -112,7 +112,7 @@ test('merge fn combines metrics correctly', () => { met3: { t: 6 }, met4: { t: 7, min: 3, max: 4, sos: 25, c: 2 } }) - expect(aggregator.get('abc', '123').metrics).toEqual({ + expect(aggregator.aggregatedData.abc['123'].metrics).toEqual({ count: 6, met1: { t: 7, min: 0, max: 4, sos: 21, c: 4 }, met2: { t: 8, min: 3, max: 5, sos: 34, c: 2 }, diff --git a/tests/unit/common/harvest/harvest-scheduler.test.js b/tests/unit/common/harvest/harvest-scheduler.test.js index 7db902b60..02df0c209 100644 --- a/tests/unit/common/harvest/harvest-scheduler.test.js +++ b/tests/unit/common/harvest/harvest-scheduler.test.js @@ -180,7 +180,7 @@ describe('runHarvest', () => { test('should not run harvest when scheduler is aborted', () => { harvestSchedulerInstance.aborted = true - harvestSchedulerInstance.runHarvest({}) + harvestSchedulerInstance.runHarvest() expect(harvestInstance.sendX).not.toHaveBeenCalled() expect(harvestInstance.send).not.toHaveBeenCalled() @@ -269,7 +269,7 @@ describe('runHarvest', () => { harvestSchedulerInstance.started = true harvestSchedulerInstance.opts.getPayload = jest.fn().mockReturnValue() - harvestSchedulerInstance.runHarvest({}) + harvestSchedulerInstance.runHarvest() expect(harvestInstance.sendX).not.toHaveBeenCalled() expect(harvestInstance.send).not.toHaveBeenCalled() diff --git a/tests/unit/common/harvest/harvest.test.js b/tests/unit/common/harvest/harvest.test.js index 4d92d7cac..ce1c7cd35 100644 --- a/tests/unit/common/harvest/harvest.test.js +++ b/tests/unit/common/harvest/harvest.test.js @@ -273,13 +273,13 @@ describe('_send', () => { test('should set body to events when endpoint is events', () => { spec.endpoint = 'events' - spec.payload.body.e = faker.lorem.sentence() + spec.payload.body = faker.lorem.sentence() const result = harvestInstance._send(spec) expect(result).toEqual(true) expect(submitMethod).toHaveBeenCalledWith({ - body: spec.payload.body.e, + body: spec.payload.body, headers: [{ key: 'content-type', value: 'text/plain' }], sync: undefined, url: expect.stringContaining(`https://${errorBeacon}/${spec.endpoint}/1/${licenseKey}?`) diff --git a/tests/unit/features/page_view_timing/aggregate/index.test.js b/tests/unit/features/page_view_timing/aggregate/index.test.js index a40a4579e..6cb5bccd5 100644 --- a/tests/unit/features/page_view_timing/aggregate/index.test.js +++ b/tests/unit/features/page_view_timing/aggregate/index.test.js @@ -21,7 +21,8 @@ describe('PVT aggregate', () => { testCases().forEach(testCase => { const expectedPayload = qp.encode(testCase.input, schema) - const payload = pvtAgg.getPayload(getAgentInternalFormat(testCase.input)) + console.log(pvtAgg.serializer) + const payload = pvtAgg.serializer(getAgentInternalFormat(testCase.input)) expect(payload).toEqual(expectedPayload) }) }) @@ -31,7 +32,7 @@ describe('PVT aggregate', () => { pvtAgg.agentRef.info.jsAttributes = { custom: 'val', cls: 'customVal' } testCases().forEach(testCase => { - const payload = pvtAgg.getPayload(getAgentInternalFormat(testCase.input)) + const payload = pvtAgg.serializer(getAgentInternalFormat(testCase.input)) const events = qp.decode(payload) const hasReserved = overriddenReservedAttributes(events) const result = haveCustomAttributes(events) @@ -43,24 +44,24 @@ describe('PVT aggregate', () => { test('addConnectionAttributes', () => { global.navigator.connection = {} pvtAgg.addTiming('abc', 1) - expect(pvtAgg.timings.buffer[0].attrs).toEqual(expect.objectContaining({})) + expect(pvtAgg.events.get()[0].attrs).toEqual(expect.objectContaining({})) global.navigator.connection.type = 'type' pvtAgg.addTiming('abc', 1) - expect(pvtAgg.timings.buffer[1].attrs).toEqual(expect.objectContaining({ + expect(pvtAgg.events.get()[1].attrs).toEqual(expect.objectContaining({ 'net-type': 'type' })) global.navigator.connection.effectiveType = 'effectiveType' pvtAgg.addTiming('abc', 1) - expect(pvtAgg.timings.buffer[2].attrs).toEqual(expect.objectContaining({ + expect(pvtAgg.events.get()[2].attrs).toEqual(expect.objectContaining({ 'net-type': 'type', 'net-etype': 'effectiveType' })) global.navigator.connection.rtt = 'rtt' pvtAgg.addTiming('abc', 1) - expect(pvtAgg.timings.buffer[3].attrs).toEqual(expect.objectContaining({ + expect(pvtAgg.events.get()[3].attrs).toEqual(expect.objectContaining({ 'net-type': 'type', 'net-etype': 'effectiveType', 'net-rtt': 'rtt' @@ -68,7 +69,7 @@ describe('PVT aggregate', () => { global.navigator.connection.downlink = 'downlink' pvtAgg.addTiming('abc', 1) - expect(pvtAgg.timings.buffer[4].attrs).toEqual(expect.objectContaining({ + expect(pvtAgg.events.get()[4].attrs).toEqual(expect.objectContaining({ 'net-type': 'type', 'net-etype': 'effectiveType', 'net-rtt': 'rtt', @@ -83,7 +84,7 @@ describe('PVT aggregate', () => { } pvtAgg.addTiming('abc', 1) - expect(pvtAgg.timings.buffer[5].attrs).toEqual(expect.objectContaining({ + expect(pvtAgg.events.get()[5].attrs).toEqual(expect.objectContaining({ 'net-type': 'type', 'net-etype': 'effectiveType', 'net-rtt': 'rtt', diff --git a/tests/unit/features/utils/event-buffer.test.js b/tests/unit/features/utils/event-buffer.test.js index 97cce624b..3924052a0 100644 --- a/tests/unit/features/utils/event-buffer.test.js +++ b/tests/unit/features/utils/event-buffer.test.js @@ -1,177 +1,107 @@ +import { MAX_PAYLOAD_SIZE } from '../../../../src/common/constants/agent-constants' import { EventBuffer } from '../../../../src/features/utils/event-buffer' let eventBuffer describe('EventBuffer', () => { beforeEach(() => { - eventBuffer = new EventBuffer() + eventBuffer = new EventBuffer(MAX_PAYLOAD_SIZE) }) it('has default values', () => { - expect(eventBuffer).toMatchObject({ - bytes: 0, - buffer: [] + expect(eventBuffer).toEqual({ + maxPayloadSize: expect.any(Number) }) }) describe('add', () => { it('should add data to the buffer while maintaining size', () => { - expect(eventBuffer).toMatchObject({ - bytes: 0, - buffer: [] - }) + expect([eventBuffer.isEmpty(), eventBuffer.byteSize()]).toEqual([true, 0]) const mockEvent = { test: 1 } eventBuffer.add(mockEvent) - expect(eventBuffer).toMatchObject({ - bytes: JSON.stringify(mockEvent).length, - buffer: [mockEvent] - }) + expect([eventBuffer.get(), eventBuffer.byteSize()]).toEqual([[mockEvent], JSON.stringify(mockEvent).length]) }) it('should not add if one event is too large', () => { - eventBuffer.add({ test: 'x'.repeat(1000000) }) - expect(eventBuffer.buffer).toEqual([]) + expect(eventBuffer.add({ test: 'x'.repeat(MAX_PAYLOAD_SIZE) })).toEqual(false) // exceeds because of quote chars + expect(eventBuffer.isEmpty()).toEqual(true) }) it('should not add if existing buffer would become too large', () => { eventBuffer.add({ test: 'x'.repeat(999988) }) - expect(eventBuffer.bytes).toEqual(999999) - expect(eventBuffer.buffer.length).toEqual(1) + expect(eventBuffer.isEmpty()).toEqual(false) + expect(eventBuffer.byteSize()).toEqual(999999) eventBuffer.add({ test2: 'testing' }) - expect(eventBuffer.bytes).toEqual(999999) - expect(eventBuffer.buffer.length).toEqual(1) - }) - - it('should be chainable', () => { - const mockEvent1 = { test: 1 } - const mockEvent2 = { test: 2 } - eventBuffer.add(mockEvent1).add(mockEvent2) - expect(eventBuffer).toMatchObject({ - bytes: JSON.stringify(mockEvent1).length + JSON.stringify(mockEvent2).length, - buffer: [mockEvent1, mockEvent2] - }) + expect(eventBuffer.byteSize()).toEqual(999999) + expect(eventBuffer.get().length).toEqual(1) }) }) - describe('merge', () => { - it('should merge two EventBuffers - append', () => { - const mockEvent1 = { test: 1 } - const mockEvent2 = { test: 2 } - eventBuffer.add(mockEvent1) + test('wouldExceedMaxSize returns boolean and does not actually add', () => { + expect(eventBuffer.wouldExceedMaxSize(MAX_PAYLOAD_SIZE + 1)).toEqual(true) + expect(eventBuffer.wouldExceedMaxSize(MAX_PAYLOAD_SIZE)).toEqual(false) + expect(eventBuffer.isEmpty()).toEqual(true) - const secondBuffer = new EventBuffer() - secondBuffer.add(mockEvent2) - eventBuffer.merge(secondBuffer) - expect(eventBuffer).toMatchObject({ - bytes: JSON.stringify({ test: 1 }).length + JSON.stringify({ test: 2 }).length, - buffer: [mockEvent1, mockEvent2] - }) - }) - - it('should merge two EventBuffers - prepend', () => { - const mockEvent1 = { test: 1 } - const mockEvent2 = { test: 2 } - eventBuffer.add(mockEvent1) + expect(eventBuffer.add('x'.repeat(MAX_PAYLOAD_SIZE - 2))).toEqual(true) // the 2 bytes are for the quotes + expect(eventBuffer.wouldExceedMaxSize(1)).toEqual(true) + expect(eventBuffer.byteSize()).toEqual(MAX_PAYLOAD_SIZE) + }) - const secondBuffer = new EventBuffer() - secondBuffer.add(mockEvent2) - eventBuffer.merge(secondBuffer, true) - expect(eventBuffer).toMatchObject({ - bytes: JSON.stringify({ test: 1 }).length + JSON.stringify({ test: 2 }).length, - buffer: [mockEvent2, mockEvent1] - }) - }) + test('clear wipes the buffer', () => { + eventBuffer.add('test') + expect(eventBuffer.isEmpty()).toEqual(false) + expect(eventBuffer.byteSize()).toBeGreaterThan(0) + eventBuffer.clear() + expect(eventBuffer.isEmpty()).toEqual(true) + expect(eventBuffer.byteSize()).toEqual(0) + }) - it('should not merge if not an EventBuffer', () => { - eventBuffer.add({ test: 1 }) - // not EventBuffer - eventBuffer.merge({ regular: 'object' }) - expect(eventBuffer.buffer).toEqual([{ test: 1 }]) - // not EventBuffer - eventBuffer.merge('string') - expect(eventBuffer.buffer).toEqual([{ test: 1 }]) - // not EventBuffer - eventBuffer.merge(123) - expect(eventBuffer.buffer).toEqual([{ test: 1 }]) - // not EventBuffer - eventBuffer.merge(true) - expect(eventBuffer.buffer).toEqual([{ test: 1 }]) - // not EventBuffer - eventBuffer.merge(Symbol('test')) - expect(eventBuffer.buffer).toEqual([{ test: 1 }]) + describe('save', () => { + test('does not clear the buffer on its own', () => { + eventBuffer.add('test') + eventBuffer.save() + expect(eventBuffer.isEmpty()).toEqual(false) }) - it('should not merge if too big', () => { - const mockEvent1 = { test: 'x'.repeat(999988) } - const mockEvent2 = { test2: 'testing' } - eventBuffer.add(mockEvent1) - - const secondBuffer = new EventBuffer() - secondBuffer.add(mockEvent2) - - eventBuffer.merge(secondBuffer) - expect(eventBuffer.buffer.length).toEqual(1) - expect(eventBuffer.bytes).toEqual(999999) + test('can be reloaded after clearing', () => { + eventBuffer.add('test') + eventBuffer.save() + eventBuffer.clear() + expect(eventBuffer.isEmpty()).toEqual(true) + expect(eventBuffer.byteSize()).toEqual(0) + eventBuffer.reloadSave() + expect(eventBuffer.isEmpty()).toEqual(false) + expect(eventBuffer.byteSize()).toEqual(6) }) - it('should be chainable', () => { - const mockEvent1 = { test1: 1 } - const mockEvent2 = { test2: 2 } - const mockEvent3 = { test3: 3 } - - const secondBuffer = new EventBuffer() - const thirdBuffer = new EventBuffer() - - eventBuffer.add(mockEvent1) - secondBuffer.add(mockEvent2) - thirdBuffer.add(mockEvent3) - - eventBuffer.merge(secondBuffer).merge(thirdBuffer) - expect(eventBuffer).toMatchObject({ - bytes: JSON.stringify(mockEvent1).length + JSON.stringify(mockEvent2).length + JSON.stringify(mockEvent3).length, - buffer: [mockEvent1, mockEvent2, mockEvent3] - }) + test('can be reloaded without clearing (doubled data)', () => { + eventBuffer.add('test') + eventBuffer.save() + eventBuffer.reloadSave() + expect(eventBuffer.get().length).toEqual(2) + expect(eventBuffer.byteSize()).toEqual(12) }) }) - describe('hasData', () => { - it('should return false if no events', () => { - jest.spyOn(eventBuffer, 'bytes', 'get').mockReturnValue(100) - expect(eventBuffer.hasData).toEqual(false) - }) - it('should return false if no bytes', () => { - jest.spyOn(eventBuffer, 'buffer', 'get').mockReturnValue({ test: 1 }) - expect(eventBuffer.hasData).toEqual(false) - }) - it('should return true if has a valid event and size', () => { - eventBuffer.add({ test: 1 }) - expect(eventBuffer.hasData).toEqual(true) - }) + test('clearSave will clear previous save calls', () => { + eventBuffer.add('test') + eventBuffer.save() + eventBuffer.clearSave() // should nullify previous step + eventBuffer.reloadSave() + expect(eventBuffer.get().length).toEqual(1) + expect(eventBuffer.byteSize()).toEqual(6) }) - describe('canMerge', () => { - it('should return false if would be too big', () => { - jest.spyOn(eventBuffer, 'bytes', 'get').mockReturnValue(999999) - expect(eventBuffer.canMerge(1)).toEqual(false) - }) - it('should return false if no size provided', () => { - eventBuffer.add({ test: 1 }) - expect(eventBuffer.canMerge()).toEqual(false) - }) - it('should return false if 0 size provided', () => { - eventBuffer.add({ test: 1 }) - expect(eventBuffer.canMerge(0)).toEqual(false) - }) - it('should return false if size is not a number', () => { - eventBuffer.add({ test: 1 }) - expect(eventBuffer.canMerge('test')).toEqual(false) - expect(eventBuffer.canMerge(false)).toEqual(false) - expect(eventBuffer.canMerge(['test'])).toEqual(false) - expect(eventBuffer.canMerge({ test: 1 })).toEqual(false) - }) - it('should return true if has a valid event and size', () => { - eventBuffer.add({ test: 1 }) - expect(eventBuffer.canMerge(20)).toEqual(true) - }) + test('reloadSave will not reload if final size exceeds limit', () => { + expect(eventBuffer.add('x'.repeat(MAX_PAYLOAD_SIZE - 2))).toEqual(true) + eventBuffer.save() + eventBuffer.clear() + expect(eventBuffer.add('x')).toEqual(true) + expect(eventBuffer.get()).toEqual(['x']) + eventBuffer.reloadSave() + expect(eventBuffer.get()).toEqual(['x']) // should not have reloaded because combined would exceed max + eventBuffer.clear() // now the buffer is emptied + eventBuffer.reloadSave() // so this should work now + expect(eventBuffer.get()).toEqual(['x'.repeat(MAX_PAYLOAD_SIZE - 2)]) }) }) diff --git a/tests/unit/features/utils/instrument-base.test.js b/tests/unit/features/utils/instrument-base.test.js index ebc6a84bd..b7d0bcd3b 100644 --- a/tests/unit/features/utils/instrument-base.test.js +++ b/tests/unit/features/utils/instrument-base.test.js @@ -10,7 +10,7 @@ import { warn } from '../../../../src/common/util/console' import * as runtimeConstantsModule from '../../../../src/common/constants/runtime' import { canEnableSessionTracking } from '../../../../src/features/utils/feature-gates' import { getConfigurationValue } from '../../../../src/common/config/init' -import { Aggregator } from '../../../../src/common/aggregate/aggregator' +import { EventAggregator } from '../../../../src/common/aggregate/event-aggregator' jest.enableAutomock() jest.unmock('../../../../src/features/utils/instrument-base') @@ -183,12 +183,12 @@ test('does not initialized Aggregator more than once with multiple features', as pve.importAggregator(agentBase) pvt.importAggregator(agentBase) - expect(Aggregator).toHaveBeenCalledTimes(0) + expect(EventAggregator).toHaveBeenCalledTimes(0) await Promise.all([ jest.mocked(onWindowLoad).mock.calls[0][0](), // PVE should import & initialize Aggregator jest.mocked(onWindowLoad).mock.calls[1][0]() // and PVT should wait for PVE to do that instead of initializing it again ]) - expect(Aggregator).toHaveBeenCalledTimes(1) + expect(EventAggregator).toHaveBeenCalledTimes(1) }) test('does initialize separate Aggregators with multiple agents', async () => { @@ -203,10 +203,10 @@ test('does initialize separate Aggregators with multiple agents', async () => { pve.importAggregator(agentBase) pve2.importAggregator(agentBase2) - expect(Aggregator).toHaveBeenCalledTimes(0) + expect(EventAggregator).toHaveBeenCalledTimes(0) await Promise.all([ jest.mocked(onWindowLoad).mock.calls[0][0](), - jest.mocked(onWindowLoad).mock.calls[1][0]() // second agent PVE reusing same module should also initialize a new Aggregator + jest.mocked(onWindowLoad).mock.calls[1][0]() // second agent PVE reusing same module should also initialize a new EventAggregator ]) - expect(Aggregator).toHaveBeenCalledTimes(2) + expect(EventAggregator).toHaveBeenCalledTimes(2) })