From 59c04f9a177883d64689a71a99e3652a2a83d2d2 Mon Sep 17 00:00:00 2001 From: David Buezas Date: Tue, 15 Nov 2022 21:21:35 +0100 Subject: [PATCH] Perf/avoid redundant fetches (#144) * Only fetch statistics once per entity, even if multiple traces show different ones (e.g max, min, ...) * Same for multiple attributes of the same entity --- src/cache/Cache.ts | 100 ++++++++++++++++++++++------------ src/cache/fetch-states.ts | 24 +++----- src/cache/fetch-statistics.ts | 17 +++--- src/plotly-graph-card.ts | 15 +++-- src/types.ts | 16 ++++-- 5 files changed, 101 insertions(+), 71 deletions(-) diff --git a/src/cache/Cache.ts b/src/cache/Cache.ts index fc23971..70f9239 100644 --- a/src/cache/Cache.ts +++ b/src/cache/Cache.ts @@ -8,9 +8,12 @@ import { EntityConfig, isEntityIdStateConfig, isEntityIdStatisticsConfig, - HistoryInRange, - EntityState, + CachedEntity, + CachedStatisticsEntity, + CachedStateEntity, } from "../types"; +import { groupBy } from "lodash"; +import { StatisticValue } from "../recorder-types"; export function mapValues( o: Record, @@ -24,7 +27,10 @@ async function fetchSingleRange( [startT, endT]: number[], significant_changes_only: boolean, minimal_response: boolean -): Promise { +): Promise<{ + range: [number, number]; + history: CachedEntity[]; +}> { // We fetch slightly more than requested (i.e the range visible in the screen). The reason is the following: // When fetching data in a range `[startT,endT]`, Home Assistant adds a fictitious datapoint at // the start of the fetched period containing a copy of the first datapoint that occurred before @@ -65,7 +71,7 @@ async function fetchSingleRange( const start = new Date(startT - 1); endT = Math.min(endT, Date.now()); const end = new Date(endT); - let history: EntityState[]; + let history: CachedEntity[]; if (isEntityIdStatisticsConfig(entity)) { history = await fetchStatistics(hass, entity, [start, end]); } else { @@ -90,9 +96,9 @@ async function fetchSingleRange( export function getEntityKey(entity: EntityConfig) { if (isEntityIdAttrConfig(entity)) { - return `${entity.entity}::${entity.attribute}`; + return `${entity.entity}::attribute`; } else if (isEntityIdStatisticsConfig(entity)) { - return `${entity.entity}::statistics::${entity.statistic}::${entity.period}`; + return `${entity.entity}::statistics::${entity.period}`; } else if (isEntityIdStateConfig(entity)) { return entity.entity; } @@ -102,10 +108,10 @@ export function getEntityKey(entity: EntityConfig) { const MIN_SAFE_TIMESTAMP = Date.parse("0001-01-02T00:00:00.000Z"); export default class Cache { ranges: Record = {}; - histories: Record = {}; + histories: Record = {}; busy = Promise.resolve(); // mutex - add(entity: EntityConfig, states: EntityState[], range: [number, number]) { + add(entity: EntityConfig, states: CachedEntity[], range: [number, number]) { const entityKey = getEntityKey(entity); let h = (this.histories[entityKey] ??= []); h.push(...states); @@ -122,13 +128,33 @@ export default class Cache { this.ranges = {}; this.histories = {}; } - getHistory(entity: EntityConfig) { + getHistory(entity: EntityConfig): CachedEntity[] { let key = getEntityKey(entity); const history = this.histories[key] || []; - return history.map((datum) => ({ - ...datum, - timestamp: datum.timestamp + entity.offset, - })); + if (isEntityIdStatisticsConfig(entity)) { + return (history as CachedStatisticsEntity[]).map((entry) => ({ + ...entry, + timestamp: entry.timestamp + entity.offset, + value: entry[entity.statistic], + })); + } + if (isEntityIdAttrConfig(entity)) { + return (history as CachedStateEntity[]).map((entry) => ({ + ...entry, + timestamp: entry.timestamp + entity.offset, + value: entry.attributes[entity.attribute], + })); + } + if (isEntityIdStateConfig(entity)) { + return (history as CachedStateEntity[]).map((entry) => ({ + ...entry, + timestamp: entry.timestamp + entity.offset, + value: entry.state, + })); + } + throw new Error( + `Unrecognised fetch type for ${(entity as EntityConfig).entity}` + ); } async update( range: TimestampRange, @@ -137,31 +163,37 @@ export default class Cache { significant_changes_only: boolean, minimal_response: boolean ) { - range = range.map((n) => Math.max(MIN_SAFE_TIMESTAMP, n)); // HA API can't handle negative years return (this.busy = this.busy .catch(() => {}) .then(async () => { - const promises = entities.map(async (entity) => { - const entityKey = getEntityKey(entity); - this.ranges[entityKey] ??= []; - const offsetRange = [ - range[0] - entity.offset, - range[1] - entity.offset, - ]; - const rangesToFetch = subtractRanges( - [offsetRange], - this.ranges[entityKey] - ); - for (const aRange of rangesToFetch) { - const fetchedHistory = await fetchSingleRange( - hass, - entity, - aRange, - significant_changes_only, - minimal_response + range = range.map((n) => Math.max(MIN_SAFE_TIMESTAMP, n)); // HA API can't handle negative years + const parallelFetches = Object.values(groupBy(entities, getEntityKey)); + const promises = parallelFetches.flatMap(async (entityGroup) => { + // Each entity in entityGroup will result in exactly the same fetch + // But these may differ once the offsets PR is merged + // Making these fetches sequentially ensures that the already fetched ranges of each + // request are not fetched more than once + for (const entity of entityGroup) { + const entityKey = getEntityKey(entity); + this.ranges[entityKey] ??= []; + const offsetRange = [ + range[0] - entity.offset, + range[1] - entity.offset, + ]; + const rangesToFetch = subtractRanges( + [offsetRange], + this.ranges[entityKey] ); - if (fetchedHistory === null) continue; - this.add(entity, fetchedHistory.history, fetchedHistory.range); + for (const aRange of rangesToFetch) { + const fetchedHistory = await fetchSingleRange( + hass, + entity, + aRange, + significant_changes_only, + minimal_response + ); + this.add(entity, fetchedHistory.history, fetchedHistory.range); + } } }); diff --git a/src/cache/fetch-states.ts b/src/cache/fetch-states.ts index 5692a37..b6e7fbc 100644 --- a/src/cache/fetch-states.ts +++ b/src/cache/fetch-states.ts @@ -1,11 +1,10 @@ import { HomeAssistant } from "custom-card-helpers"; import { + CachedEntity, EntityIdAttrConfig, EntityIdStateConfig, - EntityState, HassEntity, isEntityIdAttrConfig, - isEntityIdStateConfig, } from "../types"; async function fetchStates( @@ -14,16 +13,14 @@ async function fetchStates( [start, end]: [Date, Date], significant_changes_only?: boolean, minimal_response?: boolean -): Promise { - const no_attributes_query = isEntityIdStateConfig(entity) - ? "no_attributes&" - : ""; +): Promise { + const no_attributes_query = isEntityIdAttrConfig(entity) + ? "" + : "no_attributes&"; const minimal_response_query = - minimal_response && isEntityIdStateConfig(entity) - ? "minimal_response&" - : ""; + minimal_response && isEntityIdAttrConfig(entity) ? "" : "minimal_response&"; const significant_changes_only_query = - significant_changes_only && isEntityIdStateConfig(entity) ? "1" : "0"; + significant_changes_only && isEntityIdAttrConfig(entity) ? "0" : "1"; const uri = `history/period/${start.toISOString()}?` + `filter_entity_id=${entity.entity}&` + @@ -43,14 +40,11 @@ async function fetchStates( )}` ); } - if (!list) list = []; - return list + return (list || []) .map((entry) => ({ ...entry, - value: isEntityIdAttrConfig(entity) - ? entry.attributes[entity.attribute] || null - : entry.state, timestamp: +new Date(entry.last_updated || entry.last_changed), + value: "", // may be state or an attribute. Will be set when getting the history })) .filter(({ timestamp }) => timestamp); } diff --git a/src/cache/fetch-statistics.ts b/src/cache/fetch-statistics.ts index 251d25a..5cfc411 100644 --- a/src/cache/fetch-statistics.ts +++ b/src/cache/fetch-statistics.ts @@ -1,12 +1,12 @@ import { HomeAssistant } from "custom-card-helpers"; import { Statistics, StatisticValue } from "../recorder-types"; -import { EntityIdStatisticsConfig, EntityState } from "../types"; +import { CachedEntity, EntityIdStatisticsConfig } from "../types"; async function fetchStatistics( hass: HomeAssistant, entity: EntityIdStatisticsConfig, [start, end]: [Date, Date] -): Promise { +): Promise { let statistics: StatisticValue[] | null = null; try { const statsP = hass.callWS({ @@ -25,11 +25,12 @@ async function fetchStatistics( )}` ); } - if (!statistics) statistics = []; //throw new Error(`Error fetching ${entity.entity}`); - return statistics.map((entry) => ({ - ...entry, - timestamp: +new Date(entry.start), - value: entry[entity.statistic] ?? "", - })); + return (statistics || []) + .map((entry) => ({ + ...entry, + timestamp: +new Date(entry.start), + value: "", //depends on the statistic, will be set in getHistory + })) + .filter(({ timestamp }) => timestamp); } export default fetchStatistics; diff --git a/src/plotly-graph-card.ts b/src/plotly-graph-card.ts index c8b5e59..94ac37d 100644 --- a/src/plotly-graph-card.ts +++ b/src/plotly-graph-card.ts @@ -8,7 +8,6 @@ import Plotly from "./plotly"; import { Config, EntityConfig, - EntityState, InputConfig, isEntityIdAttrConfig, isEntityIdStateConfig, @@ -208,6 +207,7 @@ export class PlotlyGraph extends HTMLElement { } else if (isEntityIdStatisticsConfig(entity)) { shouldFetch = true; } + if (value !== undefined) { this.cache.add( entity, @@ -220,8 +220,7 @@ export class PlotlyGraph extends HTMLElement { } if (shouldFetch) { this.fetch(); - } - if (shouldPlot) { + } else if (shouldPlot) { this.plot(); } } @@ -229,7 +228,6 @@ export class PlotlyGraph extends HTMLElement { } connectedCallback() { this.setupListeners(); - this.fetch().then(() => (this.contentEl.style.visibility = "")); } async withoutRelayout(fn: Function) { this.isInternalRelayout++; @@ -741,14 +739,15 @@ export class PlotlyGraph extends HTMLElement { if (layout.paper_bgcolor) { this.titleEl.style.background = layout.paper_bgcolor as string; } - await this.withoutRelayout(() => - Plotly.react( + await this.withoutRelayout(async () => { + await Plotly.react( this.contentEl, this.getData(), layout, this.getPlotlyConfig() - ) - ); + ); + this.contentEl.style.visibility = ""; + }); } // The height of your card. Home Assistant uses this to automatically // distribute all cards over the available columns. diff --git a/src/types.ts b/src/types.ts index 003544d..27ef081 100644 --- a/src/types.ts +++ b/src/types.ts @@ -48,7 +48,7 @@ export type EntityConfig = EntityIdConfig & { lambda?: ( y: Datum[], x: Date[], - raw_entity: EntityState[] + raw_entity: CachedEntity[] ) => Datum[] | { x?: Datum[]; y?: Datum[] }; show_value: | boolean @@ -110,15 +110,19 @@ export function isEntityIdStatisticsConfig( } export type Timestamp = number; -export type EntityState = (HassEntity | StatisticValue) & { + +export type CachedStateEntity = HassEntity & { fake_boundary_datapoint?: true; timestamp: Timestamp; - value: number | string; + value: number | string | null; }; -export type HistoryInRange = { - range: [number, number]; - history: EntityState[]; +export type CachedStatisticsEntity = StatisticValue & { + fake_boundary_datapoint?: true; + timestamp: Timestamp; + value: number | string | null; }; +export type CachedEntity = CachedStateEntity | CachedStatisticsEntity; + export type TimestampRange = Timestamp[]; // [Timestamp, Timestamp]; export type HATheme = {