From b5377176de4a28d5eb149f1cfe6df319a43cf844 Mon Sep 17 00:00:00 2001
From: David Gasquez <davidgasquez@gmail.com>
Date: Fri, 1 Nov 2024 18:02:36 +0100
Subject: [PATCH] feat: :sparkles: enhance dashboard with daily retrieval
 metrics and new visualization

- Replaced "wide" theme with "dashboard" for improved aesthetics.
- Introduced a script to calculate daily retrieval metrics using DuckDB.
- Updated dashboard to include sections for "Retrieval Service Class Conformance" and "Durability" with relevant visualizations.
- Bumped `lru-cache` dependency to v11.0.2 to ensure compatibility and security.
---
 observablehq.config.js                  |  3 +-
 package-lock.json                       |  6 +--
 src/data/daily_retrieval_metrics.csv.sh | 33 +++++++++++++++
 src/index.md                            | 55 +++++++++++++++++++++++++
 src/provider/[provider].md              | 32 +++++++++++++-
 5 files changed, 123 insertions(+), 6 deletions(-)
 create mode 100644 src/data/daily_retrieval_metrics.csv.sh

diff --git a/observablehq.config.js b/observablehq.config.js
index aec4105..60dda3f 100644
--- a/observablehq.config.js
+++ b/observablehq.config.js
@@ -19,8 +19,7 @@ export default {
   root: "src",
   theme: [
     "parchment",
-    "alt",
-    "wide", // 2024-10-30: this didn't have the expected impact
+    "dashboard"
   ],
   footer: false,
   sidebar: false,
diff --git a/package-lock.json b/package-lock.json
index e135069..b973bb0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -3865,9 +3865,9 @@
       }
     },
     "node_modules/rimraf/node_modules/lru-cache": {
-      "version": "11.0.1",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.1.tgz",
-      "integrity": "sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==",
+      "version": "11.0.2",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.2.tgz",
+      "integrity": "sha512-123qHRfJBmo2jXDbo/a5YOQrJoHF/GNQTLzQ5+IdK5pWpceK17yRc6ozlWd25FxvGKQbIUs91fDFkXmDHTKcyA==",
       "dev": true,
       "license": "ISC",
       "engines": {
diff --git a/src/data/daily_retrieval_metrics.csv.sh b/src/data/daily_retrieval_metrics.csv.sh
new file mode 100644
index 0000000..6559fe9
--- /dev/null
+++ b/src/data/daily_retrieval_metrics.csv.sh
@@ -0,0 +1,33 @@
+
+#!/usr/bin/env bash
+
+duckdb :memory: << EOF
+SET enable_progress_bar = false;
+COPY (
+    with retrieval_rates as (
+    select
+        date,
+        provider_id,
+        case
+        when total_retrieval_requests > 0
+        then successful_retrieval_requests * 1.0 / total_retrieval_requests
+        else 0
+        end as retrieval_rate
+    from read_parquet('https://data.filecoindataportal.xyz/filecoin_daily_storage_providers_metrics.parquet')
+    where date >= current_date() - interval '90 days'
+    )
+
+    select
+    date,
+    count(distinct provider_id) as meet_retrieval_sli,
+    count(distinct provider_id) * 1.0 / (
+        select count(distinct provider_id)
+        from retrieval_rates
+        where date = r.date
+    ) as meet_retrieval_sli_percent
+    from retrieval_rates r
+    where retrieval_rate > 0.9
+    group by date
+    order by date desc
+) TO STDOUT (FORMAT 'CSV');
+EOF
diff --git a/src/index.md b/src/index.md
index 559f808..cb4316b 100644
--- a/src/index.md
+++ b/src/index.md
@@ -11,6 +11,7 @@ This dasbhoard is a scrappy work-in-progress to help spur discussion at [FDS-5 B
 
 ```js
 const daily_metrics = FileAttachment("data/daily_metrics.csv").csv({typed: true});
+const daily_retrieval_metrics = FileAttachment("data/daily_retrieval_metrics.csv").csv({typed: true});
 
 // Mutates the provided plotConfig by adding a caption if one doesn't exist
 function addPlotCaption(plotConfig, href) {
@@ -109,6 +110,60 @@ This is an aggregate view looking at all Storage Providers on the network.
   </div>
 </div>
 
+### Retrieval Service Class Conformance
+
+<div class="grid grid-cols-2">
+  <div class="card">
+
+  ```js
+  resize((width) => Plot.plot(addPlotCaption({
+    title: "Providers Meeting Retrieval SLI",
+    subtitle: "Number of providers with >90% successful retrievals",
+    x: { label: "Date" },
+    y: { grid: true, label: "Providers", zero: true },
+    width,
+    marks: [
+      Plot.ruleY([0]),
+      Plot.lineY(daily_retrieval_metrics, {
+        x: "date",
+        y: "meet_retrieval_sli",
+        tip: true
+      })
+    ]
+  })))
+  ```
+
+  </div>
+
+  <div class="card">
+
+  ```js
+  resize((width) => Plot.plot(addPlotCaption({
+    title: "Providers Meeting Retrieval SLI (%)",
+    subtitle: "Percentage of providers with >90% successful retrievals",
+    x: { label: "Date" },
+    y: {
+      grid: true,
+      label: "Percentage",
+      domain: [0, 100]
+    },
+    width,
+    marks: [
+      Plot.ruleY([0]),
+      Plot.lineY(daily_retrieval_metrics, {
+        x: "date",
+        y: d => d.meet_retrieval_sli_percent * 100,
+        tip: true
+      })
+    ]
+  })))
+  ```
+
+  </div>
+</div>
+
+
+
 ### Service Class Conformance
 
 <div class="grid grid-cols-2">
diff --git a/src/provider/[provider].md b/src/provider/[provider].md
index 91d366c..a494ce4 100644
--- a/src/provider/[provider].md
+++ b/src/provider/[provider].md
@@ -10,7 +10,6 @@
 const provider_metrics = FileAttachment(`../data/${observable.params.provider}_daily_metrics.csv`).csv({typed: true});
 ```
 
-
 ## Retrievals
 
 <div class="grid grid-cols-2">
@@ -72,6 +71,37 @@ const provider_metrics = FileAttachment(`../data/${observable.params.provider}_d
 </div>
 </div>
 
+## Durability
+
+<div class="card">
+
+```js
+resize((width) => Plot.plot({
+  title: "Durability",
+  subtitle: "Percentage of sectors that have faulted on a given day",
+  x: { label: "Date" },
+  y: {
+    grid: true,
+    label: "Fault Rate (%)",
+    domain: [0, 100]
+  },
+  width,
+  marks: [
+    Plot.ruleY([0]),
+    Plot.text(
+      ["To be implemented"],
+      {
+        x: 0,
+        y: 40,
+        fontSize: 48,
+        fill: "var(--theme-foreground-faint)"
+      }
+    )
+  ]
+}))
+```
+
+</div>
 
 ## Other Metrics