CheckerNetwork · NikolasHaimerl · Jan 11, 2025 · Jan 11, 2025 · Jan 12, 2025 · Jan 12, 2025
@@ -0,0 +1,76 @@
+---
+toc: false
+title: Storage Provider Summary
+---
+
+```js
+import { LineGraph } from "../components/line-graph.js";
+import { getDateXDaysAgo } from "../utils/date-utils.js";
+const data = FileAttachment(`../data/${observable.params.client}-spark-rsr-summary.json`).json();
+```
+
+<div class="hero">
+  <body><a href="/"><img src="../media/spark-logomark-blue-with-bbox.png" alt="Spark Logo" width="300" /></a><body>
+    <h2>Dashboard Beta</h2>
+    <body><a href="https://filspark.com/dashboard" target="_blank" rel="noopener noreferrer">(Click here for Legacy Spark Grafana Dashboard)</a><body>
+</div>
+
+<h4>Storage Client Spark RSR Summary</h4>
+<body>This section shows the storage client Spark Retrieval Success Rate Score summary. You can adjust the date range. Records start on the 25th February 2025.</body>
+
+
+```js
+const start = view(Inputs.date({label: "Start", value: getDateXDaysAgo(180) }));
+const end = view(Inputs.date({label: "End", value: getDateXDaysAgo(1) }));
+```
+
+<h3>Stats for ${observable.params.provider}</h3>
+
+<div class="grid grid-cols" style="grid-auto-rows: 500px;">
+  <div class="card">${
+    resize((width) => LineGraph(data, {width, title: "Retrieval Success Rate", start, end }))
+  }</div>
+</div>
+
+<style>
+
+.hero {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  font-family: var(--sans-serif);
+  margin: 4rem 0 8rem;
+  text-wrap: balance;
+  text-align: center;
+}
+
+.hero h1 {
+  margin: 1rem 0;
+  padding: 1rem 0;
+  max-width: none;
+  font-size: 14vw;
+  font-weight: 900;
+  line-height: 1;
+  background: linear-gradient(30deg, var(--theme-foreground-focus), currentColor);
+  -webkit-background-clip: text;
+  -webkit-text-fill-color: transparent;
+  background-clip: text;
+}
+
+.hero h2 {
+  margin: 0;
+  max-width: 34em;
+  font-size: 20px;
+  font-style: initial;
+  font-weight: 500;
+  line-height: 1.5;
+  color: var(--theme-foreground-muted);
+}
+
+@media (min-width: 640px) {
+  .hero h1 {
+    font-size: 90px;
+  }
+}
+
+</style>
@@ -1,6 +1,6 @@
 import * as Plot from 'npm:@observablehq/plot'
 
-export function LineGraph(events, { width, height, title, start, end } = {}) {
+export function LineGraph (events, { width, height, title, start, end } = {}) {
   const startDate = new Date(start)
   const endDate = new Date(end)
   const filteredEvents = events.filter((event) => {
@@ -11,13 +11,13 @@ export function LineGraph(events, { width, height, title, start, end } = {}) {
     ...filteredEvents.map((event) => ({
       day: event.day,
       success_rate: event.success_rate,
-      type: 'HTTP or Graphsync',
+      type: 'HTTP or Graphsync'
     })),
     ...filteredEvents.map((event) => ({
       day: event.day,
       success_rate_http: event.success_rate_http,
-      type: 'HTTP only',
-    })),
+      type: 'HTTP only'
+    }))
   ]
 
   return Plot.plot({
@@ -32,14 +32,14 @@ export function LineGraph(events, { width, height, title, start, end } = {}) {
         x: 'day',
         y: 'success_rate',
         stroke: 'type',
-        curve: 'linear',
+        curve: 'linear'
       }),
       Plot.lineY(combinedData, {
         x: 'day',
         y: 'success_rate_http',
         stroke: 'type',
-        curve: 'linear',
-      }),
-    ],
+        curve: 'linear'
+      })
+    ]
   })
 }
@@ -0,0 +1,17 @@
+import pRetry from 'p-retry'
+import { jsonFetcher } from './json-fetcher.js'
+import { getDateXDaysAgo } from '../utils/date-utils.js'
+import { parseArgs } from 'node:util'
+
+const {
+  values: { client }
+} = parseArgs({
+  options: { client: { type: 'string' } }
+})
+
+const start = '2024-04-07'
+const end = getDateXDaysAgo(1)
+
+const summary = await pRetry(() => jsonFetcher(`https://stats.filspark.com/client/${client}/retrieval-success-rate/summary?from=${start}&to=${end}`), { retries: 3 })
+
+process.stdout.write(JSON.stringify(summary))
@@ -0,0 +1,8 @@
+import { jsonFetcher } from './json-fetcher.js'
+import { getDateXDaysAgo } from '../utils/date-utils.js'
+
+const from = getDateXDaysAgo(31)
+const to = getDateXDaysAgo(1)
+
+const output = await jsonFetcher(`https://stats.filspark.com/clients/retrieval-success-rate/summary?from=${from}&to=${to}`)
+process.stdout.write(JSON.stringify(output))
@@ -24,6 +24,7 @@ const SparkMinerRsrSummaries = FileAttachment(
 const SparkRetrievalTimes = FileAttachment(
   './data/spark-retrieval-timings.json',
 ).json()
+const SparkClientRates = FileAttachment("./data/spark-clients-rsr.json").json();
 ```
 
 ```js
@@ -35,6 +36,20 @@ const sparkMinerRetrievalTimingsMap = SparkMinerRetrievalTimings.reduce(
   {},
 )
 
+const nonZeroSparkMinerRates = SparkMinerRates.filter(
+  (record) => record.success_rate != 0,
+)
+const tidySparkMinerRates = SparkMinerRates.sort(
+  (recordA, recordB) => recordB.success_rate - recordA.success_rate,
+).map((record) => {
+  const { ttfb_ms } = sparkMinerRetrievalTimingsMap[record.miner_id] ?? {}
+  return {
+    ...record,
+    ttfb_ms,
+    success_rate: `${(record.success_rate * 100).toFixed(2)}%`,
+    success_rate_http: `${(record.success_rate_http * 100).toFixed(2)}%`,
+  }
+})
 const nonZeroSparkMinerRates = SparkMinerRates.filter(
   (record) => record.success_rate != 0,
 )
@@ -100,6 +115,28 @@ const end = view(Inputs.date({ label: 'End', value: getDateXDaysAgo(1) }))
 <body></body>
 
 ```js
+<<<<<<< HEAD
+const countAbove = (a, t) => a.filter(v => v > t).length
+const nonZeroMinersOverTime = Object.entries(SparkMinerRsrSummaries).flatMap(([day, miners]) => 
+  [
+    {day: new Date(day),
+    count_succes_rate: countAbove(miners.map(m => m.success_rate), 0), type: "HTTP or Graphsync"},
+    {day: new Date(day),count_succes_rate_http: miners.some((m)=> m.success_rate_http != null )? countAbove(miners.map(m => m.success_rate_http), 0):null, type: "HTTP only"}
+])
+const percentiles = Object.entries(SparkMinerRsrSummaries)
+  .flatMap(([day, miners]) => [
+    0.8,
+    0.9,
+    0.95,
+    0.99,
+    0.995,
+    0.999
+  ].map(above => ({
+    day: new Date(day),
+    label: `> ${above * 100}%`,
+    count_succes_rate: countAbove(miners.map(m => m.success_rate), above),
+  })))
+=======
 const countAbove = (a, t) => a.filter((v) => v > t).length
 const nonZeroMinersOverTime = Object.entries(SparkMinerRsrSummaries).flatMap(
   ([day, miners]) => [
@@ -134,6 +171,7 @@ const percentiles = Object.entries(SparkMinerRsrSummaries).flatMap(
       ),
     })),
 )
+>>>>>>> main
 ```
 
 <div class="grid grid-cols-2" style="grid-auto-rows: 500px;">
@@ -318,15 +356,32 @@ ${JSON.stringify(
 <body>The following table shows the Spark RSR and TTFB values calculated in aggregate for each Filecoin Storage Provider over the past 30 days. Click on a miner id to view stats about this storage provider.</body>
 
 ```js
+<<<<<<< HEAD
+const searchMinerStats = view(Inputs.search(tidySparkMinerRates, {placeholder: "Search Storage Providers…"}));
+=======
 const search = view(
   Inputs.search(tidySparkMinerRates, {
     placeholder: 'Search Storage Providers…',
   }),
 )
+>>>>>>> main
+```
+
+<div class="card" style="padding: 0;">
+  ${Inputs.table(searchMinerStats, {rows: 16, format: {miner_id: id => htl.html`<a href=./provider/${id} target=_blank>${id}</a>`}})}
+</div>
+
+<div class="divider"></div>
+
+<h4>Spark Client RSR Table</h4>
+<body>The following table shows the Spark RSR values calculated in aggregate for each Filecoin Storage Client over the past 30 days. Click on a client id to view stats about this storage provider.</body>
+
+```js
+const searchClientStats = view(Inputs.search(tidySparkClientRates, {placeholder: "Search Storage Clients..."}));
 ```
 
 <div class="card" style="padding: 0;">
-  ${Inputs.table(search, {rows: 16, format: {miner_id: id => htl.html`<a href=./provider/${id} target=_blank>${id}</a>`}})}
+  ${Inputs.table(searchClientStats, {rows: 16, format: {client_id: id => htl.html`<a href=./client/${id} target=_blank>${id}</a>`}})}
 </div>
 
 <style>