Merge pull request #11 from hirosystems/develop

release beta

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - beta
       - develop
     paths-ignore:
       - "**/CHANGELOG.md"

api/src/api/routes/etchings.ts

@@ -44,7 +44,7 @@ export const EtchingRoutes: FastifyPluginCallback<
     async (request, reply) => {
       const offset = request.query.offset ?? 0;
       const limit = request.query.limit ?? 20;
-      const results = await fastify.db.getEtchings(offset, limit);
+      const results = await fastify.db.getRuneEtchings(offset, limit);
       await reply.send({
         limit,
         offset,
@@ -72,7 +72,7 @@ export const EtchingRoutes: FastifyPluginCallback<
       },
     },
     async (request, reply) => {
-      const rune = await fastify.db.getEtching(request.params.etching);
+      const rune = await fastify.db.getRuneEtching(request.params.etching);
       if (!rune) {
         await reply.code(404).send(Value.Create(NotFoundResponse));
       } else {

api/src/api/util/helpers.ts

@@ -8,9 +8,13 @@ function divisibility(num: string | BigNumber, decimals: number): string {
 
 export function parseEtchingResponse(rune: DbRuneWithChainTip): EtchingResponse {
   let mintable = true;
+  const minted = rune.minted == null ? '0' : rune.minted;
+  const total_mints = rune.total_mints == null ? '0' : rune.total_mints;
+  const burned = rune.burned == null ? '0' : rune.burned;
+  const total_burns = rune.total_burns == null ? '0' : rune.total_burns;
   if (
     rune.terms_amount == null ||
-    (rune.terms_cap && BigNumber(rune.total_mints).gte(rune.terms_cap)) ||
+    (rune.terms_cap && BigNumber(total_mints).gte(rune.terms_cap)) ||
     (rune.terms_height_start && BigNumber(rune.chain_tip).lt(rune.terms_height_start)) ||
     (rune.terms_height_end && BigNumber(rune.chain_tip).gt(rune.terms_height_end)) ||
     (rune.terms_offset_start &&
@@ -37,17 +41,15 @@ export function parseEtchingResponse(rune: DbRuneWithChainTip): EtchingResponse
     },
     supply: {
       premine: divisibility(rune.premine, rune.divisibility),
-      current: divisibility(
-        BigNumber(rune.minted).plus(rune.burned).plus(rune.premine),
-        rune.divisibility
-      ),
-      minted: divisibility(rune.minted, rune.divisibility),
-      total_mints: rune.total_mints,
-      burned: divisibility(rune.burned, rune.divisibility),
-      total_burns: rune.total_burns,
-      mint_percentage: rune.terms_cap
-        ? BigNumber(rune.total_mints).div(rune.terms_cap).times(100).toFixed(4)
-        : '0.0000',
+      current: divisibility(BigNumber(minted).plus(burned).plus(rune.premine), rune.divisibility),
+      minted: divisibility(minted, rune.divisibility),
+      total_mints,
+      burned: divisibility(burned, rune.divisibility),
+      total_burns,
+      mint_percentage:
+        rune.terms_cap != null && rune.terms_cap != '0'
+          ? BigNumber(total_mints).div(rune.terms_cap).times(100).toFixed(4)
+          : '0.0000',
       mintable,
     },
     turbo: rune.turbo,

api/src/pg/pg-store.ts

@@ -89,31 +89,53 @@ export class PgStore extends BasePgStore {
     return result[0]?.block_height;
   }
 
-  async getEtching(id: Rune): Promise<DbRuneWithChainTip | undefined> {
-    const result = await this.sql<DbRuneWithChainTip[]>`
-      SELECT *, (SELECT MAX(block_height) FROM ledger) AS chain_tip
-      FROM runes WHERE ${runeFilter(this.sql, id)}
-    `;
-    if (result.count == 0) return undefined;
-    return result[0];
-  }
-
-  async getEtchings(offset: Offset, limit: Limit): Promise<DbPaginatedResult<DbRuneWithChainTip>> {
+  private async getEtchings(
+    id?: Rune,
+    offset: Offset = 0,
+    limit: Limit = 1
+  ): Promise<DbPaginatedResult<DbRuneWithChainTip>> {
     const results = await this.sql<DbCountedQueryResult<DbRuneWithChainTip>[]>`
       WITH
         rune_count AS (SELECT COALESCE(MAX(number), 0) + 1 AS total FROM runes),
-        max AS (SELECT MAX(block_height) AS chain_tip FROM ledger)
-      SELECT *, (SELECT total FROM rune_count), (SELECT chain_tip FROM max)
-      FROM runes
-      ORDER BY block_height DESC, tx_index DESC
-      OFFSET ${offset} LIMIT ${limit}
+        max AS (SELECT MAX(block_height) AS chain_tip FROM ledger),
+        results AS (
+          SELECT *
+          FROM runes
+          ${id ? this.sql`WHERE ${runeFilter(this.sql, id)}` : this.sql``}
+          ORDER BY block_height DESC, tx_index DESC
+          OFFSET ${offset} LIMIT ${limit}
+        ),
+        recent_supplies AS (
+          SELECT DISTINCT ON (rune_id) *
+          FROM supply_changes
+          WHERE rune_id IN (SELECT id FROM results)
+          ORDER BY rune_id, block_height DESC
+        )
+        SELECT r.*, s.minted, s.total_mints, s.burned, s.total_burns,
+          (SELECT total FROM rune_count), (SELECT chain_tip FROM max)
+        FROM results AS r
+        INNER JOIN recent_supplies AS s ON r.id = s.rune_id
+        ORDER BY r.block_height DESC, r.tx_index DESC
     `;
     return {
       total: results[0]?.total ?? 0,
       results,
     };
   }
 
+  async getRuneEtching(id: Rune): Promise<DbRuneWithChainTip | undefined> {
+    const result = await this.getEtchings(id);
+    if (result.total == 0) return undefined;
+    return result.results[0];
+  }
+
+  async getRuneEtchings(
+    offset: Offset,
+    limit: Limit
+  ): Promise<DbPaginatedResult<DbRuneWithChainTip>> {
+    return this.getEtchings(undefined, offset, limit);
+  }
+
   private async getActivity(
     filter: PgSqlQuery,
     count: PgSqlQuery,
@@ -171,11 +193,16 @@ export class PgStore extends BasePgStore {
     limit: Limit
   ): Promise<DbPaginatedResult<DbItemWithRune<DbBalance>>> {
     const results = await this.sql<DbCountedQueryResult<DbItemWithRune<DbBalance>>[]>`
-      SELECT b.*, r.name, r.spaced_name, r.divisibility, COUNT(*) OVER() AS total
-      FROM balances AS b
-      INNER JOIN runes AS r ON r.id = b.rune_id
-      WHERE ${runeFilter(this.sql, id, 'r')}
-      ORDER BY b.balance DESC
+      WITH grouped AS (
+        SELECT DISTINCT ON (b.address) b.address, b.balance, b.total_operations, b.rune_id, r.name,
+          r.spaced_name, r.divisibility, COUNT(*) OVER() AS total
+        FROM balance_changes AS b
+        INNER JOIN runes AS r ON r.id = b.rune_id
+        WHERE ${runeFilter(this.sql, id, 'r')}
+        ORDER BY b.address, b.block_height DESC
+      )
+      SELECT * FROM grouped
+      ORDER BY balance DESC
       OFFSET ${offset} LIMIT ${limit}
     `;
     return {
@@ -189,10 +216,13 @@ export class PgStore extends BasePgStore {
     address: Address
   ): Promise<DbItemWithRune<DbBalance> | undefined> {
     const results = await this.sql<DbItemWithRune<DbBalance>[]>`
-      SELECT b.*, r.name, r.spaced_name, r.divisibility
-      FROM balances AS b
+      SELECT b.rune_id, b.address, b.balance, b.total_operations, r.name,
+        r.spaced_name, r.divisibility, COUNT(*) OVER() AS total
+      FROM balance_changes AS b
       INNER JOIN runes AS r ON r.id = b.rune_id
       WHERE ${runeFilter(this.sql, id, 'r')} AND address = ${address}
+      ORDER BY b.block_height DESC
+      LIMIT 1
     `;
     return results[0];
   }
@@ -203,10 +233,15 @@ export class PgStore extends BasePgStore {
     limit: Limit
   ): Promise<DbPaginatedResult<DbItemWithRune<DbBalance>>> {
     const results = await this.sql<DbCountedQueryResult<DbItemWithRune<DbBalance>>[]>`
-      SELECT b.*, r.name, r.spaced_name, r.divisibility, COUNT(*) OVER() AS total
-      FROM balances AS b
-      INNER JOIN runes AS r ON r.id = b.rune_id
-      WHERE address = ${address}
+      WITH grouped AS (
+        SELECT DISTINCT ON (b.rune_id) b.address, b.balance, b.total_operations, b.rune_id, r.name,
+          r.spaced_name, r.divisibility, COUNT(*) OVER() AS total
+        FROM balance_changes AS b
+        INNER JOIN runes AS r ON r.id = b.rune_id
+        WHERE address = ${address}
+        ORDER BY b.rune_id, b.block_height DESC
+      )
+      SELECT * FROM grouped
       ORDER BY balance DESC
       OFFSET ${offset} LIMIT ${limit}
     `;

api/src/pg/types.ts

@@ -24,11 +24,11 @@ type DbRune = {
   terms_offset_start: string | null;
   terms_offset_end: string | null;
   turbo: boolean;
-  minted: string;
-  total_mints: string;
-  burned: string;
-  total_burns: string;
-  total_operations: string;
+  minted: string | null;
+  total_mints: string | null;
+  burned: string | null;
+  total_burns: string | null;
+  total_operations: string | null;
   timestamp: number;
 };
 

migrations/V1__runes.sql

@@ -17,14 +17,10 @@ CREATE TABLE IF NOT EXISTS runes (
     terms_offset_start      NUMERIC,
     terms_offset_end        NUMERIC,
     turbo                   BOOLEAN NOT NULL DEFAULT FALSE,
-    minted                  NUMERIC NOT NULL DEFAULT 0,
-    total_mints             NUMERIC NOT NULL DEFAULT 0,
-    burned                  NUMERIC NOT NULL DEFAULT 0,
-    total_burns             NUMERIC NOT NULL DEFAULT 0,
-    total_operations        NUMERIC NOT NULL DEFAULT 0,
     timestamp               BIGINT NOT NULL
 );
-CREATE INDEX runes_block_height_tx_index_index ON runes (block_height, tx_index);
+
+CREATE INDEX runes_block_height_tx_index_index ON runes (block_height DESC, tx_index DESC);
 
 -- Insert default 'UNCOMMON•GOODS'
 INSERT INTO runes (

migrations/V2__supply_changes.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS supply_changes (
+    rune_id                 TEXT NOT NULL,
+    block_height            NUMERIC NOT NULL,
+    minted                  NUMERIC NOT NULL DEFAULT 0,
+    total_mints             NUMERIC NOT NULL DEFAULT 0,
+    burned                  NUMERIC NOT NULL DEFAULT 0,
+    total_burns             NUMERIC NOT NULL DEFAULT 0,
+    total_operations        NUMERIC NOT NULL DEFAULT 0,
+    PRIMARY KEY (rune_id, block_height)
+);

migrations/V4__balance_changes.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS balance_changes (
+    rune_id                 TEXT NOT NULL,
+    block_height            NUMERIC NOT NULL,
+    address                 TEXT NOT NULL,
+    balance                 NUMERIC NOT NULL,
+    total_operations        BIGINT NOT NULL DEFAULT 0,
+    PRIMARY KEY (rune_id, block_height, address)
+);
+
+CREATE INDEX balance_changes_address_balance_index ON balance_changes (address, block_height, balance DESC);
+CREATE INDEX balance_changes_rune_id_balance_index ON balance_changes (rune_id, block_height, balance DESC);

src/bitcoind/mod.rs

@@ -3,7 +3,7 @@ use chainhook_sdk::{
     utils::Context,
 };
 
-use crate::config::Config;
+use crate::{config::Config, try_error};
 
 fn get_client(config: &Config, ctx: &Context) -> Client {
     loop {
@@ -16,11 +16,7 @@ fn get_client(config: &Config, ctx: &Context) -> Client {
                 return con;
             }
             Err(e) => {
-                error!(
-                    ctx.expect_logger(),
-                    "bitcoind unable to get client: {}",
-                    e.to_string()
-                );
+                try_error!(ctx, "bitcoind unable to get client: {}", e.to_string());
                 std::thread::sleep(std::time::Duration::from_secs(1));
             }
         }
@@ -35,8 +31,8 @@ pub fn bitcoind_get_block_height(config: &Config, ctx: &Context) -> u64 {
                 return result.blocks;
             }
             Err(e) => {
-                error!(
-                    ctx.expect_logger(),
+                try_error!(
+                    ctx,
                     "bitcoind unable to get block height: {}",
                     e.to_string()
                 );

src/cli.rs

@@ -1,12 +1,15 @@
+use std::{thread::sleep, time::Duration};
+
 use clap::{Parser, Subcommand};
 
 use chainhook_sdk::utils::{BlockHeights, Context};
 
 use crate::{
     config::{generator::generate_config, Config},
-    db::{cache::new_index_cache, pg_connect},
-    scan::bitcoin::scan_blocks,
+    db::{cache::index_cache::IndexCache, pg_connect},
+    scan::bitcoin::{drop_blocks, scan_blocks},
     service::start_service,
+    try_info,
 };
 
 #[derive(Parser, Debug)]
@@ -27,6 +30,9 @@ enum Command {
     /// Scanning blocks and indexing runes
     #[clap(subcommand)]
     Scan(ScanCommand),
+    /// Perform maintenance operations on local databases
+    #[clap(subcommand)]
+    Db(DbCommand),
 }
 
 #[derive(Subcommand, PartialEq, Clone, Debug)]
@@ -105,6 +111,24 @@ struct PingCommand {
     pub config_path: String,
 }
 
+#[derive(Subcommand, PartialEq, Clone, Debug)]
+enum DbCommand {
+    /// Rebuild inscriptions entries for a given block
+    #[clap(name = "drop", bin_name = "drop")]
+    Drop(DropDbCommand),
+}
+
+#[derive(Parser, PartialEq, Clone, Debug)]
+struct DropDbCommand {
+    /// Starting block
+    pub start_block: u64,
+    /// Ending block
+    pub end_block: u64,
+    /// Load config file path
+    #[clap(long = "config-path")]
+    pub config_path: String,
+}
+
 pub fn main() {
     let logger = hiro_system_kit::log::setup_logger();
     let _guard = hiro_system_kit::log::setup_global_logger(logger.clone());
@@ -147,15 +171,35 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
         }
         Command::Service(ServiceCommand::Start(cmd)) => {
             let config = Config::from_file_path(&cmd.config_path)?;
+            let maintenance_enabled = std::env::var("MAINTENANCE_MODE").unwrap_or("0".into());
+            if maintenance_enabled.eq("1") {
+                try_info!(ctx, "Entering maintenance mode. Unset MAINTENANCE_MODE and reboot to resume operations.");
+                sleep(Duration::from_secs(u64::MAX))
+            }
             start_service(&config, &ctx).await?;
         }
         Command::Scan(ScanCommand::Start(cmd)) => {
             let config = Config::from_file_path(&cmd.config_path)?;
             let blocks = cmd.get_blocks();
             let mut pg_client = pg_connect(&config, true, &ctx).await;
-            let mut index_cache = new_index_cache(&config, &mut pg_client, &ctx).await;
+            let mut index_cache = IndexCache::new(&config, &mut pg_client, &ctx).await;
             scan_blocks(blocks, &config, &mut pg_client, &mut index_cache, &ctx).await?;
         }
+        Command::Db(DbCommand::Drop(cmd)) => {
+            let config = Config::from_file_path(&cmd.config_path)?;
+            println!(
+                "{} blocks will be deleted. Confirm? [Y/n]",
+                cmd.end_block - cmd.start_block + 1
+            );
+            let mut buffer = String::new();
+            std::io::stdin().read_line(&mut buffer).unwrap();
+            if buffer.starts_with('n') {
+                return Err("Deletion aborted".to_string());
+            }
+
+            let mut pg_client = pg_connect(&config, false, &ctx).await;
+            drop_blocks(cmd.start_block, cmd.end_block, &mut pg_client, &ctx).await;
+        }
     }
     Ok(())
 }