feat(apps/whale-api): add dumpdb

.gitignore

@@ -33,6 +33,9 @@ dist
 # debug log
 lerna-debug.log
 
+# debug dump
+dump
+
 coverage
 
 # vscode

apps/package.json

@@ -43,16 +43,17 @@
     "class-transformer": "0.5.1",
     "class-validator": "0.14.0",
     "cross-fetch": "3.1.5",
+    "fast-csv": "^4.3.6",
     "graphology": "0.25.1",
     "graphology-components": "1.5.4",
     "graphology-simple-path": "0.1.2",
     "joi": "17.8.4",
-    "lodash": "4.17.21",
     "level": "7.0.0",
     "level-js": "6.0.0",
     "level-packager": "6.0.0",
     "leveldown": "6.0.0",
     "lexicographic-integer-encoding": "1.0.1",
+    "lodash": "4.17.21",
     "memdown": "6.1.1",
     "pg": "8.9.0",
     "reflect-metadata": "0.1.13",
@@ -67,9 +68,9 @@
     "@types/cache-manager": "4.0.2",
     "@types/cron": "2.0.1",
     "@types/express": "4.17.17",
-    "@types/lodash": "4.14.194",
     "@types/level": "^6.0.0",
     "@types/levelup": "4.3.1",
+    "@types/lodash": "4.14.194",
     "@types/lossless-json": "1.0.1",
     "@types/memdown": "3.0.1",
     "@types/node": "16.11.58",

apps/whale-api/src/app.configuration.ts

@@ -23,7 +23,8 @@ export function AppConfiguration (): any {
       level: {
         location: process.env.WHALE_DATABASE_LEVEL_LOCATION
       }
-    }
+    },
+    debug: process.env.WHALE_DEBUG
   }
 }
 
@@ -34,6 +35,7 @@ export function ENV_VALIDATION_SCHEMA (): any {
     WHALE_NETWORK: Joi.string().valid('mainnet', 'testnet', 'regtest', 'devnet', 'changi').default('regtest'),
     WHALE_DEFID_URL: Joi.string().optional(),
     WHALE_DATABASE_PROVIDER: Joi.string().optional(),
-    WHALE_DATABASE_LEVEL_LOCATION: Joi.string().optional()
+    WHALE_DATABASE_LEVEL_LOCATION: Joi.string().optional(),
+    WHALE_DEBUG: Joi.bool().optional()
   })
 }

apps/whale-api/src/module.api/_module.ts

@@ -3,6 +3,7 @@ import { CacheModule, Global, Module } from '@nestjs/common'
 import { RpcController } from './rpc.controller'
 import { ActuatorController } from './actuator.controller'
 import { TransactionController } from './transaction.controller'
+import { DebugController } from './debug.controller'
 import { ApiValidationPipe } from './pipes/api.validation.pipe'
 import { AddressController } from './address.controller'
 import { PoolPairController } from './poolpair.controller'
@@ -62,6 +63,7 @@ import { GovernanceService } from './governance.service'
     LoanController,
     LegacyController,
     ConsortiumController,
+    DebugController,
     GovernanceController
   ],
   providers: [

apps/whale-api/src/module.api/debug.controller.ts

@@ -0,0 +1,18 @@
+import { Controller, ForbiddenException, Get } from '@nestjs/common'
+import { Database } from '../module.database/database'
+
+@Controller('/debug')
+export class DebugController {
+  constructor (
+    protected readonly database: Database
+  ) {
+  }
+
+  @Get('/dumpdb')
+  async dumpDb (): Promise<void> {
+    if (process.env.WHALE_DEBUG === undefined) {
+      throw new ForbiddenException('Debug mode is not enabled')
+    }
+    await this.database.dump()
+  }
+}

apps/whale-api/src/module.database/database.spec/specifications.ts

@@ -33,6 +33,11 @@
   }
 }
 
+export async function shouldDump (database: Database): Promise<void> {
+  const dump = await database.dump()
+  expect(dump).toBeTruthy()
+}
+
 export async function shouldGetByPartitionKey (database: Database): Promise<void> {
   const index = PartitionMapping.index
   for (const data of PARTITIONS) {

apps/whale-api/src/module.database/database.ts

@@ -38,6 +38,8 @@ export abstract class Database {
     mapping: ModelMapping<M>,
     id: string
   ): Promise<void>
+
+  abstract dump (): Promise<boolean>
 }
 
 export enum SortOrder {

apps/whale-api/src/module.database/provider.level/level.database.ts

@@ -1,3 +1,6 @@
+import fs from 'fs'
+import path from 'path'
+import { format } from '@fast-csv/format'
 import sub from 'subleveldown'
 import level from 'level'
 import { LevelUp } from 'levelup'
@@ -24,6 +27,40 @@
     await this.root.close()
   }
 
+  async dump (): Promise<boolean> {
+    let id = 0
+    const maxSize = 500_000
+    const dir = path.join(process.cwd(), 'dump')
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir)
+    }
+    let items: string[] = []
+    return await new Promise((resolve, reject) => {
+      console.time('dump done in: ')
+      this.root.createReadStream() // query stream
+        .on('data', function (data) {
+          items.push(data)
+          if (items.length >= maxSize) {
+            const writer = fs.createWriteStream(`${dir}/dump-${id}.csv`)
+            const csv = format()
+            csv.pipe(writer)
+            items.forEach(i => csv.write(i))
+            csv.end()
+
+            items = []
+            id += 1
+          }
+        }).on('error', function (err) {
+          reject(err)
+        }).on('close', function () {
+          reject(new Error('stream closed'))
+        }).on('end', function () {
+          resolve(true)
+          console.timeEnd('dump done in: ')
+        })
+    })
+  }
+
   /**
    * Sub index space for model indexes.
    */

apps/whale-api/src/module.database/provider.memory/memory.database.spec.ts

@@ -39,6 +39,10 @@ it('should delete and be deleted', async () => {
   await spec.shouldDelete(database)
 })
 
+it('should dump', async () => {
+  await spec.shouldDump(database)
+})
+
 it('should query by partition pagination', async () => {
   await spec.shouldQueryPartitionPagination(database)
 })