chore(apps): add sanity testing containers (#1314)

* add sanity base

* add draft test for fetching

* fix(sanity-testing): Point defid url to correct bridge with regtest credentials

* fix(sanity-testing): Add APP to buildargs

- Build with tar to rely only on Dockerfile, reduce maintenance burden
- Add logging to image building step

* minor reorganization

* set up port forwarding

* fix(sanity-testing): wait for whale to start listening on its container port

* chore(sanity-testing): add sanity tests (mapped from whale postman_collection.json)

* polish and document solution

* remove initial test being added to follow up PR

* extract port randomizer

Move port randomizer code into a readable method and link the source contributor from stackoverflow.

* fix image check logic

* build images before running tests

* remove console log

* chore: update package-lock.json

* chore(sanity-testing): rename SanityContainer to AppContainer

* chore(sanity-testing): remove redundant inclusion

* chore(sanity-testing): remove redundant new line at start of file

* chore(sanity-testing): increase sanity tests timeout to 5 mins

* chore(sanity-testing): change container name generation to use uuidv4

Co-authored-by: Eli <[email protected]>
Co-authored-by: Eli <[email protected]>

apps/whale/__sanity__/WhaleSanityContainer.sanity.ts

@@ -0,0 +1,59 @@
+import { WhaleSanityContainer } from '@defichain/testcontainers'
+import waitForExpect from 'wait-for-expect'
+
+const whale = new WhaleSanityContainer()
+
+// TODO(kodemon/eli-lim):
+//   would it make sense to use WhaleApiClient for sanity tests,
+//   instead of raw http requests?
+
+beforeAll(async () => {
+  await whale.start()
+
+  async function mockRealisticState (): Promise<void> {
+    await whale.blockchain.waitForWalletCoinbaseMaturity()
+    await whale.blockchain.waitForWalletBalanceGTE(100)
+
+    // TODO(kodemon/eli-lim): Create tokens, pool pairs, etc. to sanity test the endpoints
+  }
+  await mockRealisticState()
+
+  await waitForExpect(async () => {
+    const response = await whale.get('/_actuator/probes/readiness')
+    const json = await response.json()
+    expect(json.details.model.status).toStrictEqual('up')
+    expect(json.details.defid.blocks).toBeGreaterThanOrEqual(100)
+  }, 60_000)
+})
+
+afterAll(async () => {
+  await whale.stop()
+})
+
+describe('/_actuator', () => {
+  describe('/_actuator/probes/liveness', () => {
+    test('Status in JSON body is ok', async () => {
+      const response = await whale.get('/_actuator/probes/liveness')
+      expect(await response.json()).toStrictEqual({
+        details: {
+          defid: { status: 'up' },
+          model: { status: 'up' }
+        },
+        error: {},
+        info: {
+          defid: { status: 'up' },
+          model: { status: 'up' }
+        },
+        status: 'ok'
+      })
+      expect(response.status).toStrictEqual(200)
+    })
+  })
+
+  describe('/_actuator/probes/readiness', () => {
+    test('Status code is 503', async () => {
+      const response = await whale.get('/_actuator/probes/readiness')
+      expect(response.status).toStrictEqual(503)
+    })
+  })
+})

jest.config.js

@@ -12,6 +12,9 @@ module.exports = {
   verbose: true,
   clearMocks: true,
   testTimeout: 180000,
+  testPathIgnorePatterns: [
+    '__sanity__'
+  ],
   coveragePathIgnorePatterns: [
     '/node_modules/',
     '/examples/',

jest.sanity.js

@@ -0,0 +1,9 @@
+const config = require('./jest.config.js')
+
+module.exports = {
+  ...config,
+  testRegex: '((\\.|/)(sanity))\\.ts$',
+  testPathIgnorePatterns: [],
+  globalSetup: './jest.sanity.setup.js',
+  testTimeout: 300000
+}

jest.sanity.setup.js

@@ -0,0 +1,36 @@
+const Dockerode = require('dockerode')
+const path = require('path')
+const { pack } = require('tar-fs')
+
+const apps = ['whale']
+
+module.exports = async function () {
+  console.log('\nPreloading sanity images, this may take a while...')
+  await Promise.all(apps.map(build))
+}
+
+/**
+ * Builds a new image with a :sanity tag that can be pulled into unit tests and
+ * run on the current state of the code base. These steps are required to
+ * ensure that we are sanity testing against the current code state and not
+ * pre-built solutions which is tested seperately during our standard unit
+ * tests.
+ *
+ * @remarks Images are built with tar
+ * @see     https://github.com/apocas/dockerode/issues/432
+ */
+async function build (app) {
+  console.log(`Building '${app}:sanity' image`)
+  const docker = new Dockerode()
+  const image = pack(path.resolve(__dirname))
+  const stream = await docker.buildImage(image, {
+    t: `${app}:sanity`,
+    buildargs: {
+      APP: app
+    }
+  })
+  await new Promise((resolve, reject) => {
+    docker.modem.followProgress(stream, (err, res) => (err != null) ? reject(err) : resolve(res))
+  })
+  console.log(`Finished '${app}:sanity' image`)
+}

package.json

@@ -19,6 +19,7 @@
     "prepare": "husky install",
     "lint": "eslint . --fix",
     "test": "jest --maxWorkers=100%",
+    "sanity": "jest --maxWorkers=100% --config=jest.sanity.js",
     "ci:test": "jest --ci --coverage --forceExit --maxWorkers=4",
     "all:clean": "rm -rf ./packages/**/dist && rm -rf ./apps/dist && rm -rf ./packages/**/tsconfig.build.tsbuildinfo",
     "all:build": "lerna run build",

packages/testcontainers/package.json

@@ -17,9 +17,11 @@
     "@defichain/jellyfish-network": "^0.0.0",
     "cross-fetch": "^3.1.5",
     "dockerode": "^3.3.1",
+    "tar-fs": "^2.1.1",
     "uuid": "^8.3.2"
   },
   "peerDependencies": {
+    "@types/tar-fs": "^2.0.1",
     "defichain": "^0.0.0"
   },
   "devDependencies": {

packages/testcontainers/src/containers/AppContainer/WhaleSanityContainer.ts

@@ -0,0 +1,51 @@
+import { MasterNodeRegTestContainer } from '../RegTestContainer/Masternode'
+import { AppContainer } from '.'
+import { waitForCondition } from '../../utils'
+
+export class WhaleSanityContainer extends AppContainer {
+  constructor (port?: number, blockchain?: MasterNodeRegTestContainer) {
+    super('whale', port, blockchain)
+  }
+
+  /**
+   * Start the whale container by initiating a build procedure, instantiate the
+   * underlying blockchain node, and create a container instance to send sanity
+   * requests to.
+   *
+   * We provide the blockchain node ip and port to the internal whale configuration
+   * which links it to the node allowing it to hit the chain with RPC requests.
+   *
+   * @remarks
+   *
+   * The method performs a wait for condition to ensure the container is ready
+   * before the start method is considered resolved. Otherwise the unit tests
+   * will run before the container is ready which can result in various network
+   * or request errors.
+   */
+  public async start (): Promise<void> {
+    const { hostRegTestIp, hostRegTestPort } = await this.startMasterNode()
+
+    this.container = await this.docker.createContainer({
+      name: this.name,
+      Image: this.image,
+      Tty: true,
+      Env: [
+        `WHALE_DEFID_URL=http://testcontainers-user:testcontainers-password@${hostRegTestIp}:${hostRegTestPort}`,
+        'WHALE_NETWORK=regtest',
+        'WHALE_DATABASE_PROVIDER=memory'
+      ],
+      ExposedPorts: { '3000/tcp': {} },
+      HostConfig: {
+        PortBindings: { '3000/tcp': [{ HostPort: this.port.toString() }] },
+        PublishAllPorts: true
+      }
+    })
+
+    await this.container.start()
+
+    await waitForCondition(async () => {
+      const res = await this.get('/_actuator/probes/liveness')
+      return res.status === 200
+    }, 30_000) // 30s
+  }
+}