diff --git a/README.md b/README.md index 7a39f00..1786b0a 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ to accelerate the adoption of blockchain technology. ```js import hardhat from '@chainfile/eip-155-31337/hardhat.json'; -const testcontainers = new ChainfileTestcontainers(hardhat); +const testcontainers = new CFTestcontainers(hardhat); beforeAll(async () => { await testcontainers.start(); diff --git a/package.json b/package.json index 2f6325c..ab34d83 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "turbo": "^2.0.4", "typescript": "5.4.5" }, - "packageManager": "pnpm@9.3.0", + "packageManager": "pnpm@9.4.0", "engines": { "node": "^20 <21" } diff --git a/packages/chainfile-agent/src/routers/_context.ts b/packages/chainfile-agent/src/routers/_context.ts index c73c2da..cee01eb 100644 --- a/packages/chainfile-agent/src/routers/_context.ts +++ b/packages/chainfile-agent/src/routers/_context.ts @@ -17,22 +17,22 @@ function getChainfile(): Chainfile { return chainfile; } -function getValues(): Record { - const values = process.env.CHAINFILE_VALUES; - if (values !== undefined) { - return JSON.parse(values); +function getParams(): Record { + const params = process.env.CHAINFILE_PARAMS; + if (params !== undefined) { + return JSON.parse(params); } - throw new Error('CHAINFILE_VALUES is not defined, cannot start @chainfile/agent.'); + throw new Error('CHAINFILE_PARAMS is not defined, cannot start @chainfile/agent.'); } const chainfile = getChainfile(); -const values = getValues(); +const params = getParams(); export const createContext = async () => { return { chainfile: chainfile, - values: values, + params: params, }; }; diff --git a/packages/chainfile-agent/src/routers/agent.test.ts b/packages/chainfile-agent/src/routers/agent.test.ts index 992939f..8330dd6 100644 --- a/packages/chainfile-agent/src/routers/agent.test.ts +++ b/packages/chainfile-agent/src/routers/agent.test.ts @@ -7,7 +7,7 @@ const chainfile: Chainfile = { $schema: 'https://chainfile.org/schema.json', caip2: 'bip122:0f9188f13cb7b2c71f2a335e3a4fc328', name: 'Bitcoin Regtest', - values: { + params: { rpc_user: 'agent', rpc_password: 'agent', }, @@ -59,7 +59,7 @@ const chainfile: Chainfile = { const caller = createCaller({ chainfile: chainfile, - values: {}, + params: {}, }); it('should getChainfile', async () => { diff --git a/packages/chainfile-agent/src/routers/agent.ts b/packages/chainfile-agent/src/routers/agent.ts index d520c51..017b917 100644 --- a/packages/chainfile-agent/src/routers/agent.ts +++ b/packages/chainfile-agent/src/routers/agent.ts @@ -11,7 +11,8 @@ export const agentRouter = router({ $schema: z.string().optional(), caip2: z.string(), name: z.string(), - values: z.any().optional(), + params: z.any().optional(), + volumes: z.any().optional(), containers: z.any(), }), ) diff --git a/packages/chainfile-agent/src/routers/probes.ts b/packages/chainfile-agent/src/routers/probes.ts index 3b0a2fb..1c9d692 100644 --- a/packages/chainfile-agent/src/routers/probes.ts +++ b/packages/chainfile-agent/src/routers/probes.ts @@ -7,7 +7,7 @@ import { EndpointHttpAuthorization, EndpointHttpJsonRpc, EndpointHttpRest, - ValueReference, + ParamReference, } from '@chainfile/schema'; import Ajv from 'ajv'; import addFormats from 'ajv-formats'; @@ -29,7 +29,7 @@ const probeProcedure = publicProcedure .use((opts) => { return opts.next({ ctx: { - probes: new Probes(opts.ctx.chainfile, opts.ctx.values) as any, + probes: new Probes(opts.ctx.chainfile, opts.ctx.params) as any, }, }); }); @@ -68,7 +68,7 @@ type ProbeFunction = () => Promise; class Probes { constructor( private readonly chainfile: Chainfile, - private readonly values: Record, + private readonly params: Record, private readonly ajv: Ajv = new Ajv(), ) { addFormats(this.ajv); @@ -253,16 +253,16 @@ class Probes { } } - private resolve(value: string | ValueReference): string { - if (typeof value === 'string') { - return value; + private resolve(param: string | ParamReference): string { + if (typeof param === 'string') { + return param; } - return this.values[value.$value] ?? ''; + return this.params[param.$param] ?? ''; } } -// TODO(fuxingloh): Probes (Liveness, Readiness, Startup) currently only allow a single probe per endpoint +// TODO(?): Probes (Liveness, Readiness, Startup) currently only allow a single probe per endpoint // We should allow multiple endpoints per container where some conditions can only be checked by through calling // multiple endpoints. // For example, a container with 3 conditions required for it to be liveness: diff --git a/packages/chainfile-agent/src/server.ts b/packages/chainfile-agent/src/server.ts index acf60ca..c0e59be 100644 --- a/packages/chainfile-agent/src/server.ts +++ b/packages/chainfile-agent/src/server.ts @@ -12,5 +12,5 @@ const server = http.createServer( }), ); -/** The sum of the ASCII values for the string "@chainfile/agent" is 1569. */ +/** The sum of the ASCII for the string "@chainfile/agent" is 1569. */ server.listen(1569); diff --git a/packages/chainfile-cdk8s/README.md b/packages/chainfile-cdk8s/README.md index 17d7906..c208e05 100644 --- a/packages/chainfile-cdk8s/README.md +++ b/packages/chainfile-cdk8s/README.md @@ -2,31 +2,32 @@ This package contains the CDK8s application that deploys the Chainfile application to a Kubernetes cluster. -## Contributing Guidelines +## Local Development -You need to install `kind` to run the tests. You can install it with the following command: +You can test kubernetes locally with Kubernetes-in-Docker (kind). +Kind literally runs Kubernetes in Docker containers where each container is a node in the cluster. +Allowing you to test complex Kubernetes configurations locally. +To install `kind` on macOS and set up a cluster, run the following commands: -```shell +```bash brew install kind +kind create cluster --name cdk8s ``` -### Setting up a kind cluster for local development +To synth and deploy to the local cluster, run the following commands: -```shell -kind create cluster --config kind.k8s.yaml +```bash +turbo synth +kubectl apply --context kind-cdk8s -f [file_name].k8s.yaml ``` -Optionally, you can install the Kubernetes Dashboard to monitor the cluster for better visibility: +Other Add-Ons: + +
+Metrics Server ```shell -# Add the Kubernetes Dashboard Helm repository -helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/ -# Install the Kubernetes Dashboard -helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard -# Get the token to access the Kubernetes Dashboard -kubectl create serviceaccount dashboard -kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=default:dashboard -kubectl create token dashboard -# Start the proxy and access the Kubernetes Dashboard on https://localhost:8443 -kubectl -n kubernetes-dashboard port-forward svc/kubernetes-dashboard-kong-proxy 8443:443 +kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/high-availability-1.21+.yaml ``` + +
diff --git a/packages/chainfile-cdk8s/src/__snapshots__/chart.test.ts.snap b/packages/chainfile-cdk8s/src/__snapshots__/chart.test.ts.snap index 90811f2..940ff74 100644 --- a/packages/chainfile-cdk8s/src/__snapshots__/chart.test.ts.snap +++ b/packages/chainfile-cdk8s/src/__snapshots__/chart.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` +exports[`bitcoin_mainnet.json should synth bitcoin_mainnet.json and match snapshot 1`] = ` [ { "apiVersion": "v1", @@ -10,33 +10,59 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` "bitcoind": "true", "caip2": "bip122.000000000019d6689c085ae165831e93", }, - "name": "bitcoin_mainnet.json-values-secret-c8e15ab9", + "name": "bitcoin_mainnet.json-secret-c89840a1", }, "stringData": { - "CHAINFILE_VALUES": "{"rpc_user":"user","rpc_password":"pass"}", + "CHAINFILE_PARAMS": "{"rpc_user":"user","rpc_password":"pass"}", "rpc_password": "pass", "rpc_user": "user", }, "type": "Opaque", }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "bitcoind": "true", + "caip2": "bip122.000000000019d6689c085ae165831e93", + }, + "name": "bitcoin_mainnet.json-service-c8c0db95", + }, + "spec": { + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 8332, + }, + ], + "selector": { + "bitcoind": "true", + "caip2": "bip122.000000000019d6689c085ae165831e93", + }, + "type": "LoadBalancer", + }, + }, { "apiVersion": "apps/v1", - "kind": "Deployment", + "kind": "StatefulSet", "metadata": { "labels": { "bitcoind": "true", "caip2": "bip122.000000000019d6689c085ae165831e93", }, - "name": "bitcoin_mainnet.json-deployment-c855ba6b", + "name": "bitcoin_mainnet.json-stateful-set-c8a4e78e", }, "spec": { - "replicas": 1, + "replicas": 2, "selector": { "matchLabels": { "bitcoind": "true", "caip2": "bip122.000000000019d6689c085ae165831e93", }, }, + "serviceName": "bitcoin_mainnet.json-service-c8c0db95", "template": { "metadata": { "labels": { @@ -50,20 +76,26 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` "env": [ { "name": "CHAINFILE_JSON", - "value": "{"caip2":"bip122:000000000019d6689c085ae165831e93","name":"Bitcoin Mainnet","values":{"rpc_user":{"description":"Username for RPC authentication","secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}},"rpc_password":{"description":"Password for RPC authentication","secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}}},"containers":{"bitcoind":{"image":"docker.io/kylemanna/bitcoind","tag":"latest","source":"https://github.com/kylemanna/docker-bitcoind","endpoints":{"p2p":{"port":8333},"rpc":{"port":8332,"protocol":"HTTP JSON-RPC 2.0","authorization":{"type":"HttpBasic","username":{"$value":"rpc_user"},"password":{"$value":"rpc_password"}},"probes":{"readiness":{"method":"getblockchaininfo","params":[],"match":{"result":{"type":"object","properties":{"blocks":{"type":"number"}},"required":["blocks"]}}}}}},"resources":{"cpu":1,"memory":2048},"environment":{"DISABLEWALLET":"1","RPCUSER":{"$value":"rpc_user"},"RPCPASSWORD":{"$value":"rpc_password"}},"volumes":{"persistent":{"paths":["/bitcoin/.bitcoin"],"size":{"initial":"600G","from":"2024-01-01","growth":"20G","rate":"monthly"}}}}}}", + "value": "{"caip2":"bip122:000000000019d6689c085ae165831e93","name":"Bitcoin Mainnet","params":{"rpc_user":{"description":"Username for RPC authentication","secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}},"rpc_password":{"description":"Password for RPC authentication","secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}}},"volumes":{"data":{"type":"persistent","size":"600Gi","expansion":{"startFrom":"2024-01-01","monthlyRate":"20Gi"}}},"containers":{"bitcoind":{"image":"docker.io/kylemanna/bitcoind","tag":"latest","source":"https://github.com/kylemanna/docker-bitcoind","endpoints":{"p2p":{"port":8333},"rpc":{"port":8332,"protocol":"HTTP JSON-RPC 2.0","authorization":{"type":"HttpBasic","username":{"$param":"rpc_user"},"password":{"$param":"rpc_password"}},"probes":{"readiness":{"method":"getblockchaininfo","params":[],"match":{"result":{"type":"object","properties":{"blocks":{"type":"number"}},"required":["blocks"]}}}}}},"resources":{"cpu":1,"memory":2048},"environment":{"DISABLEWALLET":"1","RPCUSER":{"$param":"rpc_user"},"RPCPASSWORD":{"$param":"rpc_password"}},"mounts":[{"volume":"data","mountPath":"/bitcoin/.bitcoin","subPath":"bitcoind"}]}}}", }, { - "name": "CHAINFILE_VALUES", + "name": "CHAINFILE_PARAMS", "valueFrom": { "secretKeyRef": { - "key": "CHAINFILE_VALUES", - "name": "bitcoin_mainnet.json-values-secret-c8e15ab9", + "key": "CHAINFILE_PARAMS", + "name": "bitcoin_mainnet.json-secret-c89840a1", "optional": false, }, }, }, ], "image": "ghcr.io/vetumorg/chainfile-agent:0.0.0", + "livenessProbe": { + "httpGet": { + "path": "/probes/liveness", + "port": "agent", + }, + }, "name": "agent", "ports": [ { @@ -71,12 +103,18 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` "name": "agent", }, ], - "volumeMounts": [ - { - "mountPath": "/var/chainfile", - "name": "chainfile", + "readinessProbe": { + "httpGet": { + "path": "/probes/readiness", + "port": "agent", }, - ], + }, + "startupProbe": { + "httpGet": { + "path": "/probes/startup", + "port": "agent", + }, + }, }, { "env": [ @@ -89,7 +127,7 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` "valueFrom": { "secretKeyRef": { "key": "rpc_user", - "name": "bitcoin_mainnet.json-values-secret-c8e15ab9", + "name": "bitcoin_mainnet.json-secret-c89840a1", "optional": false, }, }, @@ -99,7 +137,7 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` "valueFrom": { "secretKeyRef": { "key": "rpc_password", - "name": "bitcoin_mainnet.json-values-secret-c8e15ab9", + "name": "bitcoin_mainnet.json-secret-c89840a1", "optional": false, }, }, @@ -110,11 +148,9 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` "ports": [ { "containerPort": 8333, - "name": "6f9c3078fd45444", }, { "containerPort": 8332, - "name": "2b5bf27123bd8f8", }, ], "resources": { @@ -125,8 +161,9 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` }, "volumeMounts": [ { - "mountPath": "/var/chainfile", - "name": "chainfile", + "mountPath": "/bitcoin/.bitcoin", + "name": "bitcoin_mainnet.json-pvc-data-c89840a1", + "subPath": "bitcoind", }, ], }, @@ -139,38 +176,26 @@ exports[`should synth bitcoin_mainnet.json and match snapshot 1`] = ` "ip": "127.0.0.1", }, ], - "volumes": [ - { - "name": "chainfile", - }, - ], + "volumes": [], }, }, - }, - }, - { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "labels": { - "bitcoind": "true", - "caip2": "bip122.000000000019d6689c085ae165831e93", - }, - "name": "bitcoin_mainnet.json-service-c8c0db95", - }, - "spec": { - "ports": [ + "volumeClaimTemplates": [ { - "name": "http", - "port": 80, - "targetPort": "2b5bf27123bd8f8", + "metadata": { + "name": "bitcoin_mainnet.json-pvc-data-c89840a1", + }, + "spec": { + "accessModes": [ + "ReadWriteOnce", + ], + "resources": { + "requests": { + "storage": "600Gi", + }, + }, + }, }, ], - "selector": { - "bitcoind": "true", - "caip2": "bip122.000000000019d6689c085ae165831e93", - }, - "type": "LoadBalancer", }, }, ] diff --git a/packages/chainfile-cdk8s/src/chart.test.ts b/packages/chainfile-cdk8s/src/chart.test.ts index 4b39b58..d20029e 100644 --- a/packages/chainfile-cdk8s/src/chart.test.ts +++ b/packages/chainfile-cdk8s/src/chart.test.ts @@ -7,12 +7,12 @@ import { Testing } from 'cdk8s'; import getPort from 'get-port'; import { version } from '../package.json'; -import { ChainfileChart } from './chart'; +import { CFChart } from './chart'; const bitcoin_mainnet: Chainfile = { caip2: 'bip122:000000000019d6689c085ae165831e93', name: 'Bitcoin Mainnet', - values: { + params: { rpc_user: { description: 'Username for RPC authentication', secret: true, @@ -34,6 +34,16 @@ const bitcoin_mainnet: Chainfile = { }, }, }, + volumes: { + data: { + type: 'persistent', + size: '600Gi', + expansion: { + startFrom: '2024-01-01', + monthlyRate: '20Gi', + }, + }, + }, containers: { bitcoind: { image: 'docker.io/kylemanna/bitcoind', @@ -49,10 +59,10 @@ const bitcoin_mainnet: Chainfile = { authorization: { type: 'HttpBasic', username: { - $value: 'rpc_user', + $param: 'rpc_user', }, password: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, probes: { @@ -81,53 +91,124 @@ const bitcoin_mainnet: Chainfile = { environment: { DISABLEWALLET: '1', RPCUSER: { - $value: 'rpc_user', + $param: 'rpc_user', }, RPCPASSWORD: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, - volumes: { - persistent: { - paths: ['/bitcoin/.bitcoin'], - size: { - initial: '600G', - from: '2024-01-01', - growth: '20G', - rate: 'monthly', - }, + mounts: [ + { + volume: 'data', + mountPath: '/bitcoin/.bitcoin', + subPath: 'bitcoind', }, - }, + ], }, }, }; -it('should synth bitcoin_mainnet.json and match snapshot', async () => { +describe('bitcoin_mainnet.json', () => { const app = Testing.app(); - const chart = new ChainfileChart(app, 'bitcoin_mainnet.json', { + const chart = new CFChart(app, 'bitcoin_mainnet.json', { chainfile: bitcoin_mainnet, - values: { + params: { rpc_user: 'user', rpc_password: 'pass', }, spec: { - deployment: { replicas: 1 }, - service: { - ports: [ - { - name: 'http', - port: 80, - target: { - container: 'bitcoind', - endpoint: 'rpc', - }, + replicas: 2, + exposes: [ + { + name: 'http', + port: 80, + target: { + container: 'bitcoind', + endpoint: 'rpc', }, - ], - }, + }, + ], }, }); const results = Testing.synth(chart); - expect(results).toMatchSnapshot(); + + it('should have 3 objects', async () => { + expect(results).toHaveLength(3); + }); + + it('should synth bitcoin_mainnet.json and match snapshot', async () => { + expect(results).toMatchSnapshot(); + }); + + it('should get labels', async () => { + expect(chart.labels).toStrictEqual({ + bitcoind: 'true', + caip2: 'bip122.000000000019d6689c085ae165831e93', + }); + }); + + it('should get serviceName', async () => { + expect(chart.serviceName).toMatch(/bitcoin_mainnet.json-service-[0-9a-f]{8}/); + }); +}); + +describe.skip('repl', () => { + // Convenient tests to repeatedly apply chart to observe behavior + // Set to skip by default + const cluster = 'repl'; + const app = Testing.app(); + const chart = new CFChart(app, cluster, { + chainfile: bitcoin_mainnet, + params: { + rpc_user: 'user', + rpc_password: 'pass', + }, + spec: { + replicas: 1, + exposes: [ + { + port: 80, + name: 'http', + target: { + container: 'bitcoind', + endpoint: 'rpc', + }, + }, + ], + }, + }); + + describe('cluster', () => { + it('should create cluster', async () => { + execSync(`kind create cluster --name ${cluster} --config kind.k8s.yaml`, { stdio: 'inherit' }); + execSync(`kind load docker-image docker.io/kylemanna/bitcoind:latest --name ${cluster}`, { stdio: 'inherit' }); + execSync(`kind load docker-image ghcr.io/vetumorg/chainfile-agent:${version} --name ${cluster}`, { + stdio: 'inherit', + }); + }); + + it('should delete cluster', async () => { + execSync(`kind delete cluster --name ${cluster}`, { stdio: 'inherit' }); + }); + }); + + it('should apply chart', async () => { + for (const resource of Testing.synth(chart)) { + execSync(`kubectl apply --context kind-${cluster} -f -`, { + input: JSON.stringify(resource), + stdio: ['pipe', 'inherit', 'inherit'], + }); + } + }); + + it('should delete chart', async () => { + for (const resource of Testing.synth(chart)) { + execSync(`kubectl delete --context kind-${cluster} -f -`, { + input: JSON.stringify(resource), + stdio: ['pipe', 'inherit', 'inherit'], + }); + } + }); }); describe('kind (k8s-in-docker)', () => { @@ -145,31 +226,29 @@ describe('kind (k8s-in-docker)', () => { }); afterAll(() => { - execSync(`kind delete cluster --name ${cluster}`, { stdio: 'inherit' }); + // execSync(`kind delete cluster --name ${cluster}`, { stdio: 'inherit' }); }); it('should deploy bitcoin_mainnet.json chart and connect to service', async () => { const app = Testing.app(); - const chart = new ChainfileChart(app, cluster, { + const chart = new CFChart(app, cluster, { chainfile: bitcoin_mainnet, - values: { + params: { rpc_user: 'user', rpc_password: 'pass', }, spec: { - deployment: { replicas: 1 }, - service: { - ports: [ - { - port: 80, - name: 'http', - target: { - container: 'bitcoind', - endpoint: 'rpc', - }, + replicas: 1, + exposes: [ + { + port: 80, + name: 'http', + target: { + container: 'bitcoind', + endpoint: 'rpc', }, - ], - }, + }, + ], }, }); @@ -189,7 +268,10 @@ describe('kind (k8s-in-docker)', () => { }); const port = await getPort(); - const forwarding = spawn('kubectl', ['port-forward', 'service/cf-bitcoin-main-service-c884ebf6', `${port}:http`], { + + // This is a simple test to check if the pod is running. + // Service routing can't actually be tested with kubectl port-forward. + const forwarding = spawn('kubectl', ['port-forward', `service/${chart.serviceName}`, `${port}:http`], { stdio: 'inherit', }); diff --git a/packages/chainfile-cdk8s/src/chart.ts b/packages/chainfile-cdk8s/src/chart.ts index 5669800..5cc674e 100644 --- a/packages/chainfile-cdk8s/src/chart.ts +++ b/packages/chainfile-cdk8s/src/chart.ts @@ -2,87 +2,108 @@ import { Chainfile, validate } from '@chainfile/schema'; import { Chart } from 'cdk8s'; import { Construct } from 'constructs'; -import { ChainfileDeployment } from './controller'; -import { ChainfileService, ChainfileServiceProps } from './service'; -import { ChainfileValues } from './values'; +import { CFSecret } from './params'; +import { CFService, CFServiceProps } from './service'; +import { CFStatefulSet } from './sts'; -interface ChainfileChartProps { +export interface CFChartProps { + chainfile: object; + /** + * Namespace to deploy the chart into. + */ namespace?: string; - metadata?: { - labels?: Record; - }; - chainfile: object; - values?: Record; + /** + * Override params in Chainfile. + * For parameters that use `default.random` + * the value must be injected as CFChart does not support generating random values to ensure reproducibility. + */ + params?: Record; + + labels?: Record; + spec: { - selector?: Record; - deployment: { - replicas: number; - }; - service: { - /** - * Ports to expose on the service. - */ - ports: ChainfileServiceProps['spec']['ports']; - }; + replicas?: number; + + /** + * Ports to expose on the service. + * Each port must have a unique name. + */ + exposes: CFServiceProps['spec']['ports']; }; } -export class ChainfileChart extends Chart { - private readonly service: ChainfileService; +export class CFChart extends Chart { + private readonly service: CFService; - constructor(scope: Construct, id: string, props: ChainfileChartProps) { + constructor(scope: Construct, id: string, props: CFChartProps) { validate(props.chainfile); const chainfile = props.chainfile as Chainfile; - const selector = props.spec.selector ?? getSelector(chainfile); + const labels = props.labels ?? CFChart.getLabels(chainfile); + super(scope, id, { namespace: props.namespace, - labels: { - ...selector, - ...(props.metadata?.labels ?? {}), - }, + labels: labels, }); - const values = new ChainfileValues(this, 'values', { + const secret = new CFSecret(this, 'secret', { chainfile: chainfile, - values: props.values, + params: props.params, }); - new ChainfileDeployment(this, 'deployment', { - chainfile: chainfile, - values: values, + this.service = new CFService(this, 'service', { + chainfile, + metadata: { + labels: labels, + }, spec: { - replicas: props.spec.deployment.replicas, - selector: selector, + type: 'LoadBalancer', + selector: labels, + ports: props.spec.exposes, }, }); - this.service = new ChainfileService(this, 'service', { + new CFStatefulSet(this, 'stateful-set', { + chainfile: chainfile, + params: secret, + metadata: { + labels: labels, + }, spec: { - type: 'LoadBalancer', - selector: selector, - ports: props.spec.service.ports, + template: { + metadata: { + labels: labels, + }, + }, + // TODO(?): https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ + serviceName: this.serviceName, + replicas: props.spec.replicas, + selector: { + matchLabels: labels, + }, }, }); } - public getServiceName(): string { + /** + * Get the service name used in the chart. + */ + public get serviceName(): string { return this.service.name; } -} -/** - * Get default labels from chainfile. - * Which includes the name of chainfile, caip2, and available containers. - * - * More specificity can be implemented by using the container version, environment, etc. - * But that is beyond the scope of a default selector. - */ -export function getSelector(chainfile: Chainfile): Record { - // Regex: (([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])? - return { - // name: chainfile.name, - caip2: chainfile.caip2.replace(/[^A-Za-z0-9]/g, '.'), - ...Object.fromEntries(Object.keys(chainfile.containers).map((name) => [name, 'true'])), - }; + /** + * Get default labels from chainfile. + * Which includes the name of chainfile, caip2, and available containers. + * + * More specificity can be implemented by using the container version, environment, etc. + * But that is beyond the scope of a default selector. + */ + public static getLabels(chainfile: Chainfile): Record { + // Regex: (([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])? + return { + caip2: chainfile.caip2.replace(/[^A-Za-z0-9]/g, '.'), + ...Object.fromEntries(Object.keys(chainfile.containers).map((name) => [name, 'true'])), + }; + } } diff --git a/packages/chainfile-cdk8s/src/container.ts b/packages/chainfile-cdk8s/src/container.ts new file mode 100644 index 0000000..e7b1ee5 --- /dev/null +++ b/packages/chainfile-cdk8s/src/container.ts @@ -0,0 +1,101 @@ +import * as schema from '@chainfile/schema'; +import { Container, EnvVar, IntOrString, Quantity, VolumeMount } from 'cdk8s-plus-25/lib/imports/k8s'; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import { version } from '../package.json'; +import { CFParamsSource } from './params'; + +interface CFAgentProps { + chainfile: schema.Chainfile; + params: CFParamsSource; +} + +export function CFAgent(props: CFAgentProps): Container { + return { + name: 'agent', + image: `ghcr.io/vetumorg/chainfile-agent:${version}`, + ports: [ + { + name: 'agent', + containerPort: 1569, + }, + ], + startupProbe: { + httpGet: { + path: '/probes/startup', + port: IntOrString.fromString('agent'), + }, + }, + livenessProbe: { + httpGet: { + path: '/probes/liveness', + port: IntOrString.fromString('agent'), + }, + }, + readinessProbe: { + httpGet: { + path: '/probes/readiness', + port: IntOrString.fromString('agent'), + }, + }, + env: [ + { + name: 'CHAINFILE_JSON', + value: JSON.stringify(props.chainfile), + }, + { + name: 'CHAINFILE_PARAMS', + valueFrom: props.params.valueFrom({ + $param: 'CHAINFILE_PARAMS', + }), + }, + ], + }; +} + +interface CFContainerProps { + params: CFParamsSource; + name: string; + container: schema.Container; + getVolumeName: (mount: schema.VolumeMount) => string; +} + +export function CFContainer(props: CFContainerProps): Container { + return { + name: props.name, + image: props.container.image + ':' + props.params.unwrap(props.container.tag), + command: props.container.command, + env: Object.entries(props.container.environment ?? {}).map(([key, valueOrRef]): EnvVar => { + if (typeof valueOrRef === 'string') { + return { + name: key, + value: valueOrRef, + }; + } + + return { + name: key, + valueFrom: props.params.valueFrom(valueOrRef), + }; + }), + ports: Object.entries(props.container.endpoints ?? {}).map(([, endpoint]) => { + return { + containerPort: endpoint.port, + }; + }), + volumeMounts: (props.container.mounts ?? []).map((mount): VolumeMount => { + return { + name: props.getVolumeName(mount), + mountPath: mount.mountPath, + subPath: mount.subPath, + }; + }), + resources: { + limits: { + cpu: Quantity.fromNumber(props.container.resources.cpu), + memory: Quantity.fromString(`${props.container.resources.memory}Mi`), + }, + }, + }; +} diff --git a/packages/chainfile-cdk8s/src/controller.ts b/packages/chainfile-cdk8s/src/controller.ts deleted file mode 100644 index 83db843..0000000 --- a/packages/chainfile-cdk8s/src/controller.ts +++ /dev/null @@ -1,240 +0,0 @@ -import { Chainfile, Container as ChainfileContainer } from '@chainfile/schema'; -import { - Container, - ContainerPort, - EnvVar, - KubeDeployment, - KubeStatefulSet, - Quantity, - VolumeMount, -} from 'cdk8s-plus-25/lib/imports/k8s'; -import { Construct } from 'constructs'; -import { createHash } from 'crypto'; - -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -import { version } from '../package.json'; -import { ValuesSource } from './values'; - -export class ChainfileDeployment extends KubeDeployment { - constructor( - scope: Construct, - id: string, - props: { - chainfile: Chainfile; - values: ValuesSource; - spec: { - selector: Record; - replicas: number; - }; - }, - ) { - const controller = new Controller(props.chainfile, props.values); - super(scope, id, { - spec: { - replicas: props.spec.replicas, - selector: { - matchLabels: props.spec.selector, - }, - template: { - spec: { - hostAliases: [ - { - ip: '127.0.0.1', - hostnames: Object.keys(props.chainfile.containers), - }, - ], - containers: controller.containers(), - volumes: controller.volumes(), - }, - metadata: { - labels: props.spec.selector, - }, - }, - }, - }); - } -} - -export class ChainfileStatefulSet extends KubeStatefulSet { - constructor( - scope: Construct, - id: string, - props: { - chainfile: Chainfile; - values: ValuesSource; - spec: { - selector: Record; - serviceName: string; - replicas: number; - }; - }, - ) { - const controller = new Controller(props.chainfile, props.values); - super(scope, id, { - spec: { - serviceName: props.spec.serviceName, - replicas: props.spec.replicas, - selector: { - matchLabels: props.spec.selector, - }, - template: { - spec: { - containers: controller.containers(), - volumes: controller.volumes(), - }, - }, - }, - }); - } -} - -class Controller { - constructor( - private readonly chainfile: Chainfile, - private readonly values: ValuesSource, - ) {} - - volumes(): VolumeMount[] { - return [ - { - name: 'chainfile', - mountPath: '/var/chainfile', - }, - ]; - } - - containers(): Container[] { - return [ - this.createAgent(), - ...Object.entries(this.chainfile.containers).map(([name, container]) => { - return this.createContainer(name, container); - }), - ]; - } - - private createAgent(): Container { - return { - name: 'agent', - image: `ghcr.io/vetumorg/chainfile-agent:${version}`, - ports: [ - { - name: 'agent', - containerPort: 1569, - }, - ], - // TODO(?): Probes (Liveness, Readiness, Startup) - volumeMounts: [ - { - name: 'chainfile', - mountPath: '/var/chainfile', - }, - ], - env: [ - { - name: 'CHAINFILE_JSON', - value: JSON.stringify(this.chainfile), - }, - { - name: 'CHAINFILE_VALUES', - valueFrom: this.values.valueFrom({ - $value: 'CHAINFILE_VALUES', - }), - }, - ], - }; - } - - private createContainer(name: string, container: ChainfileContainer): Container { - return { - name: name, - image: container.image + ':' + this.values.unwrap(container.tag), - command: container.command, - env: this.createContainerEnv(container), - ports: this.createContainerPorts(name, container), - volumeMounts: this.createContainerVolumeMounts(name, container), - resources: { - limits: { - cpu: Quantity.fromNumber(container.resources.cpu), - memory: Quantity.fromString(`${container.resources.memory}Mi`), - }, - }, - }; - } - - private createContainerEnv(container: ChainfileContainer): EnvVar[] { - if (container.environment === undefined) { - return []; - } - - return Object.entries(container.environment).map(([key, valueOrRef]): EnvVar => { - if (typeof valueOrRef === 'string') { - return { - name: key, - value: valueOrRef, - }; - } - - return { - name: key, - valueFrom: this.values.valueFrom(valueOrRef), - }; - }); - } - - private createContainerPorts(containerName: string, container: ChainfileContainer): ContainerPort[] { - if (container.endpoints === undefined) { - return []; - } - - return Object.entries(container.endpoints).map(([endpointName, endpoint]): ContainerPort => { - // TODO(?): Support Binding P2P Port Statically - return { - name: getPortName(containerName, endpointName), - containerPort: endpoint.port, - }; - }); - } - - // @ts-expect-error TODO implement volumes - // eslint-disable-next-line @typescript-eslint/no-unused-vars - private createContainerVolumeMounts(containerName: string, container: ChainfileContainer): VolumeMount[] { - const volumes: VolumeMount[] = [ - { - name: 'chainfile', - mountPath: '/var/chainfile', - }, - ]; - - // TODO(?): PV and PVC, also ephemeral, snapshot and restore. - // container.volumes?.persistent?.paths.forEach((path) => { - // volumes.push({ - // name: getVolumeName(containerName, 'persistent', path), - // mountPath: path, - // }); - // }); - // - // container.volumes?.ephemeral?.paths.forEach((path) => { - // volumes.push({ - // name: getVolumeName(containerName, 'ephemeral', path), - // mountPath: path, - // }); - // }); - - return volumes; - } -} - -export function getPortName(container: string, endpoint: string): string { - return createHash('sha256') - .update(container + '-' + endpoint) - .digest('hex') - .substring(0, 15); -} - -export function getVolumeName(container: string, type: 'persistent' | 'ephemeral', mountPath: string): string { - return createHash('sha256') - .update(container + '-' + type + '-' + mountPath) - .digest('hex') - .substring(0, 32); -} diff --git a/packages/chainfile-cdk8s/src/deploy.ts b/packages/chainfile-cdk8s/src/deploy.ts new file mode 100644 index 0000000..efac9ee --- /dev/null +++ b/packages/chainfile-cdk8s/src/deploy.ts @@ -0,0 +1,81 @@ +import * as schema from '@chainfile/schema'; +import { + DeploymentSpec, + KubeDeployment, + ObjectMeta, + PodSpec, + PodTemplateSpec, + Quantity, +} from 'cdk8s-plus-25/lib/imports/k8s'; +import { Construct } from 'constructs'; + +import { CFAgent, CFContainer } from './container'; +import { CFParamsSource } from './params'; + +export interface CFDeploymentProps { + readonly chainfile: schema.Chainfile; + readonly params: CFParamsSource; + readonly metadata?: ObjectMeta; + readonly spec: Omit & { + readonly template?: Omit & { + readonly spec?: Omit; + }; + }; + getPersistentVolumeClaimName: (mount: schema.VolumeMount) => string; +} + +/** + * Implements schema.Chainfile as a Kubernetes Deployment. + * For CFDeployment, persistent volumes are not automatically created. + * You need to create PersistentVolumeClaims separately and pass the volume name via props.getVolumeName. + */ +export class CFDeployment extends KubeDeployment { + constructor(scope: Construct, id: string, props: CFDeploymentProps) { + const volumes = props.chainfile.volumes ?? {}; + super(scope, id, { + metadata: props.metadata, + spec: { + ...props.spec, + template: { + metadata: props.spec.template?.metadata, + spec: { + ...props.spec.template?.spec, + hostAliases: [ + { + ip: '127.0.0.1', + hostnames: Object.keys(props.chainfile.containers), + }, + ], + volumes: Object.entries(volumes) + .filter(([, volume]) => volume.type === 'ephemeral') + .map(([volumeName, volume]) => { + return { + name: volumeName, + emptyDir: { + sizeLimit: Quantity.fromString(volume.size), + }, + }; + }), + containers: [ + CFAgent({ params: props.params, chainfile: props.chainfile }), + ...Object.entries(props.chainfile.containers).map(([containerName, container]) => { + return CFContainer({ + params: props.params, + name: containerName, + container, + getVolumeName: (mount) => { + if (volumes[mount.volume].type === 'ephemeral') { + return mount.volume; + } + + return props.getPersistentVolumeClaimName(mount); + }, + }); + }), + ], + }, + }, + }, + }); + } +} diff --git a/packages/chainfile-cdk8s/src/index.ts b/packages/chainfile-cdk8s/src/index.ts index e517497..27f065e 100644 --- a/packages/chainfile-cdk8s/src/index.ts +++ b/packages/chainfile-cdk8s/src/index.ts @@ -1,4 +1,6 @@ export * from './chart'; -export * from './controller'; +export * from './container'; +export * from './deploy'; +export * from './params'; export * from './service'; -export * from './values'; +export * from './sts'; diff --git a/packages/chainfile-cdk8s/src/params.ts b/packages/chainfile-cdk8s/src/params.ts new file mode 100644 index 0000000..c5907e9 --- /dev/null +++ b/packages/chainfile-cdk8s/src/params.ts @@ -0,0 +1,82 @@ +import { ComposeParams } from '@chainfile/docker'; +import * as schema from '@chainfile/schema'; +import { Names } from 'cdk8s'; +import { EnvVarSource, KubeSecret, KubeSecretProps, ObjectMeta } from 'cdk8s-plus-25/lib/imports/k8s'; +import { Construct } from 'constructs'; + +/** + * ValuesSource is a source of params for Chainfile for unwrapping and resolving references. + * This interface serves as a way to resolve params from different sources of implementations. + */ +export interface CFParamsSource { + /** + * For unwrapping and resolving references that can't be used as EnvVarSource. + */ + unwrap(param: string | schema.ParamReference): string; + + /** + * For resolving references that can be used as EnvVarSource. + */ + valueFrom(reference: schema.ParamReference): EnvVarSource; +} + +export interface CFSecretProps extends Omit { + readonly chainfile: schema.Chainfile; + /** + * Override params in Chainfile. + */ + readonly params?: Record; + readonly metadata?: ObjectMeta; +} + +/** + * CFSecret is a Kubernetes Secret implementation that provides params as secrets. + */ +export class CFSecret extends KubeSecret implements CFParamsSource { + private readonly params: Record; + + constructor(scope: Construct, id: string, { chainfile, params, ...props }: CFSecretProps) { + const values = new Cdk8sParams(chainfile).init(params); + super(scope, id, { + ...props, + type: props.type ?? 'Opaque', + stringData: { ...params, CHAINFILE_PARAMS: JSON.stringify(params) }, + metadata: { + name: Names.toDnsLabel(scope, { extra: [id] }), + ...props.metadata, + }, + }); + this.params = values; + } + + unwrap(param: string | schema.ParamReference): string { + return typeof param === 'string' ? param : this.params[param.$param]; + } + + valueFrom(reference: schema.ParamReference): EnvVarSource { + return { + secretKeyRef: { + name: this.name, + key: reference.$param, + optional: false, + }, + }; + } +} + +/** + * Based on Docker Compose's Values implementation. + */ +class Cdk8sParams extends ComposeParams { + protected default(name: string, options: NonNullable): [string, string] { + if (typeof options === 'object') { + if (options.random !== undefined) { + throw new Error( + `@chainfile/cdk8s does not support random params generation for '${name}' to ensure reproducibility and prevent irrecoverable lost of data.`, + ); + } + } + + return super.default(name, options); + } +} diff --git a/packages/chainfile-cdk8s/src/service.ts b/packages/chainfile-cdk8s/src/service.ts index 1356292..6d5be66 100644 --- a/packages/chainfile-cdk8s/src/service.ts +++ b/packages/chainfile-cdk8s/src/service.ts @@ -1,37 +1,44 @@ -import { IntOrString, KubeService, ServiceSpec } from 'cdk8s-plus-25/lib/imports/k8s'; +import * as schema from '@chainfile/schema'; +import { IntOrString, KubeService, ObjectMeta, ServicePort, ServiceSpec } from 'cdk8s-plus-25/lib/imports/k8s'; import { Construct } from 'constructs'; -import { getPortName } from './controller'; - -export interface ChainfileServiceProps { - spec: { - type: ServiceSpec['type']; - selector: Record; - ports: { - port: number; - name: string; - target: { - container: string; - endpoint: string; - }; - }[]; +export interface CFServiceProps { + readonly chainfile: schema.Chainfile; + readonly metadata?: ObjectMeta; + readonly spec: Omit & { + readonly ports: Array< + Omit & { + target: { + container: string; + endpoint: string; + }; + } + >; }; } -export class ChainfileService extends KubeService { - constructor(scope: Construct, id: string, props: ChainfileServiceProps) { +export class CFService extends KubeService { + constructor(scope: Construct, id: string, props: CFServiceProps) { super(scope, id, { spec: { type: props.spec.type, selector: props.spec.selector, ports: props.spec.ports.map((port) => { return { - port: port.port, - name: port.name, - targetPort: IntOrString.fromString(getPortName(port.target.container, port.target.endpoint)), + ...port, + targetPort: findTargetPort(props.chainfile, port.target.container, port.target.endpoint), }; }), }, }); } } + +function findTargetPort(chainfile: schema.Chainfile, container: string, endpoint: string): IntOrString { + const port = chainfile.containers[container].endpoints?.[endpoint]?.port; + if (port === undefined) { + throw new Error(`Port not found for container ${container} endpoint ${endpoint}`); + } + + return IntOrString.fromNumber(port); +} diff --git a/packages/chainfile-cdk8s/src/sts.ts b/packages/chainfile-cdk8s/src/sts.ts new file mode 100644 index 0000000..fd0f1ed --- /dev/null +++ b/packages/chainfile-cdk8s/src/sts.ts @@ -0,0 +1,101 @@ +import * as schema from '@chainfile/schema'; +import { Names } from 'cdk8s'; +import { + KubeStatefulSet, + ObjectMeta, + PodSpec, + PodTemplateSpec, + Quantity, + StatefulSetSpec, +} from 'cdk8s-plus-25/lib/imports/k8s'; +import { Construct } from 'constructs'; + +import { CFAgent, CFContainer } from './container'; +import { CFParamsSource } from './params'; + +export interface CFStatefulSetProps { + readonly chainfile: schema.Chainfile; + readonly params: CFParamsSource; + readonly metadata?: ObjectMeta; + readonly spec: Omit & { + readonly template?: Omit & { + readonly spec?: Omit; + }; + }; +} + +/** + * Implements schema.Chainfile as a Kubernetes StatefulSet. + * Persistent volumes are automatically created for volumes of type 'persistent'. + * Ephemeral volumes are created as emptyDir. + */ +export class CFStatefulSet extends KubeStatefulSet { + constructor(scope: Construct, id: string, props: CFStatefulSetProps) { + const volumes = props.chainfile.volumes ?? {}; + super(scope, id, { + metadata: props.metadata, + spec: { + ...props.spec, + template: { + metadata: props.spec.template?.metadata, + spec: { + ...props.spec.template?.spec, + hostAliases: [ + { + ip: '127.0.0.1', + hostnames: Object.keys(props.chainfile.containers), + }, + ], + volumes: Object.entries(volumes) + .filter(([, volume]) => volume.type === 'ephemeral') + .map(([volumeName, volume]) => { + return { + name: volumeName, + emptyDir: { + sizeLimit: Quantity.fromString(volume.size), + }, + }; + }), + containers: [ + CFAgent({ params: props.params, chainfile: props.chainfile }), + ...Object.entries(props.chainfile.containers).map(([containerName, container]) => { + return CFContainer({ + params: props.params, + name: containerName, + container, + getVolumeName: (mount) => { + if (volumes[mount.volume].type === 'ephemeral') { + return mount.volume; + } + + return Names.toDnsLabel(scope, { + extra: ['pvc', mount.volume], + }); + }, + }); + }), + ], + }, + }, + volumeClaimTemplates: Object.entries(volumes) + .filter(([, volume]) => volume.type === 'persistent') + .map(([volumeName, volume]) => { + return { + metadata: { + name: Names.toDnsLabel(scope, { extra: ['pvc', volumeName] }), + }, + spec: { + accessModes: ['ReadWriteOnce'], + resources: { + requests: { + // TODO(?): expansion support, + storage: Quantity.fromString(volume.size), + }, + }, + }, + }; + }), + }, + }); + } +} diff --git a/packages/chainfile-cdk8s/src/values.ts b/packages/chainfile-cdk8s/src/values.ts deleted file mode 100644 index 58bb2c3..0000000 --- a/packages/chainfile-cdk8s/src/values.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { Values } from '@chainfile/docker'; -import { Chainfile, ValueOptions, ValueReference } from '@chainfile/schema'; -import { Names } from 'cdk8s'; -import { EnvVarSource, KubeSecret } from 'cdk8s-plus-25/lib/imports/k8s'; -import { Construct } from 'constructs'; - -/** - * ValuesSource is a source of values for Chainfile for unwrapping and resolving references. - * This interface serves as a way to resolve values from different sources of implementations. - */ -export interface ValuesSource { - unwrap(value: string | ValueReference): string; - - valueFrom(reference: ValueReference): EnvVarSource; -} - -export interface ValuesProps { - chainfile: Chainfile; - /** - * Override values from Chainfile. - */ - values?: Record; -} - -export class ChainfileValues extends Construct implements ValuesSource { - private readonly values: Record; - private readonly name: string; - - constructor(scope: Construct, id: string, props: ValuesProps) { - super(scope, id); - this.values = new Cdk8sValues(props.chainfile).init(props.values); - this.name = Names.toDnsLabel(this, { extra: ['secret'] }); - - new KubeSecret(this, 'secret', { - stringData: { - CHAINFILE_VALUES: JSON.stringify(this.values), - ...this.values, - }, - type: 'Opaque', - metadata: { - name: this.name, - }, - }); - } - - unwrap(value: string | ValueReference): string { - return typeof value === 'string' ? value : this.values[value.$value]; - } - - valueFrom(reference: ValueReference): EnvVarSource { - return { - secretKeyRef: { - name: this.name, - key: reference.$value, - optional: false, - }, - }; - } -} - -/** - * Based on Docker Compose's Values implementation. - */ -class Cdk8sValues extends Values { - protected default(name: string, options: NonNullable): [string, string] { - if (typeof options === 'object') { - if (options.random !== undefined) { - throw new Error( - `@chainfile/cdk8s does not support random values generation for '${name}' to ensure reproducibility and prevent irrecoverable lost of data.`, - ); - } - } - - return super.default(name, options); - } -} diff --git a/packages/chainfile-docker/src/compose.test.ts b/packages/chainfile-docker/src/compose.test.ts index 72e790b..39e6f0f 100644 --- a/packages/chainfile-docker/src/compose.test.ts +++ b/packages/chainfile-docker/src/compose.test.ts @@ -8,7 +8,7 @@ describe('synth', () => { $schema: 'https://chainfile.org/schema.json', caip2: 'eip155:0', name: 'Example', - values: { + params: { url: 'http://${rpc_user}:${rpc_password}@dns:1234', version: {}, rpc_user: { @@ -30,11 +30,17 @@ describe('synth', () => { }, }, }, + volumes: { + data: { + type: 'ephemeral', + size: '1Gi', + }, + }, containers: { dns: { image: 'docker.io/trufflesuite/ganache', tag: { - $value: 'version', + $param: 'version', }, source: 'https://github.com/trufflesuite/ganache', endpoints: { @@ -44,10 +50,10 @@ describe('synth', () => { authorization: { type: 'HttpBasic', username: { - $value: 'rpc_user', + $param: 'rpc_user', }, password: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, probes: { @@ -65,16 +71,23 @@ describe('synth', () => { }, environment: { RPCUSER: { - $value: 'rpc_user', + $param: 'rpc_user', }, RPCPASSWORD: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, resources: { cpu: 0.25, memory: 256, }, + mounts: [ + { + volume: 'data', + mountPath: '/data', + subPath: '/dns', + }, + ], }, another: { image: 'docker.io/trufflesuite/ganache', @@ -84,13 +97,20 @@ describe('synth', () => { environment: { ENV_1: 'value_1', ENV_2: { - $value: 'url', + $param: 'url', }, }, resources: { cpu: 0.25, memory: 256, }, + mounts: [ + { + volume: 'data', + mountPath: '/data', + subPath: '/another', + }, + ], }, }, }; @@ -109,7 +129,7 @@ describe('synth', () => { 'version=v1', expect.stringMatching(/^rpc_user=[0-9a-f]{32}$/), expect.stringMatching(/^rpc_password=[0-9a-f]{32}$/), - expect.stringMatching(/^CHAINFILE_VALUES=\{.+}$/), + expect.stringMatching(/^CHAINFILE_PARAMS=\{.+}$/), ]); }); @@ -129,17 +149,14 @@ describe('synth', () => { " - '0:1569'", ' environment:', ' CHAINFILE_JSON: >-', - ' {"$$schema":"https://chainfile.org/schema.json","caip2":"eip155:0","name":"Example","values":{"url":"http://$${rpc_user}:$${rpc_password}@dns:1234","version":{},"rpc_user":{"secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}},"rpc_password":{"secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}}},"containers":{"dns":{"image":"docker.io/trufflesuite/ganache","tag":{"$$value":"version"},"source":"https://github.com/trufflesuite/ganache","endpoints":{"rpc":{"port":8545,"protocol":"HTTP', + ' {"$$schema":"https://chainfile.org/schema.json","caip2":"eip155:0","name":"Example","params":{"url":"http://$${rpc_user}:$${rpc_password}@dns:1234","version":{},"rpc_user":{"secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}},"rpc_password":{"secret":true,"default":{"random":{"bytes":16,"encoding":"hex"}}}},"volumes":{"data":{"type":"ephemeral","size":"1Gi"}},"containers":{"dns":{"image":"docker.io/trufflesuite/ganache","tag":{"$$param":"version"},"source":"https://github.com/trufflesuite/ganache","endpoints":{"rpc":{"port":8545,"protocol":"HTTP', ' JSON-RPC', - ' 2.0","authorization":{"type":"HttpBasic","username":{"$$value":"rpc_user"},"password":{"$$value":"rpc_password"}},"probes":{"readiness":{"params":[],"method":"eth_blockNumber","match":{"result":{"type":"string"}}}}}},"environment":{"RPCUSER":{"$$value":"rpc_user"},"RPCPASSWORD":{"$$value":"rpc_password"}},"resources":{"cpu":0.25,"memory":256}},"another":{"image":"docker.io/trufflesuite/ganache","tag":"v7.9.2","source":"https://github.com/trufflesuite/ganache","command":["sh","-c","echo', + ' 2.0","authorization":{"type":"HttpBasic","username":{"$$param":"rpc_user"},"password":{"$$param":"rpc_password"}},"probes":{"readiness":{"params":[],"method":"eth_blockNumber","match":{"result":{"type":"string"}}}}}},"environment":{"RPCUSER":{"$$param":"rpc_user"},"RPCPASSWORD":{"$$param":"rpc_password"}},"resources":{"cpu":0.25,"memory":256},"mounts":[{"volume":"data","mountPath":"/data","subPath":"/dns"}]},"another":{"image":"docker.io/trufflesuite/ganache","tag":"v7.9.2","source":"https://github.com/trufflesuite/ganache","command":["sh","-c","echo', ' $${ENV_1}', - ' $${ENV_2}"],"environment":{"ENV_1":"value_1","ENV_2":{"$$value":"url"}},"resources":{"cpu":0.25,"memory":256}}}}', - ' CHAINFILE_VALUES: ${CHAINFILE_VALUES}', + ' $${ENV_2}"],"environment":{"ENV_1":"value_1","ENV_2":{"$$param":"url"}},"resources":{"cpu":0.25,"memory":256},"mounts":[{"volume":"data","mountPath":"/data","subPath":"/another"}]}}}', + ' CHAINFILE_PARAMS: ${CHAINFILE_PARAMS}', expect.stringMatching(/ {6}DEBUG: .+/), - ' volumes:', - ' - type: volume', - ' source: chainfile', - ' target: /var/chainfile', + ' volumes: []', ' networks:', ' chainfile: {}', ' dns:', @@ -152,8 +169,10 @@ describe('synth', () => { " - '0:8545'", ' volumes:', ' - type: volume', - ' source: chainfile', - ' target: /var/chainfile', + ' source: data', + ' target: /data', + ' volume:', + ' subpath: /dns', ' networks:', ' chainfile: {}', ' another:', @@ -169,14 +188,16 @@ describe('synth', () => { ' ports: []', ' volumes:', ' - type: volume', - ' source: chainfile', - ' target: /var/chainfile', + ' source: data', + ' target: /data', + ' volume:', + ' subpath: /another', ' networks:', ' chainfile: {}', 'networks:', ' chainfile: {}', 'volumes:', - ' chainfile: {}', + ' data: {}', '', ]); }); diff --git a/packages/chainfile-docker/src/compose.ts b/packages/chainfile-docker/src/compose.ts index ab3fdfd..108ec78 100644 --- a/packages/chainfile-docker/src/compose.ts +++ b/packages/chainfile-docker/src/compose.ts @@ -1,6 +1,6 @@ import { randomBytes } from 'node:crypto'; -import { Chainfile, Container, validate, ValueOptions, ValueReference } from '@chainfile/schema'; +import { Chainfile, Container, ParamOptions, ParamReference, validate } from '@chainfile/schema'; import yaml from 'js-yaml'; import { version } from '../package.json'; @@ -10,26 +10,26 @@ import { version } from '../package.json'; */ export class Compose { public readonly chainfile: Chainfile; - public readonly values: Record; + public readonly params: Record; public readonly suffix: string; /** * @param chainfile definition to synthesize. - * @param values to override in the chainfile + * @param params to override in the chainfile * @param suffix for the container names to prevent conflicts. */ - constructor(chainfile: object, values: Record, suffix: string = randomBytes(4).toString('hex')) { + constructor(chainfile: object, params: Record, suffix: string = randomBytes(4).toString('hex')) { validate(chainfile); this.chainfile = chainfile as Chainfile; this.suffix = suffix; - this.values = new Values(this.chainfile).init(values); + this.params = new ComposeParams(this.chainfile).init(params); } public synthDotEnv(): string { return Object.entries({ - // TODO(?): this.values should filter out values that are not used in the compose file - ...this.values, - CHAINFILE_VALUES: JSON.stringify(this.values), + // TODO(?): this.params should filter out values that are not used in the compose file + ...this.params, + CHAINFILE_PARAMS: JSON.stringify(this.params), }) .map(([key, value]) => `${key}=${value}`) .join('\n'); @@ -47,15 +47,13 @@ export class Compose { { name: this.chainfile.name.toLowerCase().replaceAll(/[^a-z0-9_-]/g, '_'), services: { - ...this.createAgent(), - ...this.createServices(), + ...this.newAgent(), + ...this.newServices(), }, networks: { chainfile: {}, }, - volumes: { - chainfile: {}, - }, + volumes: this.newVolumes(), }, { lineWidth: 120, @@ -65,7 +63,7 @@ export class Compose { .join('\n') // Replace all $ with $$$ to escape them in the compose file .replaceAll('$', '$$$') - .replaceAll('$$$${CHAINFILE_VALUES}$$$$', '${CHAINFILE_VALUES}') + .replaceAll('$$$${CHAINFILE_PARAMS}$$$$', '${CHAINFILE_PARAMS}') // Unescape $$$${value}$$$$ to ${value} .replaceAll(/\$\$\$\$\{([a-z]+(_[a-z0-9]+)*)}\$\$\$\$/g, (_, key) => { return `$\{${key}}`; @@ -73,7 +71,16 @@ export class Compose { ); } - private createAgent(): Record<'agent', object> { + private newVolumes(): Record { + return Object.fromEntries( + Object.entries(this.chainfile.volumes ?? {}).map(([name]) => { + // Volume.type are currently ignored. + return [name, {}]; + }), + ); + } + + private newAgent(): Record<'agent', object> { return { agent: { container_name: `agent-${this.suffix}`, @@ -82,16 +89,10 @@ export class Compose { environment: { // Docker compose automatically evaluate environment literals here CHAINFILE_JSON: JSON.stringify(this.chainfile), - CHAINFILE_VALUES: '$${CHAINFILE_VALUES}$$', + CHAINFILE_PARAMS: '$${CHAINFILE_PARAMS}$$', DEBUG: process.env.DEBUG ?? 'false', }, - volumes: [ - { - type: 'volume', - source: 'chainfile', - target: '/var/chainfile', - }, - ], + volumes: [], networks: { chainfile: {}, }, @@ -99,7 +100,7 @@ export class Compose { }; } - private createServices(): Record { + private newServices(): Record { // TODO: resources (cpu, memory) is not supported for this runtime: // https://docs.docker.com/compose/compose-file/compose-file-v3/#resources // I'm not sure if we should since docker-compose typically runs on a single machine @@ -115,36 +116,17 @@ export class Compose { }); } - interface Volume { - type: 'volume'; - source?: string; - target: string; - } - - function createVolumes(container: Container): Volume[] { - const volumes: Volume[] = [ - { + function createVolumes(container: Container): object[] { + return (container.mounts ?? []).map((mount) => { + return { type: 'volume', - source: 'chainfile', - target: '/var/chainfile', - }, - ]; - - container.volumes?.persistent?.paths.forEach((path) => { - volumes.push({ - type: 'volume', - target: path, - }); - }); - - container.volumes?.ephemeral?.paths.forEach((path) => { - volumes.push({ - type: 'volume', - target: path, - }); + source: mount.volume, + target: mount.mountPath, + volume: { + subpath: mount.subPath, + }, + }; }); - - return volumes; } return Object.fromEntries( @@ -153,11 +135,11 @@ export class Compose { name, { container_name: `${name}-${this.suffix}`, - image: container.image + ':' + this.resolveValue(container.tag), + image: container.image + ':' + this.getParam(container.tag), command: container.command, environment: Object.fromEntries( Object.entries(container.environment ?? {}).map(([key, valueOrReference]) => { - return [key, this.resolveValue(valueOrReference)]; + return [key, this.getParam(valueOrReference)]; }), ), ports: createPorts(container), @@ -171,47 +153,47 @@ export class Compose { ); } - private resolveValue(value: string | ValueReference): string { - if (typeof value === 'string') { - return value; + private getParam(param: string | ParamReference): string { + if (typeof param === 'string') { + return param; } - return `$$\{${value.$value}}$$`; + return `$$\{${param.$param}}$$`; } } -export class Values { +export class ComposeParams { constructor(protected readonly chainfile: Chainfile) {} /** - * @param override values in the chainfile + * @param override params in the chainfile */ public init(override: Record = {}): Record { - if (this.chainfile.values === undefined) { + if (this.chainfile.params === undefined) { return {}; } - const values = Object.fromEntries( - Object.entries(this.chainfile.values).map(([name, value]) => { + const params = Object.fromEntries( + Object.entries(this.chainfile.params).map(([name, param]) => { if (override[name] !== undefined) { return [name, override[name]]; } - if (typeof value === 'string') { - return [name, value]; + if (typeof param === 'string') { + return [name, param]; } - if (value.default !== undefined) { - return this.default(name, value.default); + if (param.default !== undefined) { + return this.default(name, param.default); } - throw new Error(`Unsupported value: ${JSON.stringify(value)}`); + throw new Error(`Unsupported param: ${JSON.stringify(param)}`); }), ); - return this.interpolate(values); + return this.interpolate(params); } - protected default(name: string, options: NonNullable): [string, string] { + protected default(name: string, options: NonNullable): [string, string] { if (typeof options === 'string') { return [name, options]; } @@ -223,17 +205,17 @@ export class Values { throw new Error(`Default options not supported: ${JSON.stringify(options)}`); } - protected interpolate(values: Record): Record { + protected interpolate(params: Record): Record { let updated: boolean; do { updated = false; - for (const [name, value] of Object.entries(values)) { - values[name] = value.replace(/\$\{([a-z]+(_[a-z0-9]+)*)}/g, (_, key) => { + for (const [name, value] of Object.entries(params)) { + params[name] = value.replace(/\$\{([a-z]+(_[a-z0-9]+)*)}/g, (_, key) => { updated = true; - return values[key]; + return params[key]; }); } } while (updated); - return values; + return params; } } diff --git a/packages/chainfile-schema/schema.json b/packages/chainfile-schema/schema.json index c19a2ee..d9a3edc 100644 --- a/packages/chainfile-schema/schema.json +++ b/packages/chainfile-schema/schema.json @@ -28,8 +28,31 @@ "minLength": 1, "maxLength": 128 }, - "values": { - "$ref": "#/definitions/Values" + "params": { + "type": "object", + "maxProperties": 30, + "patternProperties": { + "^[a-z]+(_[a-z0-9]+)*$": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ParamOptions" + } + ] + } + }, + "additionalProperties": false + }, + "volumes": { + "type": "object", + "patternProperties": { + "^(?!chainfile)[a-z0-9][a-z0-9-]{0,28}[a-z0-9]$": { + "$ref": "#/definitions/Volume" + } + }, + "additionalProperties": false }, "containers": { "type": "object", @@ -45,24 +68,7 @@ }, "required": ["caip2", "name", "containers"] }, - "Values": { - "type": "object", - "maxProperties": 30, - "patternProperties": { - "^[a-z]+(_[a-z0-9]+)*$": { - "oneOf": [ - { - "type": "string" - }, - { - "$ref": "#/definitions/ValueOptions" - } - ] - } - }, - "additionalProperties": false - }, - "ValueOptions": { + "ParamOptions": { "type": "object", "properties": { "secret": { @@ -107,16 +113,66 @@ }, "additionalProperties": false }, - "ValueReference": { + "ParamReference": { "type": "object", "properties": { - "$value": { + "$param": { "type": "string", - "description": "Name of the value to reference in the values section.", + "description": "Name of the value to reference in the params section.", "pattern": "^[a-z]+(_[a-z0-9]+)*$" } }, - "required": ["$value"], + "required": ["$param"], + "additionalProperties": false + }, + "Volume": { + "type": "object", + "properties": { + "type": { + "enum": ["persistent", "ephemeral"] + }, + "size": { + "type": "string", + "pattern": "^[1-9][0-9]*[MGT]i$" + }, + "expansion": { + "type": "object", + "description": "Optional growth rate for volumes for evolving storage needs. [(size) + (rate * time)]", + "properties": { + "startFrom": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "description": "Formatted as YYYY-MM-DD to denote the date the growth rate starts." + }, + "monthlyRate": { + "type": "string", + "pattern": "^[1-9][0-9]*[MGT]i$", + "description": "The monthly rate at which the volume grows." + } + }, + "required": ["startFrom", "monthlyRate"], + "additionalProperties": false + } + }, + "required": ["type", "size"], + "additionalProperties": false + }, + "VolumeMount": { + "type": "object", + "properties": { + "volume": { + "type": "string", + "pattern": "^(?!chainfile)[a-z0-9][a-z0-9-]{0,28}[a-z0-9]$", + "description": "Name of the volume to mount, defined in the volumes section." + }, + "mountPath": { + "type": "string" + }, + "subPath": { + "type": "string" + } + }, + "required": ["volume", "mountPath"], "additionalProperties": false }, "Container": { @@ -132,7 +188,7 @@ "type": "string" }, { - "$ref": "#/definitions/ValueReference" + "$ref": "#/definitions/ParamReference" } ] }, @@ -189,7 +245,7 @@ "type": "string" }, { - "$ref": "#/definitions/ValueReference" + "$ref": "#/definitions/ParamReference" } ] } @@ -202,17 +258,11 @@ "type": "string" } }, - "volumes": { - "type": "object", - "properties": { - "persistent": { - "$ref": "#/definitions/ContainerVolume" - }, - "ephemeral": { - "$ref": "#/definitions/ContainerVolume" - } - }, - "additionalProperties": false + "mounts": { + "type": "array", + "items": { + "$ref": "#/definitions/VolumeMount" + } } }, "required": ["image", "tag", "source", "resources"], @@ -418,7 +468,7 @@ "type": "string" }, { - "$ref": "#/definitions/ValueReference" + "$ref": "#/definitions/ParamReference" } ] }, @@ -428,7 +478,7 @@ "type": "string" }, { - "$ref": "#/definitions/ValueReference" + "$ref": "#/definitions/ParamReference" } ] } @@ -448,7 +498,7 @@ "type": "string" }, { - "$ref": "#/definitions/ValueReference" + "$ref": "#/definitions/ParamReference" } ] } @@ -457,50 +507,6 @@ "additionalProperties": false } ] - }, - "ContainerVolume": { - "type": "object", - "properties": { - "paths": { - "type": "array", - "items": { - "type": "string" - } - }, - "size": { - "oneOf": [ - { - "type": "string", - "pattern": "^[1-9][0-9]*([MGT])$" - }, - { - "type": "object", - "properties": { - "initial": { - "type": "string", - "pattern": "^[1-9][0-9]*([MGT])$" - }, - "from": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "description": "YYYY-MM-DD" - }, - "growth": { - "type": "string", - "pattern": "^[1-9][0-9]*([MGT])$" - }, - "rate": { - "enum": ["daily", "weekly", "monthly", "yearly"] - } - }, - "required": ["initial", "from", "growth", "rate"], - "additionalProperties": false - } - ] - } - }, - "required": ["paths", "size"], - "additionalProperties": false } } } diff --git a/packages/chainfile-schema/validate.test.ts b/packages/chainfile-schema/validate.test.ts index 43fef5a..0d071ce 100644 --- a/packages/chainfile-schema/validate.test.ts +++ b/packages/chainfile-schema/validate.test.ts @@ -10,10 +10,16 @@ it('should pass validate', async () => { validate({ caip2: 'bip122:0f9188f13cb7b2c71f2a335e3a4fc328', name: 'Bitcoin Regtest', - values: { + params: { rpc_user: 'user', rpc_password: 'password', }, + volumes: { + data: { + type: 'persistent', + size: '250Mi', + }, + }, containers: { bitcoind: { image: 'docker.io/kylemanna/bitcoind', @@ -30,10 +36,10 @@ it('should pass validate', async () => { authorization: { type: 'HttpBasic', username: { - $value: 'rpc_user', + $param: 'rpc_user', }, password: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, probes: { @@ -63,25 +69,25 @@ it('should pass validate', async () => { REGTEST: '1', DISABLEWALLET: '0', RPCUSER: { - $value: 'rpc_user', + $param: 'rpc_user', }, RPCPASSWORD: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, - volumes: { - persistent: { - paths: ['/bitcoin/.bitcoin'], - size: '250M', + mounts: [ + { + volume: 'data', + mountPath: '/bitcoin/.bitcoin', }, - }, + ], }, }, }); }); describe('fail with duplicate ports', () => { - it('should fail validate with duplicate ports in a single container', async () => { + it('should fail validation with duplicate ports in a single container', async () => { expect(() => validate({ caip2: 'bip122:0f9188f13cb7b2c71f2a335e3a4fc328', @@ -110,7 +116,7 @@ describe('fail with duplicate ports', () => { ).toThrow('All ports in all containers must be unique.'); }); - it('should fail validate with duplicate ports across container', async () => { + it('should fail validation with duplicate ports across container', async () => { expect(() => validate({ caip2: 'bip122:0f9188f13cb7b2c71f2a335e3a4fc328', @@ -157,3 +163,55 @@ describe('fail with duplicate ports', () => { ).toThrow('All ports in all containers must be unique.'); }); }); + +describe('fail with missing volumes', () => { + it('should fail validation with when volume[data] is missing', async () => { + expect(() => + validate({ + caip2: 'bip122:0f9188f13cb7b2c71f2a335e3a4fc328', + name: '1 Container', + volumes: { + database: { + type: 'persistent', + size: '1Gi', + }, + }, + containers: { + btc1: { + image: 'docker.io/kylemanna/bitcoind', + tag: 'latest', + source: 'https://github.com/kylemanna/docker-bitcoind', + endpoints: {}, + mounts: [{ volume: 'data', mountPath: '/data' }], + resources: { + cpu: 0.25, + memory: 256, + }, + }, + }, + }), + ).toThrow('Volume data is not defined.'); + }); + + it('should fail validation with when volumes is undefined', async () => { + expect(() => + validate({ + caip2: 'bip122:0f9188f13cb7b2c71f2a335e3a4fc328', + name: '1 Container', + containers: { + btc2: { + image: 'docker.io/kylemanna/bitcoind', + tag: 'latest', + source: 'https://github.com/kylemanna/docker-bitcoind', + endpoints: {}, + mounts: [{ volume: 'database', mountPath: '/data' }], + resources: { + cpu: 0.25, + memory: 256, + }, + }, + }, + }), + ).toThrow('Volume database is not defined.'); + }); +}); diff --git a/packages/chainfile-schema/validate.ts b/packages/chainfile-schema/validate.ts index 672b8b5..269c4e1 100644 --- a/packages/chainfile-schema/validate.ts +++ b/packages/chainfile-schema/validate.ts @@ -8,16 +8,32 @@ addFormats(ajv); const validateFunction = ajv.compile(schema); +/** + * Validate against the schema.json file with additional checks: + * - check that all ports in all containers are unique. + * - check that all volumes are mountable. + */ export function validate(chainfile: object): void { if (!validateFunction(chainfile)) { throw new Error(ajv.errorsText(validateFunction.errors)); } + const cf = chainfile as Chainfile; // Check that all ports in all containers are unique, so they can be deployed onto a single host. - const ports = Object.values((chainfile as Chainfile).containers) + const ports = Object.values(cf.containers) .flatMap((container: Container) => Object.values(container.endpoints ?? {})) .map((endpoint) => endpoint.port); if (new Set(ports).size !== ports.length) { throw new Error('All ports in all containers must be unique.'); } + + // Check that all volumes can be mounted + const volumes = cf.volumes ?? {}; + for (const container of Object.values(cf.containers)) { + for (const mount of container.mounts ?? []) { + if (!volumes[mount.volume]) { + throw new Error(`Volume ${mount.volume} is not defined.`); + } + } + } } diff --git a/packages/chainfile-testcontainers-node/src/agent.ts b/packages/chainfile-testcontainers-node/src/agent.ts index 68c95fb..f9dc8cf 100644 --- a/packages/chainfile-testcontainers-node/src/agent.ts +++ b/packages/chainfile-testcontainers-node/src/agent.ts @@ -1,7 +1,7 @@ import { Chainfile } from '@chainfile/schema'; import { AbstractStartedContainer, StartedTestContainer } from 'testcontainers'; -export class AgentContainer extends AbstractStartedContainer { +export class CFAgentContainer extends AbstractStartedContainer { constructor(started: StartedTestContainer) { super(started); } diff --git a/packages/chainfile-testcontainers-node/src/container.ts b/packages/chainfile-testcontainers-node/src/container.ts index 02b8f72..d45347c 100644 --- a/packages/chainfile-testcontainers-node/src/container.ts +++ b/packages/chainfile-testcontainers-node/src/container.ts @@ -6,15 +6,15 @@ import { EndpointHttpAuthorization, EndpointHttpJsonRpc, EndpointHttpRest, - ValueReference, + ParamReference, } from '@chainfile/schema'; import { AbstractStartedContainer, StartedTestContainer } from 'testcontainers'; -export class ChainfileContainer extends AbstractStartedContainer { +export class CFContainer extends AbstractStartedContainer { constructor( started: StartedTestContainer, protected container: Container, - protected values: Record, + protected params: Record, ) { super(started); } @@ -98,13 +98,13 @@ export class ChainfileContainer extends AbstractStartedContainer { const type = auth.type; if (type === 'HttpBasic') { - const username = this.resolveValue(auth.username); - const password = this.resolveValue(auth.password); + const username = this.resolveParam(auth.username); + const password = this.resolveParam(auth.password); return { Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`, }; } else if (type === 'HttpBearer') { - const token = this.resolveValue(auth.token); + const token = this.resolveParam(auth.token); return { Authorization: `Bearer ${token}`, }; @@ -195,10 +195,10 @@ export class ChainfileContainer extends AbstractStartedContainer { }); } - private resolveValue(value: string | ValueReference): string { - if (typeof value === 'string') { - return value; + private resolveParam(param: string | ParamReference): string { + if (typeof param === 'string') { + return param; } - return this.values[value.$value] ?? ''; + return this.params[param.$param] ?? ''; } } diff --git a/packages/chainfile-testcontainers-node/src/testcontainers.test.ts b/packages/chainfile-testcontainers-node/src/testcontainers.test.ts index 934f359..c9b31bf 100644 --- a/packages/chainfile-testcontainers-node/src/testcontainers.test.ts +++ b/packages/chainfile-testcontainers-node/src/testcontainers.test.ts @@ -1,14 +1,14 @@ import { Chainfile } from '@chainfile/schema'; import { afterAll, beforeAll, describe, expect, it } from '@workspace/jest/globals'; -import { AgentContainer } from './agent'; -import { ChainfileTestcontainers } from './testcontainers'; +import { CFAgentContainer } from './agent'; +import { CFTestcontainers } from './testcontainers'; const chainfile: Chainfile = { $schema: 'https://chainfile.org/schema.json', caip2: 'bip122:0f9188f13cb7b2c71f2a335e3a4fc328', name: 'Bitcoin Regtest', - values: { + params: { rpc_user: 'user', rpc_password: 'password', }, @@ -24,10 +24,10 @@ const chainfile: Chainfile = { authorization: { type: 'HttpBasic', username: { - $value: 'rpc_user', + $param: 'rpc_user', }, password: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, probes: { @@ -56,10 +56,10 @@ const chainfile: Chainfile = { environment: { REGTEST: '1', RPCUSER: { - $value: 'rpc_user', + $param: 'rpc_user', }, RPCPASSWORD: { - $value: 'rpc_password', + $param: 'rpc_password', }, }, }, @@ -67,7 +67,7 @@ const chainfile: Chainfile = { }; describe('testcontainers.start()', () => { - const testcontainers = new ChainfileTestcontainers(chainfile); + const testcontainers = new CFTestcontainers(chainfile); beforeAll(async () => { await testcontainers.start(); @@ -101,7 +101,7 @@ describe('testcontainers.start()', () => { }); describe('agent', () => { - let agent: AgentContainer; + let agent: CFAgentContainer; beforeAll(() => { agent = testcontainers.getAgent(); @@ -163,8 +163,8 @@ describe('new ChainfileTestcontainers()', () => { }, }; - const test1 = new ChainfileTestcontainers(file); - const test2 = new ChainfileTestcontainers(file); + const test1 = new CFTestcontainers(file); + const test2 = new CFTestcontainers(file); expect(test1.suffix).not.toEqual(test2.suffix); }); }); diff --git a/packages/chainfile-testcontainers-node/src/testcontainers.ts b/packages/chainfile-testcontainers-node/src/testcontainers.ts index 71a8c41..14d07d9 100644 --- a/packages/chainfile-testcontainers-node/src/testcontainers.ts +++ b/packages/chainfile-testcontainers-node/src/testcontainers.ts @@ -7,10 +7,10 @@ import { Chainfile } from '@chainfile/schema'; import { DockerComposeEnvironment, StartedDockerComposeEnvironment as ComposeInstance, Wait } from 'testcontainers'; import { StartedGenericContainer } from 'testcontainers/build/generic-container/started-generic-container'; -import { AgentContainer } from './agent'; -import { ChainfileContainer } from './container'; +import { CFAgentContainer } from './agent'; +import { CFContainer } from './container'; -export class ChainfileTestcontainers { +export class CFTestcontainers { public readonly suffix = randomBytes(4).toString('hex'); protected readonly cwd: string = join(process.cwd(), '.chainfile', 'testcontainers'); protected readonly filename = `compose.${this.suffix}.yml`; @@ -19,9 +19,9 @@ export class ChainfileTestcontainers { protected composeSynth: Compose; protected composeInstance?: ComposeInstance; - public constructor(chainfile: Chainfile | object, values: Record = {}) { + public constructor(chainfile: Chainfile | object, params: Record = {}) { mkdirSync(this.cwd, { recursive: true }); - this.composeSynth = new Compose(chainfile as any, values, this.suffix); + this.composeSynth = new Compose(chainfile as any, params, this.suffix); this.chainfile = chainfile as Chainfile; } @@ -48,16 +48,16 @@ export class ChainfileTestcontainers { rmSync(join(this.cwd, this.filename)); } - get(name: string): ChainfileContainer { + get(name: string): CFContainer { const containerDef = this.chainfile.containers[name]; if (containerDef === undefined) { throw new Error(`Container ${name} not found`); } - return new ChainfileContainer(this.getContainer(name), containerDef, this.composeSynth.values); + return new CFContainer(this.getContainer(name), containerDef, this.composeSynth.params); } - getAgent(): AgentContainer { - return new AgentContainer(this.getContainer(`agent`)); + getAgent(): CFAgentContainer { + return new CFAgentContainer(this.getContainer(`agent`)); } private getContainer(name: string): StartedGenericContainer { diff --git a/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.json b/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.json index d970d63..6157ca5 100644 --- a/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.json +++ b/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.json @@ -2,7 +2,7 @@ "$schema": "../node_modules/@chainfile/schema/schema.json", "caip2": "bip122:000000000019d6689c085ae165831e93", "name": "Bitcoin Mainnet", - "values": { + "params": { "rpc_user": { "description": "Username for RPC authentication", "secret": true, @@ -24,6 +24,16 @@ } } }, + "volumes": { + "data": { + "type": "persistent", + "size": "600Gi", + "expansion": { + "startFrom": "2024-01-01", + "monthlyRate": "20Gi" + } + } + }, "containers": { "bitcoind": { "image": "docker.io/kylemanna/bitcoind", @@ -39,10 +49,10 @@ "authorization": { "type": "HttpBasic", "username": { - "$value": "rpc_user" + "$param": "rpc_user" }, "password": { - "$value": "rpc_password" + "$param": "rpc_password" } }, "probes": { @@ -71,23 +81,18 @@ "environment": { "DISABLEWALLET": "1", "RPCUSER": { - "$value": "rpc_user" + "$param": "rpc_user" }, "RPCPASSWORD": { - "$value": "rpc_password" + "$param": "rpc_password" } }, - "volumes": { - "persistent": { - "paths": ["/bitcoin/.bitcoin"], - "size": { - "initial": "600G", - "from": "2024-01-01", - "growth": "20G", - "rate": "monthly" - } + "mounts": [ + { + "volume": "data", + "mountPath": "/bitcoin/.bitcoin" } - } + ] } } } diff --git a/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.test.ts b/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.test.ts index 5051800..550309b 100644 --- a/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.test.ts +++ b/packages/chainfile-testcontainers-node/tests/bitcoin-mainnet.test.ts @@ -1,10 +1,10 @@ import { afterAll, beforeAll, describe, expect, it } from '@workspace/jest/globals'; import waitFor from '@workspace/jest/wait-for'; -import { ChainfileContainer, ChainfileTestcontainers } from '../src'; +import { CFContainer, CFTestcontainers } from '../src'; import mainnet from './bitcoin-mainnet.json'; -const testcontainers = new ChainfileTestcontainers(mainnet); +const testcontainers = new CFTestcontainers(mainnet); beforeAll(async () => { await testcontainers.start(); @@ -15,7 +15,7 @@ afterAll(async () => { }); describe('bitcoind', () => { - let bitcoind: ChainfileContainer; + let bitcoind: CFContainer; beforeAll(() => { bitcoind = testcontainers.get('bitcoind'); diff --git a/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.json b/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.json index 9d93c6e..2795179 100644 --- a/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.json +++ b/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.json @@ -2,10 +2,16 @@ "$schema": "../node_modules/@chainfile/schema/schema.json", "caip2": "bip122:0f9188f13cb7b2c71f2a335e3a4fc328", "name": "Bitcoin Regtest", - "values": { + "params": { "rpc_user": "user", "rpc_password": "password" }, + "volumes": { + "data": { + "type": "persistent", + "size": "250Mi" + } + }, "containers": { "bitcoind": { "image": "docker.io/kylemanna/bitcoind", @@ -22,10 +28,10 @@ "authorization": { "type": "HttpBasic", "username": { - "$value": "rpc_user" + "$param": "rpc_user" }, "password": { - "$value": "rpc_password" + "$param": "rpc_password" } }, "probes": { @@ -55,18 +61,18 @@ "REGTEST": "1", "DISABLEWALLET": "0", "RPCUSER": { - "$value": "rpc_user" + "$param": "rpc_user" }, "RPCPASSWORD": { - "$value": "rpc_password" + "$param": "rpc_password" } }, - "volumes": { - "persistent": { - "paths": ["/bitcoin/.bitcoin"], - "size": "250M" + "mounts": [ + { + "volume": "data", + "mountPath": "/bitcoin/.bitcoin" } - } + ] } } } diff --git a/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.test.ts b/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.test.ts index 1c81794..2b5a9c4 100644 --- a/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.test.ts +++ b/packages/chainfile-testcontainers-node/tests/bitcoin-regtest.test.ts @@ -1,9 +1,9 @@ import { afterAll, beforeAll, describe, expect, it } from '@workspace/jest/globals'; -import { ChainfileContainer, ChainfileTestcontainers } from '../src'; +import { CFContainer, CFTestcontainers } from '../src'; import regtest from './bitcoin-regtest.json'; -const testcontainers = new ChainfileTestcontainers(regtest); +const testcontainers = new CFTestcontainers(regtest); beforeAll(async () => { await testcontainers.start(); @@ -14,7 +14,7 @@ afterAll(async () => { }); describe('bitcoind', () => { - let bitcoind: ChainfileContainer; + let bitcoind: CFContainer; beforeAll(() => { bitcoind = testcontainers.get('bitcoind'); diff --git a/packages/chainfile-testcontainers-node/tests/ganache.json b/packages/chainfile-testcontainers-node/tests/ganache.json index 3660193..988c230 100644 --- a/packages/chainfile-testcontainers-node/tests/ganache.json +++ b/packages/chainfile-testcontainers-node/tests/ganache.json @@ -2,14 +2,14 @@ "$schema": "../node_modules/@chainfile/schema/schema.json", "caip2": "eip155:1337", "name": "Ganache", - "values": { + "params": { "version": "v7.9.2" }, "containers": { "ganache": { "image": "docker.io/trufflesuite/ganache", "tag": { - "$value": "version" + "$param": "version" }, "source": "https://github.com/trufflesuite/ganache", "resources": { diff --git a/packages/chainfile-testcontainers-node/tests/ganache.test.ts b/packages/chainfile-testcontainers-node/tests/ganache.test.ts index bc30314..5941e23 100644 --- a/packages/chainfile-testcontainers-node/tests/ganache.test.ts +++ b/packages/chainfile-testcontainers-node/tests/ganache.test.ts @@ -1,9 +1,9 @@ import { afterAll, beforeAll, expect, it } from '@workspace/jest/globals'; -import { ChainfileTestcontainers } from '../src'; +import { CFTestcontainers } from '../src'; import localhost from './ganache.json'; -const testcontainers = new ChainfileTestcontainers(localhost); +const testcontainers = new CFTestcontainers(localhost); beforeAll(async () => { await testcontainers.start(); diff --git a/packages/chainfile-testcontainers-node/tests/hardhat.json b/packages/chainfile-testcontainers-node/tests/hardhat.json index df6363b..3913ea7 100644 --- a/packages/chainfile-testcontainers-node/tests/hardhat.json +++ b/packages/chainfile-testcontainers-node/tests/hardhat.json @@ -2,14 +2,14 @@ "$schema": "../node_modules/@chainfile/schema/schema.json", "caip2": "eip155:31337", "name": "Hardhat", - "values": { + "params": { "version": "2.22.1" }, "containers": { "hardhat": { "image": "ghcr.io/fuxingloh/hardhat-container", "tag": { - "$value": "version" + "$param": "version" }, "source": "https://github.com/fuxingloh/hardhat-container", "resources": { diff --git a/packages/chainfile-testcontainers-node/tests/hardhat.test.ts b/packages/chainfile-testcontainers-node/tests/hardhat.test.ts index 5fccd0a..892a6c2 100644 --- a/packages/chainfile-testcontainers-node/tests/hardhat.test.ts +++ b/packages/chainfile-testcontainers-node/tests/hardhat.test.ts @@ -1,9 +1,9 @@ import { afterAll, beforeAll, expect, it } from '@workspace/jest/globals'; -import { ChainfileTestcontainers } from '../src'; +import { CFTestcontainers } from '../src'; import localhost from './hardhat.json'; -const testcontainers = new ChainfileTestcontainers(localhost); +const testcontainers = new CFTestcontainers(localhost); beforeAll(async () => { await testcontainers.start(); diff --git a/packages/chainfile-testcontainers-node/tests/solana-test-validator.json b/packages/chainfile-testcontainers-node/tests/solana-test-validator.json index a3c5d89..2255e10 100644 --- a/packages/chainfile-testcontainers-node/tests/solana-test-validator.json +++ b/packages/chainfile-testcontainers-node/tests/solana-test-validator.json @@ -2,14 +2,14 @@ "$schema": "../node_modules/@chainfile/schema/schema.json", "caip2": "solana:00000000000000000000000000000000", "name": "Solana Test Validator", - "values": { + "params": { "version": "1.17.26" }, "containers": { "solana-test-validator": { "image": "ghcr.io/fuxingloh/solana-container", "tag": { - "$value": "version" + "$param": "version" }, "source": "https://github.com/fuxingloh/solana-container", "resources": { diff --git a/packages/chainfile-testcontainers-node/tests/solana-test-validator.test.ts b/packages/chainfile-testcontainers-node/tests/solana-test-validator.test.ts index f18266b..70a3842 100644 --- a/packages/chainfile-testcontainers-node/tests/solana-test-validator.test.ts +++ b/packages/chainfile-testcontainers-node/tests/solana-test-validator.test.ts @@ -1,9 +1,9 @@ import { afterAll, beforeAll, describe, expect, it } from '@workspace/jest/globals'; -import { ChainfileContainer, ChainfileTestcontainers } from '../src'; +import { CFContainer, CFTestcontainers } from '../src'; import solana from './solana-test-validator.json'; -const testcontainers = new ChainfileTestcontainers(solana); +const testcontainers = new CFTestcontainers(solana); beforeAll(async () => { await testcontainers.start(); @@ -14,7 +14,7 @@ afterAll(async () => { }); describe('solana-test-validator', () => { - let validator: ChainfileContainer; + let validator: CFContainer; beforeAll(() => { validator = testcontainers.get('solana-test-validator'); diff --git a/website/components/ChainfileDefinition.mdx b/website/components/ChainfileDefinition.mdx index 52a34c2..37c6494 100644 --- a/website/components/ChainfileDefinition.mdx +++ b/website/components/ChainfileDefinition.mdx @@ -1,12 +1,12 @@

- Values + Params

-These values have reasonable defaults or are randomly generated values for secured fields +These params have reasonable defaults or are randomly generated for secured fields which will determine how the Chainfile is run. -You may override these values to customize the Chainfile for your specific needs. +You may override these params to customize the Chainfile for your specific needs. -{Object.entries(props.schema.values).map(([key, value]) => { +{Object.entries(props.schema.params ?? {}).map(([key, value]) => { const defaultValue = typeof value === 'string' ? value : typeof value.default === 'string' ? value.default : null; const random = typeof value.default === 'object' ? value.default.random : null; return ( diff --git a/website/pages/core-concepts/schema.mdx b/website/pages/core-concepts/schema.mdx index 2050151..2653b9a 100644 --- a/website/pages/core-concepts/schema.mdx +++ b/website/pages/core-concepts/schema.mdx @@ -9,21 +9,42 @@ import { Callout } from 'nextra/components'; "$schema": "https://chainfile.org/schema.json", "caip2": "bip122:000000000019d6689c085ae165831e93", "name": "Bitcoin Mainnet", - "env": { - "RPC_USER": { - "type": "RandomBytes", - "length": 16, - "encoding": "hex" + "params": { + "rpc_user": { + "description": "Username for RPC authentication", + "secret": true, + "default": { + "random": { + "bytes": 16, + "encoding": "hex" + } + } }, - "RPC_PASSWORD": { - "type": "RandomBytes", - "length": 16, - "encoding": "hex" + "rpc_password": { + "description": "Password for RPC authentication", + "secret": true, + "default": { + "random": { + "bytes": 16, + "encoding": "hex" + } + } + } + }, + "volumes": { + "data": { + "type": "persistent", + "size": "600Gi", + "expansion": { + "startFrom": "2024-01-01", + "monthlyRate": "20Gi" + } } }, "containers": { "bitcoind": { - "image": "docker.io/kylemanna/bitcoind@sha256:1492fa0306cb7eb5de8d50ba60367cff8d29b00b516e45e93e05f8b54fa2970e", + "image": "docker.io/kylemanna/bitcoind", + "tag": "latest", "source": "https://github.com/kylemanna/docker-bitcoind", "endpoints": { "p2p": { @@ -35,10 +56,10 @@ import { Callout } from 'nextra/components'; "authorization": { "type": "HttpBasic", "username": { - "$value": "RPC_USER" + "$param": "rpc_user" }, "password": { - "$value": "RPC_PASSWORD" + "$param": "rpc_password" } }, "probes": { @@ -67,23 +88,18 @@ import { Callout } from 'nextra/components'; "environment": { "DISABLEWALLET": "1", "RPCUSER": { - "$value": "RPC_USER" + "$param": "rpc_user" }, "RPCPASSWORD": { - "$value": "RPC_PASSWORD" + "$param": "rpc_password" } }, - "volumes": { - "persistent": { - "paths": ["/bitcoin/.bitcoin"], - "size": { - "initial": "600G", - "from": "2024-01-01", - "growth": "20G", - "rate": "monthly" - } + "mounts": [ + { + "volume": "data", + "mountPath": "/bitcoin/.bitcoin" } - } + ] } } }