From 06a8c3ebcc6e8d8bf8983ff8b18ca19eaf0f3831 Mon Sep 17 00:00:00 2001 From: 0x009922 <43530070+0x009922@users.noreply.github.com> Date: Thu, 20 Feb 2025 11:59:13 +0900 Subject: [PATCH] build: enhance prep workflow, update the guide (#228) --- .github/workflows/pull-request.yml | 36 +++---- CONTRIBUTING.md | 83 +++++++--------- deno.jsonc | 52 +++++----- etc/task-codegen.ts | 7 +- etc/task-prep-crypto-wasm.ts | 38 +++++--- etc/task-prep-iroha.ts | 150 ++++++++++++++--------------- tests/node/deno.jsonc | 2 +- 7 files changed, 185 insertions(+), 183 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 52265378..de3bfc20 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -64,8 +64,9 @@ jobs: deno task prep:iroha --git $IROHA_GIT --git-rev $IROHA_REV deno task prep:iroha:build - check: - needs: ['prep-crypto-wasm', 'prep-iroha'] + check-only: + if: github.event_name == 'pull_request' + needs: [prep-iroha, prep-crypto-wasm] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -83,7 +84,7 @@ jobs: prep/crypto-wasm - uses: denoland/setup-deno@v2 with: - deno-version: v2.x + deno-version: v2.2.x - uses: actions/setup-node@v4 with: node-version: 22 @@ -93,20 +94,11 @@ jobs: npm install -g corepack@latest echo "After : corepack version => $(corepack --version)" corepack enable pnpm - - name: Deno install - run: deno task install - - name: Deno check - run: deno task check:all - - name: Deno lint, fmt - run: | - deno lint - deno fmt --check - - name: Test - run: deno task test + - run: deno task ok - publish: + check-and-publish: if: github.event_name == 'push' - needs: [check] + needs: [prep-iroha, prep-crypto-wasm] runs-on: ubuntu-latest permissions: contents: read @@ -127,6 +119,14 @@ jobs: prep/crypto-wasm - uses: denoland/setup-deno@v2 with: - deno-version: v2.x - - name: Publish - run: deno task publish + deno-version: v2.2.x + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Enable pnpm via Corepack + run: | + echo "Before: corepack version => $(corepack --version || echo 'not installed')" + npm install -g corepack@latest + echo "After : corepack version => $(corepack --version)" + corepack enable pnpm + - run: deno task publish diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c7a39d8..ef26ca03 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,71 +2,58 @@ This document explains how this repo works, and how to work with it. -Steps, in short: +## Pull Requests -1. Install Rust and `wasm-pack`. -2. Install Deno v2 and Node.js v22 -3. Link Iroha repository -4. Explore `tasks` in the root `deno.json` -5. Work with the code +1. [Install the Deno CLI](https://docs.deno.com/runtime/manual/getting_started/installation). +2. [Install Node.js v22+](https://nodejs.org/en/download). + - And enable Corepack by `corepack enable` (making `pnpm` available). +3. Install [the Rust toolchain](https://rustup.rs/) and [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer/). + - Also make sure to install components: `rustup component add rust-src --target wasm32-unknown-unknown`. +4. Clone & fork this repository. +5. Link Iroha repository (see [below](#linking--building-iroha)). +6. Create a new branch for your changes. +7. Make your changes to the repo and ensure `deno task ok` passes successfully. +8. Commit your changes with clear messages, preferably following + [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). +9. Submit a pull request. -TODO: setting up commit hooks +## Linking & Building Iroha -## Installing Rust and `wasm-pack` +For this project to function, you must link Iroha repository. There are two ways to do so. -```sh -# install rustup -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - -# add necessary components -rustup component add rust-src --target wasm32-unknown-unknown +**Clone Iroha repository:** -# install wasm-pack -curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh +```sh +deno task prep:iroha --git https://github.com/hyperledger-iroha/iroha.git --git-rev v2.0.0-rc.1.0 ``` -## Installing Deno & Node.js - -This must be pretty straightforward. A few notes though: - -- While this project is mostly driven by Deno, there are some Node.js parts, unfortunately. Specifically, - `tests/browser` is a `pnpm` project, because otherwise it's hard to make Cypress work. -- Thus, run `corepack enable` (or install `npm i -g pnpm`) so that `pnpm` is also available. - -## Linking & building Iroha - -For this project to function, you must link Iroha repository. There are two ways to do so: +**Symlink to a local path:** ```sh -# symlink to the local path deno task prep:iroha --path /path/to/local/iroha/clone ``` -```sh -# clone the repo -deno task prep:iroha --git https://github.com/hyperledger-iroha/iroha.git --git-rev v2.0.0-rc.1.0 -``` +## Making Releases -After Iroha is linked, you need to prepare some artifacts from it (binaries such as `irohad`, `kagami`, `iroha_codec`; -`executor.wasm`; `schema.json`): +[`@deno/bump-workspaces`](https://github.com/denoland/bump-workspaces) could be of help: -```sh -deno task prep:iroha:build +```shell +deno run -A jsr:@deno/bump-workspaces@0.1.22/cli --dry-run ``` -## Running tests +Guideline: -Please explore `tasks` in the root `deno.jsonc`. +1. Bump versions using the tool. +2. Submit a PR with the changes of the versions and `Releases.md`. +3. Merge, tag the commit in format `release-2025-02-20`, and push it. +4. Create a release on GitHub at this tag, using the new piece of `Releases.md` as a description and date (e.g. + "2025.02.20") as a title. -```sh -# run them all -deno task test +## Setting Up Commit Hooks _(optional)_ -# unit, non-integration tests -deno run npm:vitest -# or -pnpm dlx vitest -``` +You can add this to `.git/hooks/pre-commit`: -Tests are mostly written in Vitest (except the doctests), and it doesn't work very well with Deno (considering -migration). To improve development experience, consider running Vitest via `pnpm dlx` or similar. +```shell +#!/bin/sh +deno task ok +``` diff --git a/deno.jsonc b/deno.jsonc index b7f524d4..c8ef1648 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -14,47 +14,41 @@ ], "tasks": { "install": "deno install --allow-scripts=npm:cypress,npm:@parcel/watcher,npm:vue-demi", - "prep:iroha": "deno run --allow-all ./etc/task-prep-iroha.ts", - "prep:iroha:check": { - "description": "Ensure that Iroha is built", - "command": "deno task prep:iroha --check" - }, - "prep:iroha:build": { - "description": "Build Iroha artifacts and produce `/prep/iroha` directory", - "command": "deno task prep:iroha --build" + "prep:iroha": { + "description": "Prepare Iroha. Must be manually linked first (via `--path` or `--git`)", + "command": "deno run -A ./etc/task-prep-iroha.ts" }, "prep:crypto-wasm": { - "description": "Prepare crypto wasm and put into `@iroha/core`", + "description": "Prepare crypto wasm and put it into `@iroha/core`", "command": "deno run -A ./etc/task-prep-crypto-wasm.ts" }, - "prep:crypto-wasm:copy": { - "description": "Only copy already built crypto wasm into `@iroha/core`", - "command": "deno task prep:crypto-wasm --copy" - }, "prep:codegen": { - "command": "deno run --allow-write --allow-read ./etc/task-codegen.ts", - "dependencies": ["prep:iroha:check"] + "command": "deno run -WR ./etc/task-codegen.ts", + "dependencies": ["prep:iroha"] }, "prep:codegen:watch": "watchexec -e ts deno task prep:codegen", - "prep:ensure-ready": { - "dependencies": ["prep:codegen", "prep:crypto-wasm:copy", "prep:iroha:check"] + "prep:ok": { + "dependencies": ["prep:codegen", "prep:crypto-wasm", "prep:iroha"] }, - "check:all": { + "check": { "command": "deno check --doc .", - "dependencies": ["prep:ensure-ready"] + "dependencies": ["prep:ok"] + }, + "test:deno": { + "command": "deno test --doc", + "dependencies": ["prep:ok"] }, - "test:deno": "deno test --doc --allow-read", "test:vitest": { - "dependencies": ["prep:ensure-ready"], + "dependencies": ["prep:ok"], "description": "Run Vitest", "command": "vitest run" }, "test:integration:node": { - "dependencies": ["prep:ensure-ready"], + "dependencies": ["prep:ok"], "command": "cd tests/node && deno task test" }, "test:integration:browser": { - "dependencies": ["prep:ensure-ready"], + "dependencies": ["prep:ok"], "command": "cd tests/browser && deno task test" }, "test": { @@ -62,14 +56,20 @@ "command": "deno task test:deno && deno task test:vitest && deno task test:integration:node && deno task test:integration:browser" }, "dev:run-test-peer": { - "dependencies": ["prep:ensure-ready"], + "description": "Run an Iroha peer with test configuration. Could be useful for development", + "dependencies": ["prep:ok"], "command": "cd tests/support/test-peer && deno task run" }, + "ok": { + "description": "Run all checks", + "dependencies": ["prep:ok", "install"], + "command": "deno lint && deno fmt --check && deno task check && deno task test" + }, "publish": { // https://github.com/denoland/deno/issues/28096 "description": "Publish workspace packages", - "command": "deno run --allow-read --allow-run --allow-env ./etc/task-publish.ts", - "dependencies": ["check:all"] + "command": "deno run -A ./etc/task-publish.ts", + "dependencies": ["ok"] } }, "fmt": { diff --git a/etc/task-codegen.ts b/etc/task-codegen.ts index 09b51a9f..bc6a6fce 100644 --- a/etc/task-codegen.ts +++ b/etc/task-codegen.ts @@ -8,7 +8,12 @@ import * as dprintTS from 'npm:@dprint/typescript' import * as colors from '@std/fmt/colors' const tsFormatter = dprint.createFromBuffer(await Deno.readFile(dprintTS.getPath())) -const formatTS = (code: string) => tsFormatter.formatText({ filePath: 'file.ts', fileText: code }) +const formatTS = (code: string) => { + console.time('dprint') + const text = tsFormatter.formatText({ filePath: 'file.ts', fileText: code }) + console.timeEnd('dprint') + return text +} async function write({ file, code }: { file: string; code: string }) { let status: string diff --git a/etc/task-prep-crypto-wasm.ts b/etc/task-prep-crypto-wasm.ts index f9793d16..51d0cf45 100644 --- a/etc/task-prep-crypto-wasm.ts +++ b/etc/task-prep-crypto-wasm.ts @@ -4,6 +4,8 @@ import $ from 'jsr:@david/dax' import * as colors from '@std/fmt/colors' import { pathRel, resolveFromRoot } from './util.ts' import { copy, emptyDir } from 'jsr:@std/fs' +import { assert } from '@std/assert/assert' +import { assertEquals } from '@std/assert/equals' const PROJECT_DIR = resolveFromRoot('crypto-wasm') const PREP_DIR = resolveFromRoot('prep/crypto-wasm') @@ -12,37 +14,51 @@ const CRATE_NAME = 'iroha_crypto_wasm' async function buildWasm() { const outDir = PREP_DIR - // TODO: empty dir? + console.log(' ' + colors.yellow(`empty ${pathRel(PREP_DIR)}`)) + await emptyDir(PREP_DIR) - $.logStep(`Running @deno/wasmbuild...`) + $.logStep(`Building with @deno/wasmbuild...`) await $`deno run -A jsr:@deno/wasmbuild --out ${outDir}`.cwd(PROJECT_DIR) - console.log(` ${colors.yellow(`wasmbuild ready at ${pathRel(outDir)}`)}'`) + console.log(` ${colors.yellow(`write ${pathRel(outDir)}`)}`) +} + +async function checkBuildReady() { + try { + const files = new Set() + for await (const i of Deno.readDir(PREP_DIR)) { + assert(i.isFile) + files.add(i.name) + } + assertEquals(files, new Set([`.d.ts`, '.internal.js', '.js', '.wasm'].map((x) => `${CRATE_NAME}${x}`))) + return true + } catch (err) { + $.logWarn('Error whiule checking build artifacts:', err) + return false + } } async function copyOutputs() { const targetDir = resolveFromRoot('packages/core/crypto/wasm') const files = [`${CRATE_NAME}.d.ts`, `${CRATE_NAME}.wasm`, `${CRATE_NAME}.internal.js`] + console.log(' ' + colors.yellow(`empty ${pathRel(targetDir)}`)) await emptyDir(targetDir) for (const i of files) { const src = resolveFromRoot(PREP_DIR, i) const dest = path.join(targetDir, i) console.log(` ${colors.yellow(`write ${pathRel(dest)}`)}`) - try { - await copy(src, dest) - } catch (err) { - $.logError(`Failed to copy. Is ${PREP_DIR} ready?`) - throw err - } + await copy(src, dest) } } const args = parseArgs(Deno.args, { - boolean: ['copy'], + boolean: ['force'], }) -if (!args.copy) { +if (!args.force && (await checkBuildReady())) { + $.logStep(`Skipping build step (override with ${colors.cyan('--force')})`) +} else { await buildWasm() } await copyOutputs() diff --git a/etc/task-prep-iroha.ts b/etc/task-prep-iroha.ts index 4a67d5ed..c19a7853 100644 --- a/etc/task-prep-iroha.ts +++ b/etc/task-prep-iroha.ts @@ -1,10 +1,9 @@ import { parseArgs } from 'jsr:@std/cli/parse-args' -import { resolveFromRoot } from './util.ts' -import { match, P } from 'ts-pattern' +import { pathRel, resolveFromRoot } from './util.ts' import * as path from 'jsr:@std/path' import * as colors from 'jsr:@std/fmt/colors' import $ from 'jsr:@david/dax' -import { assert } from '@std/assert' +import { assert, assertEquals } from '@std/assert' import { copy, emptyDir } from 'jsr:@std/fs' const IROHA_REPO_DIR = resolveFromRoot('.iroha') @@ -22,11 +21,29 @@ async function dirExists(dir: string) { } async function assertRepoIsReady() { - assert(await dirExists(IROHA_REPO_DIR), 'Iroha repo is not ready; make sure to run "deno task prep:iroha" first.') + if (await dirExists(IROHA_REPO_DIR)) return + + $.logError('Cannot find Iroha repository') + const cmd = `deno task prep:iroha` + console.log(`You shall link it first by either + ${ + colors.magenta( + cmd + colors.bold(` --git ${colors.gray('')} --git-rev ${colors.gray('')}`), + ) + } +or + ${colors.magenta(cmd + colors.bold(` --path ${colors.gray('')}`))}`) + + Deno.exit(1) } async function clean() { - await emptyDir(IROHA_REPO_DIR) + console.log(' ' + colors.yellow(`remove ${pathRel(IROHA_REPO_DIR)}`)) + try { + await Deno.remove(IROHA_REPO_DIR, { recursive: true }) + } catch (err) { + if (!(err instanceof Deno.errors.NotFound)) throw err + } } async function linkPath(target: string) { @@ -47,11 +64,10 @@ async function cloneRepo(repo: string, tagOrRevision: string) { await $`git fetch origin ${tagOrRevision}`.cwd(IROHA_REPO_DIR) await $`git reset --hard FETCH_HEAD`.cwd(IROHA_REPO_DIR) - $.logStep('Finished cloning repo') + $.logStep('Cloned repo') } async function buildBinaries() { - await assertRepoIsReady() const binaries = ['irohad', 'iroha_kagami', 'iroha_codec'] $.logStep('Building binaries:', binaries) const args = binaries.map((x) => ['-p', x]) @@ -59,15 +75,22 @@ async function buildBinaries() { $.logStep('Finished building binaries') } -async function buildWasms() { - await assertRepoIsReady() - $.logStep('Building lib wasms') +async function buildWasmLibs() { + $.logStep('Building WASM libs') await $`/bin/bash ./scripts/build_wasm.sh libs`.cwd(IROHA_REPO_DIR) - $.logStep('Finished building WASMs') + $.logStep('Finished WASM libs') +} + +async function copySchemaJson() { + const dest = resolveFromRoot('packages/core/data-model/schema/schema.json') + console.log(' ' + colors.yellow(`write ${pathRel(dest)}`)) + await copy(path.join(PREP_OUTPUT_DIR, 'schema.json'), dest, { overwrite: true }) } -async function copyArtifacts() { +async function copyFromRepoToPrep() { + console.log(` ${colors.yellow(`empty ${pathRel(PREP_OUTPUT_DIR)}`)}`) await emptyDir(PREP_OUTPUT_DIR) + for ( const artifactPath of [ 'target/release/irohad', @@ -77,81 +100,52 @@ async function copyArtifacts() { 'docs/source/references/schema.json', ] ) { - await copy(path.join(IROHA_REPO_DIR, artifactPath), path.join(PREP_OUTPUT_DIR, '/', path.basename(artifactPath))) + const out = path.join(PREP_OUTPUT_DIR, '/', path.basename(artifactPath)) + console.log(` ` + colors.yellow(`write ${pathRel(out)}`)) + await copy(path.join(IROHA_REPO_DIR, artifactPath), out) } - $.logStep(`Finished copying artifacts to ${colors.bold(colors.cyan(PREP_OUTPUT_DIR))}`) } -async function copySchemaJson() { - const dest = resolveFromRoot('packages/core/data-model/schema/schema.json') - +async function artifactsReady(): Promise { try { - await Deno.remove(dest) - } catch (err) { - if (!(err instanceof Deno.errors.NotFound)) { - throw err + const files = new Set() + for await (const i of Deno.readDir(PREP_OUTPUT_DIR)) { + assert(i.isFile) + files.add(path.basename(i.name)) } + assertEquals( + files, + new Set(['irohad', 'iroha_codec', 'kagami', 'executor.wasm', 'schema.json']), + 'all artifacts must be present', + ) + return true + } catch (error) { + $.logWarn('Error while checking artifacts:', error) + return false } - - await copy(path.join(PREP_OUTPUT_DIR, 'schema.json'), dest) - $.logStep('Copied', dest) } const args = parseArgs(Deno.args, { string: ['git', 'git-rev', 'path'], - boolean: ['check', 'build'], + boolean: ['force'], }) -await match(args) - .with( - { git: P.string, 'git-rev': P.string }, - async ({ git: repo, 'git-rev': tagOrRevision }) => { - await cloneRepo(repo, tagOrRevision) - }, - ) - .with({ path: P.string }, async ({ path }) => { - await linkPath(path) - }) - .with({ check: true }, async () => { - if (await dirExists(PREP_OUTPUT_DIR)) { - $.logStep(`Checked that ${colors.cyan(PREP_OUTPUT_DIR)} exists`) - await copySchemaJson() - } else { - $.logError( - `Error: ${PREP_OUTPUT_DIR} doesn't exist. Make sure to run ${ - colors.bold(colors.magenta('deno task prep:iroha:build')) - } first`, - ) - Deno.exit(1) - } - }) - .with({ build: true }, async () => { - await buildBinaries() - await buildWasms() - await copyArtifacts() - }) - .otherwise(() => { - const cmd = colors.gray(`deno task prep:iroha`) - - console.info(`Usage: - - ${colors.blue('## Link Iroha')} - - ${colors.magenta(`${cmd} --git --git-rev `)} - to clone Iroha repo - - or - - ${colors.magenta(`${cmd} --path `)} - to symlink a local Iroha repo - - ${colors.blue('## Build')} - - ${colors.magenta(`${cmd} --build`)} - to build Iroha artifacts - - ${colors.magenta(`${cmd} --check`)} - to check that the artifacts are ready - `) - Deno.exit(1) - }) +if (args.git) { + assert(args['git-rev'], '--git requires --git-rev') + assert(!args.path, 'either --git or --path, not both') + await cloneRepo(args.git, args['git-rev']) +} else if (args.path) { + assert(!args.git && !args['git-rev'], `--path conflicts with --git and --git-rev`) + await linkPath(args.path) +} + +if (!args.force && (await artifactsReady())) { + $.logStep(`Skipping build step (override with ${colors.cyan('--force')})`) +} else { + await assertRepoIsReady() + await buildBinaries() + await buildWasmLibs() + await copyFromRepoToPrep() +} + +await copySchemaJson() diff --git a/tests/node/deno.jsonc b/tests/node/deno.jsonc index 2c2da465..93a05f5d 100644 --- a/tests/node/deno.jsonc +++ b/tests/node/deno.jsonc @@ -1,5 +1,5 @@ { "tasks": { - "test": "DEBUG=\"@iroha*\" vitest" + "test": "DEBUG=\"@iroha*\" vitest run" } }