From c50a7e01ff4ed29c634cea556aaa284a47557400 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sat, 21 Jan 2023 00:17:44 +0800 Subject: [PATCH 1/8] emnapi WebAssembly build --- .github/workflows/ci.yml | 26 ++++++++++++++ binding.gyp | 57 ++++++++++++++++++++++++------- lib/sqlite3-binding.js | 6 +++- package.json | 2 ++ test/async_calls.test.js | 7 ++++ wasm/common.gypi | 73 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 158 insertions(+), 13 deletions(-) create mode 100644 wasm/common.gypi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a68cc5e25..1028f7418 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,10 @@ jobs: node: 16 host: arm64 target: arm64 + - os: ubuntu-20.04 + node: 16 + host: x64 + target: wasm32 name: ${{ matrix.os }} (node=${{ matrix.node }}, host=${{ matrix.host }}, target=${{ matrix.target }}) steps: - uses: actions/checkout@v3 @@ -52,6 +56,13 @@ jobs: with: node-version: ${{ matrix.node }} architecture: ${{ matrix.host }} + + - name: Setup Emscripten (WebAssembly) + if: matrix.target == 'wasm32' + uses: mymindstorm/setup-emsdk@v11 + with: + version: '3.1.29' + actions-cache-folder: 'emsdk-cache' - name: Add yarn (self-hosted) if: matrix.os == 'macos-m1' @@ -84,10 +95,20 @@ jobs: echo "CXXFLAGS=${CXXFLAGS:-} -include ../src/gcc-preinclude.h" >> $GITHUB_ENV - name: Configure build + if: matrix.target != 'wasm32' run: yarn node-pre-gyp configure --target_arch=${{ env.TARGET }} + + - name: Configure build (WebAssembly) + if: matrix.target == 'wasm32' + run: yarn node-pre-gyp configure --nodedir=./wasm --target_arch=${{ env.TARGET }} - name: Build binaries + if: matrix.target != 'wasm32' run: yarn node-pre-gyp build --target_arch=${{ env.TARGET }} + + - name: Build binaries (WebAssembly) + if: matrix.target == 'wasm32' + run: emmake yarn node-pre-gyp build --nodedir=./wasm --target_arch=${{ env.TARGET }} - name: Print binary info if: contains(matrix.os, 'ubuntu') @@ -99,7 +120,12 @@ jobs: file lib/binding/napi-v*/* - name: Run tests + if: matrix.target != 'wasm32' run: yarn test + + - name: Run tests (WebAssembly) + if: matrix.target == 'wasm32' + run: yarn test --arch=wasm32 - name: Package prebuilt binaries run: yarn node-pre-gyp package --target_arch=${{ env.TARGET }} diff --git a/binding.gyp b/binding.gyp index f1336f6f7..be570b9ec 100644 --- a/binding.gyp +++ b/binding.gyp @@ -16,10 +16,26 @@ "msvs_settings": { "VCCLCompilerTool": { "ExceptionHandling": 1 }, }, - "include_dirs": [ - " JSON.stringify(path.relative(process.cwd(), x))).join(\' \')")' + ], + + 'default_configuration': 'Release', + 'configurations': { + 'Debug': { + 'defines': [ 'DEBUG', '_DEBUG' ], + 'cflags': [ '-g', '-O0' ], + 'ldflags': [ '-g', '-O0', '-sSAFE_HEAP=1' ], + }, + 'Release': { + 'cflags': [ '-O3' ], + 'ldflags': [ '-O3' ], + } + }, + + 'conditions': [ + ['target_arch == "wasm64"', { + 'cflags': [ + '-sMEMORY64=1', + ], + 'ldflags': [ + '-sMEMORY64=1' + ] + }], + ['wasm_threads == 1', { + 'cflags': [ '-sUSE_PTHREADS=1' ], + 'ldflags': [ '-sUSE_PTHREADS=1' ], + }], + ], + } +} From 2045955f69ff4e92305242a5b57b1e05a445128f Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sat, 21 Jan 2023 00:26:47 +0800 Subject: [PATCH 2/8] Fix ci test script --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1028f7418..8965b37cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: - name: Run tests (WebAssembly) if: matrix.target == 'wasm32' - run: yarn test --arch=wasm32 + run: npx --arch=wasm32 yarn test - name: Package prebuilt binaries run: yarn node-pre-gyp package --target_arch=${{ env.TARGET }} From 153baebac54261fd959d338c7e128ff25a498708 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Tue, 24 Jan 2023 22:41:57 +0800 Subject: [PATCH 3/8] support async_hooks --- lib/sqlite3-binding.js | 16 ++++++++++++++-- package.json | 5 +++-- test/async_calls.test.js | 11 +++++------ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/sqlite3-binding.js b/lib/sqlite3-binding.js index bfbd68bd7..1f2135235 100644 --- a/lib/sqlite3-binding.js +++ b/lib/sqlite3-binding.js @@ -1,9 +1,21 @@ const binary = require('@mapbox/node-pre-gyp'); const path = require('path'); -const binding_path = binary.find(path.resolve(path.join(__dirname,'../package.json'))); +const binding_path = binary.find(path.resolve(path.join(__dirname,'../package.json')), { + target_arch: process.env.npm_config_target_arch || process.env.npm_config_arch || process.arch +}); let binding = require(binding_path); if (typeof binding === 'function') { const emnapiContext = require('@tybys/emnapi-runtime').createContext(); - binding = binding().emnapiInit({ context: emnapiContext }); + const emnapiInitOptions = { + context: emnapiContext + }; + try { + // optional dependency + // support async_hooks on Node.js + emnapiInitOptions.nodeBinding = require('@tybys/emnapi-node-binding'); + } catch (_) { + // ignore + } + binding = binding().emnapiInit(emnapiInitOptions); } module.exports = exports = binding; diff --git a/package.json b/package.json index c17b62fb4..0d210d4e7 100644 --- a/package.json +++ b/package.json @@ -49,12 +49,12 @@ }, "dependencies": { "@mapbox/node-pre-gyp": "^1.0.0", + "@tybys/emnapi-runtime": "^0.28.1", "node-addon-api": "^4.2.0", "tar": "^6.1.11" }, "devDependencies": { - "@tybys/emnapi": "^0.27.0", - "@tybys/emnapi-runtime": "^0.27.0", + "@tybys/emnapi": "^0.28.1", "eslint": "6.8.0", "mocha": "7.2.0", "node-pre-gyp-github": "1.4.4" @@ -68,6 +68,7 @@ } }, "optionalDependencies": { + "@tybys/emnapi-node-binding": "^0.28.1", "node-gyp": "8.x" }, "scripts": { diff --git a/test/async_calls.test.js b/test/async_calls.test.js index b76459864..6f2ef3625 100644 --- a/test/async_calls.test.js +++ b/test/async_calls.test.js @@ -6,12 +6,11 @@ const { createHook, executionAsyncId } = require("async_hooks"); describe('async_hooks', function() { - before(function() { - if (process.env.npm_config_arch.includes('wasm')) { - // async_hooks is not supported by emnapi - this.skip(); - } - }); + // before(function() { + // if (process.env.npm_config_arch.includes('wasm')) { + // this.skip(); + // } + // }); let db; let dbId; From a4a27ca60a34b227bd84ce62ba7dd93dc7d47aac Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Sun, 5 Feb 2023 01:01:56 +0800 Subject: [PATCH 4/8] Fix binding.gyp --- binding.gyp | 54 +++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/binding.gyp b/binding.gyp index 438f7722d..a73a13176 100644 --- a/binding.gyp +++ b/binding.gyp @@ -35,32 +35,34 @@ "deps/sqlite3.gyp:sqlite3" ] }, { - "conditions": ["sqlite != 'internal'", { - "include_dirs": [ - " Date: Tue, 7 Feb 2023 20:43:30 +0800 Subject: [PATCH 5/8] Fix CI print binary info --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc9c17f31..0442fe9b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,13 +111,18 @@ jobs: run: emmake yarn node-pre-gyp build --nodedir=./wasm --target_arch=${{ env.TARGET }} - name: Print binary info - if: contains(matrix.os, 'ubuntu') + if: contains(matrix.os, 'ubuntu') && matrix.target != 'wasm32' run: | ldd lib/binding/*/node_sqlite3.node echo "---" nm lib/binding/*/node_sqlite3.node | grep "GLIBC_" | c++filt || true echo "---" file lib/binding/napi-v*/* + + - name: Print binary info (WebAssembly) + if: contains(matrix.os, 'ubuntu') && matrix.target == 'wasm32' + run: | + file lib/binding/napi-v*/* - name: Run tests if: matrix.target != 'wasm32' From 6a3478b05df6973d8cd3541605c5b3b5aa81747f Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Thu, 9 Feb 2023 21:00:14 +0800 Subject: [PATCH 6/8] Preserve authorship of common.gypi --- wasm/common.gypi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wasm/common.gypi b/wasm/common.gypi index 4848e83f3..4d7f548bd 100644 --- a/wasm/common.gypi +++ b/wasm/common.gypi @@ -1,3 +1,5 @@ +# This file is originally created by [RReverser](https://github.com/RReverser) +# in https://github.com/lovell/sharp/pull/3522 { 'variables': { 'OS': 'emscripten', From 88fd747328c58d3e34667e26305ebb7c0d7777bc Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Tue, 14 Feb 2023 22:24:03 +0800 Subject: [PATCH 7/8] Upgrade emnapi --- lib/sqlite3-binding.js | 5 ++--- package.json | 6 +++--- wasm/common.gypi | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/sqlite3-binding.js b/lib/sqlite3-binding.js index 1f2135235..225182e98 100644 --- a/lib/sqlite3-binding.js +++ b/lib/sqlite3-binding.js @@ -5,14 +5,13 @@ const binding_path = binary.find(path.resolve(path.join(__dirname,'../package.js }); let binding = require(binding_path); if (typeof binding === 'function') { - const emnapiContext = require('@tybys/emnapi-runtime').createContext(); const emnapiInitOptions = { - context: emnapiContext + context: require('@emnapi/runtime').getDefaultContext() }; try { // optional dependency // support async_hooks on Node.js - emnapiInitOptions.nodeBinding = require('@tybys/emnapi-node-binding'); + emnapiInitOptions.nodeBinding = require('@emnapi/node-binding'); } catch (_) { // ignore } diff --git a/package.json b/package.json index 0d210d4e7..7236cb41e 100644 --- a/package.json +++ b/package.json @@ -49,12 +49,12 @@ }, "dependencies": { "@mapbox/node-pre-gyp": "^1.0.0", - "@tybys/emnapi-runtime": "^0.28.1", + "@emnapi/runtime": "^0.31.0", "node-addon-api": "^4.2.0", "tar": "^6.1.11" }, "devDependencies": { - "@tybys/emnapi": "^0.28.1", + "emnapi": "^0.31.0", "eslint": "6.8.0", "mocha": "7.2.0", "node-pre-gyp-github": "1.4.4" @@ -68,7 +68,7 @@ } }, "optionalDependencies": { - "@tybys/emnapi-node-binding": "^0.28.1", + "@emnapi/node-binding": "^0.31.0", "node-gyp": "8.x" }, "scripts": { diff --git a/wasm/common.gypi b/wasm/common.gypi index 4d7f548bd..4831e9db7 100644 --- a/wasm/common.gypi +++ b/wasm/common.gypi @@ -26,7 +26,7 @@ '-std=gnu++17' ], 'ldflags': [ - '--js-library= JSON.stringify(path.relative(process.cwd(), x))).join(\' \')")' + ' JSON.stringify(path.relative(process.cwd(), x))).join(\' \')")' ], 'default_configuration': 'Release', From aea5b1d270f734818be563a7618934a92d8f04a7 Mon Sep 17 00:00:00 2001 From: toyobayashi Date: Thu, 16 Mar 2023 00:02:25 +0800 Subject: [PATCH 8/8] CI test --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0442fe9b9..acb47cd22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,7 +130,9 @@ jobs: - name: Run tests (WebAssembly) if: matrix.target == 'wasm32' - run: npx --arch=wasm32 yarn test + env: + npm_config_target_arch: 'wasm32' + run: yarn test - name: Package prebuilt binaries run: yarn node-pre-gyp package --target_arch=${{ env.TARGET }}