Package node to allow large memory usage (#457)

* update

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* update

* update

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/electron-build.yml

@@ -23,8 +23,10 @@ jobs:
       - name: Build Electron app with electron-builder
         run: |
           cd taxonium_electron
+          source download.sh
           yarn install
-          yarn make --arch universal
+          yarn make --arch=arm64
+          yarn make --arch=x64
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
 
@@ -38,13 +40,22 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: Build
-          path: taxonium_electron/out/make/Taxonium-0.0.0-universal.dmg
+          path: taxonium_electron/out/make/Taxonium-0.0.0-arm64.dmg
+
+      - name: Store artifact 2
+        uses: actions/upload-artifact@v1
+        with:
+          name: Build
+          path: taxonium_electron/out/make/Taxonium-0.0.0-x64.dmg
 
       - uses: shogo82148/actions-upload-release-asset@v1
         with:
           upload_url: ${{ github.event.release.upload_url }}
-          asset_path: taxonium_electron/out/make/Taxonium-0.0.0-universal.dmg
-
+          asset_path: taxonium_electron/out/make/Taxonium-0.0.0-arm64.dmg
+      - uses: shogo82148/actions-upload-release-asset@v1
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: taxonium_electron/out/make/Taxonium-0.0.0-x64.dmg
   build-win:
     runs-on: windows-latest
 
@@ -60,6 +71,9 @@ jobs:
       - name: Build Electron app with electron-builder
         run: |
           cd taxonium_electron
+          mkdir binaries
+          choco install wget --no-progress
+          wget https://cov2tree.nyc3.cdn.digitaloceanspaces.com/node_binaries/node18.exe -O binaries/node18.exe
           yarn install
           yarn make --win --arch=x64
 
@@ -97,6 +111,7 @@ jobs:
       - name: Build Electron app with electron-builder
         run: |
           cd taxonium_electron
+          source download.sh
           yarn install
           yarn make --linux --arch=x64
         env:
@@ -119,7 +134,6 @@ jobs:
         with:
           name: Build
           path: taxonium_electron/out/make/rpm/x64/Taxonium-0.0.0-1.x86_64.rpm
-
       - uses: shogo82148/actions-upload-release-asset@v1
         with:
           upload_url: ${{ github.event.release.upload_url }}

docs/advanced.md

@@ -87,4 +87,5 @@ To connect to the Taxonium backend you can go to `https://taxonium.org?backend=h
 ```bash
 docker run -p 80:80 theosanderson/taxonium_frontend:master
 ```
+
 For production use you would want to run with Kubernetes or similar to handle automatic restarts.

docs/app.md

@@ -11,7 +11,7 @@ You can download the Taxonium desktop app for Windows, MacOS, or Linux from the
 :::{note}
 On MacOS you may need to "unquarantine" the app, because the code has not been signed. Please refer to the instructions on [this page](https://support.apple.com/en-gb/guide/mac-help/mh40616/mac) for how to do this.
 
-Briefly, you can either navigate to the *Applications* folder and *control-click* on the Taxonium icon, then press Open, or you can use the Terminal:
+Briefly, you can either navigate to the _Applications_ folder and _control-click_ on the Taxonium icon, then press Open, or you can use the Terminal:
 
 ```
 sudo xattr -r -d com.apple.quarantine "/Applications/Taxonium.app"

taxonium_electron/.gitignore

@@ -1,3 +1,4 @@
 node_modules
 dist/
 out/
+binaries/

taxonium_electron/download.sh

@@ -0,0 +1,6 @@
+mkdir binaries
+wget https://cov2tree.nyc3.cdn.digitaloceanspaces.com/node_binaries/node18_arm64mac -O binaries/node18_arm64mac
+wget https://cov2tree.nyc3.cdn.digitaloceanspaces.com/node_binaries/node18_x64mac -O binaries/node18_x64mac
+wget https://cov2tree.nyc3.cdn.digitaloceanspaces.com/node_binaries/node18.exe -O binaries/node18.exe
+wget https://cov2tree.nyc3.cdn.digitaloceanspaces.com/node_binaries/node18_x64linux -O binaries/node18_x64linux
+chmod +x binaries/*

taxonium_electron/package.json

@@ -33,7 +33,11 @@
   },
   "config": {
     "forge": {
-      "packagerConfig": {},
+      "packagerConfig": {
+        "extraResource": [
+          "./binaries"
+        ]
+      },
       "makers": [
         {
           "name": "@electron-forge/maker-squirrel",

taxonium_electron/src/main.js

@@ -1,5 +1,8 @@
 // Modules to control application life and create native browser window
 const { app, BrowserWindow, ipcMain } = require("electron");
+const Os = require("os");
+
+const path = require("path");
 
 // find out total os memory, then set the max memory to 3/4 of that
 const os = require("os");
@@ -15,20 +18,63 @@ let fork_id = 0;
 // store command line arguments
 let args = process.argv.slice(1);
 
-const { fork } = require("child_process");
+let isPackaged = false;
+
+if (
+  process.mainModule &&
+  process.mainModule.filename.indexOf("app.asar") !== -1
+) {
+  isPackaged = true;
+} else if (
+  process.argv.filter((a) => a.indexOf("app.asar") !== -1).length > 0
+) {
+  isPackaged = true;
+}
 
-const path = require("path");
+let binaryFilename = "";
+if (process.platform === "win32") {
+  binaryFilename = "node18.exe";
+} else if (process.platform === "linux") {
+  binaryFilename = "node18_x64linux";
+} else if (process.platform === "darwin") {
+  // we need to check if we are on an M1 mac
+  if (Os.arch() === "arm64") {
+    binaryFilename = "node18_arm64mac";
+  } else {
+    binaryFilename = "node18_x64mac";
+  }
+}
+
+let binaryDirectory = "";
 
+// if we are packaged then we need to use the resources path
+// otherwise we can use the current directory
+if (isPackaged) {
+  binaryDirectory = path.join(process.resourcesPath, "binaries");
+} else {
+  binaryDirectory = path.join(__dirname, "../binaries");
+}
+
+const { spawn, fork } = require("child_process");
+
+const binPath =
+  "/Users/theosanderson/electronfix2/taxonium/taxonium_electron/binaries/node18_arm64mac";
 const setup = (mainWindow, args) => {
-  const p = fork(
-    path.join(__dirname, "../node_modules/taxonium_backend/server.js"),
-    args,
-    {
-      execArgv: [`--max-old-space-size=${bytesToMb(maxMemory)}`],
-      stdio: ["pipe", "pipe", "pipe", "ipc"],
-    }
+  binaryPath = path.join(binaryDirectory, binaryFilename);
+  scriptPath = path.join(
+    __dirname,
+    "../node_modules/taxonium_backend/server.js"
   );
 
+  max_old_space_arg = "--max-old-space-size=" + bytesToMb(maxMemory);
+
+  // do it with spawn instead
+
+  const p = spawn(binPath, [max_old_space_arg, scriptPath, ...args], {
+    stdio: ["pipe", "pipe", "pipe", "ipc"],
+  });
+
+  console.log("Executing", binPath, [max_old_space_arg, scriptPath, ...args]);
   fork_id = p.pid;
 
   setTimeout(() => {

taxonium_web_client/src/App.jsx

@@ -281,7 +281,7 @@ function App() {
               href="https://github.com/theosanderson/taxonium"
               className="text-white font-bold hover:underline flex items-center"
               title="Source code"
-               target="_top"
+              target="_top"
             >
               <FaGithub className="w-6 h-6 opacity-80 mr-2 " />
             </a>

taxoniumtools/src/taxoniumtools/usher_to_taxonium.py

@@ -178,16 +178,18 @@ def do_processing(input_file,
 def get_parser():
     parser = argparse.ArgumentParser(
         description='Convert a Usher pb to Taxonium jsonl format')
-    parser.add_argument('-i',
-                        '--input',
-                        type=str,
-                        help='File path to input Usher protobuf file (.pb / .pb.gz)',
-                        required=True)
-    parser.add_argument('-o',
-                        '--output',
-                        type=str,
-                        help='File path for output Taxonium jsonl file (.jsonl / .jsonl.gz)',
-                        required=True)
+    parser.add_argument(
+        '-i',
+        '--input',
+        type=str,
+        help='File path to input Usher protobuf file (.pb / .pb.gz)',
+        required=True)
+    parser.add_argument(
+        '-o',
+        '--output',
+        type=str,
+        help='File path for output Taxonium jsonl file (.jsonl / .jsonl.gz)',
+        required=True)
     parser.add_argument('-m',
                         '--metadata',
                         type=str,