From 2cd7019ba590ad33eb23ce124a4fb81cc384b396 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Tue, 15 Oct 2024 04:33:41 +0000
Subject: [PATCH 01/35] compute server: add a trivial "hello world" http
 interface to compute server in dev mode

---
 src/compute/compute/dev/start-compute.js |   4 +
 src/compute/compute/lib/http-server.ts   |  35 ++
 src/compute/compute/lib/manager.ts       |  25 +-
 src/compute/compute/package.json         |   1 +
 src/compute/pnpm-lock.yaml               | 523 +++++++++++++++++++++++
 src/compute/static                       |   1 +
 src/package.json                         |   2 +-
 7 files changed, 589 insertions(+), 2 deletions(-)
 create mode 100644 src/compute/compute/lib/http-server.ts
 create mode 120000 src/compute/static

diff --git a/src/compute/compute/dev/start-compute.js b/src/compute/compute/dev/start-compute.js
index b58f1c051a..d006d01463 100644
--- a/src/compute/compute/dev/start-compute.js
+++ b/src/compute/compute/dev/start-compute.js
@@ -6,6 +6,8 @@ console.log("API_SERVER=", process.env.API_SERVER);
 const { manager } = require("../dist/lib");
 
 const PROJECT_HOME = process.env.PROJECT_HOME ?? "/tmp/home";
+const PORT = process.env.PORT ?? 5004;
+const HOST = process.env.HOST ?? "0.0.0.0";
 
 async function main() {
   const exitHandler = async () => {
@@ -28,6 +30,8 @@ async function main() {
       process.env.UNIONFS_UPPER && process.env.UNIONFS_LOWER
         ? "fuse.unionfs-fuse"
         : "fuse",
+    host: HOST,
+    port: PORT,
   });
   exports.manager = M;
   await M.init();
diff --git a/src/compute/compute/lib/http-server.ts b/src/compute/compute/lib/http-server.ts
new file mode 100644
index 0000000000..8ede516018
--- /dev/null
+++ b/src/compute/compute/lib/http-server.ts
@@ -0,0 +1,35 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2024 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+import express from "express";
+import { createServer } from "http";
+import { getLogger } from "@cocalc/backend/logger";
+import type { Manager } from "./manager";
+
+const logger = getLogger("compute:http-server");
+
+export function initHttpServer({
+  port = 5004,
+  host = "localhost",
+  manager,
+}: {
+  port?: number;
+  host?: string;
+  manager: Manager;
+}) {
+  logger.info("starting http-server...");
+
+  const app = express();
+  const server = createServer(app);
+
+  app.get("/", (_req, res) => {
+    const files = manager.getOpenFiles();
+    res.send(`<h1>Compute Server</h1>  Open Files: ${files.join(", ")}`);
+  });
+
+  server.listen(port, host, () => {
+    logger.info(`Server listening http://${host}:${port}`);
+  });
+}
diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts
index c8f913054d..6fff12ae63 100644
--- a/src/compute/compute/lib/manager.ts
+++ b/src/compute/compute/lib/manager.ts
@@ -28,6 +28,7 @@ import { apiCall } from "@cocalc/api-client";
 import { get_blob_store as initJupyterBlobStore } from "@cocalc/jupyter/blobs";
 import { delay } from "awaiting";
 import { executeCode } from "@cocalc/backend/execute-code";
+import { initHttpServer } from "./http-server";
 
 const logger = debug("cocalc:compute:manager");
 
@@ -47,6 +48,9 @@ interface Options {
   // If true, doesn't do anything until the type of the file system that home is
   // mounted on is of this type, e.g., "fuse".
   waitHomeFilesystemType?: string;
+
+  host?: string;
+  port?: number;
 }
 
 process.on("exit", () => {
@@ -75,11 +79,13 @@ export function manager(opts: Options) {
   return new Manager(opts);
 }
 
-class Manager {
+export class Manager {
   private state: "new" | "init" | "ready" = "new";
   private sync_db;
   private project_id: string;
   private home: string;
+  private host?: string;
+  private port?: number;
   private waitHomeFilesystemType?: string;
   private compute_server_id: number;
   private connections: { [path: string]: any } = {};
@@ -91,6 +97,8 @@ class Manager {
     compute_server_id = parseInt(process.env.COMPUTE_SERVER_ID ?? "0"),
     home = process.env.HOME ?? "/home/user",
     waitHomeFilesystemType,
+    host,
+    port,
   }: Options) {
     if (!project_id) {
       throw Error("project_id or process.env.PROJECT_ID must be given");
@@ -102,6 +110,8 @@ class Manager {
     // @ts-ignore -- can't true type, since constructed via plain javascript startup script.
     this.compute_server_id = parseInt(compute_server_id);
     this.home = home;
+    this.host = host;
+    this.port = port;
     this.waitHomeFilesystemType = waitHomeFilesystemType;
     const env = this.env();
     for (const key in env) {
@@ -115,6 +125,9 @@ class Manager {
     }
     this.log("initialize the Manager");
     this.state = "init";
+
+    await this.initHttpServer();
+
     // Ping to start the project and ensure there is a hub connection to it.
     await pingProjectUntilSuccess(this.project_id);
     // wait for home direcotry file system to be mounted:
@@ -421,4 +434,14 @@ class Manager {
         throw Error(`unknown event '${data?.event}'`);
     }
   };
+
+  private initHttpServer = () => {
+    if (this.host != null && this.port != null) {
+      initHttpServer({ port: this.port, host: this.host, manager: this });
+    }
+  };
+
+  getOpenFiles = (): string[] => {
+    return Object.keys(this.connections);
+  };
 }
diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index ebd6e823ab..6c8366a196 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -41,6 +41,7 @@
     "awaiting": "^3.0.0",
     "cookie": "^1.0.0",
     "debug": "^4.3.2",
+    "express": "^4.20.0",
     "websocketfs": "^0.17.4",
     "ws": "^8.18.0"
   },
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index 2eea7010d4..029b410b4f 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -47,6 +47,9 @@ importers:
       debug:
         specifier: ^4.3.2
         version: 4.3.4
+      express:
+        specifier: ^4.20.0
+        version: 4.21.1
       websocketfs:
         specifier: ^0.17.4
         version: 0.17.4
@@ -86,6 +89,13 @@ packages:
   '@wwa/statvfs@1.1.18':
     resolution: {integrity: sha512-C33QeTo2Nma9gMAJy3l1AQc0Qz5Lbf7mCY2C3F1W3noCdukODWH8nB8sjavdwjw9S7Qa+zrvQAfbbYCOzIphAw==}
 
+  accepts@1.3.8:
+    resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==}
+    engines: {node: '>= 0.6'}
+
+  array-flatten@1.1.1:
+    resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==}
+
   awaiting@3.0.0:
     resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==}
     engines: {node: '>=7.6.x'}
@@ -102,16 +112,43 @@ packages:
   bl@4.1.0:
     resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
 
+  body-parser@1.20.3:
+    resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==}
+    engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
+
   buffer@5.7.1:
     resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
 
+  bytes@3.1.2:
+    resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}
+    engines: {node: '>= 0.8'}
+
+  call-bind@1.0.7:
+    resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==}
+    engines: {node: '>= 0.4'}
+
   chownr@1.1.4:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
 
+  content-disposition@0.5.4:
+    resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
+    engines: {node: '>= 0.6'}
+
+  content-type@1.0.5:
+    resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==}
+    engines: {node: '>= 0.6'}
+
+  cookie-signature@1.0.6:
+    resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==}
+
   cookie@0.5.0:
     resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==}
     engines: {node: '>= 0.6'}
 
+  cookie@0.7.1:
+    resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==}
+    engines: {node: '>= 0.6'}
+
   cookie@1.0.0:
     resolution: {integrity: sha512-bsSztFoaR8bw9MlFCrTHzc1wOKCUKOBsbgFdoDilZDkETAOOjKSqV7L+EQLbTaylwvZasd9vM4MGKotJaUfSpA==}
     engines: {node: '>=18'}
@@ -119,6 +156,14 @@ packages:
   cuint@0.2.2:
     resolution: {integrity: sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==}
 
+  debug@2.6.9:
+    resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+
   debug@4.3.4:
     resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
     engines: {node: '>=6.0'}
@@ -136,26 +181,113 @@ packages:
     resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
     engines: {node: '>=4.0.0'}
 
+  define-data-property@1.1.4:
+    resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==}
+    engines: {node: '>= 0.4'}
+
+  depd@2.0.0:
+    resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
+    engines: {node: '>= 0.8'}
+
+  destroy@1.2.0:
+    resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==}
+    engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
+
   detect-libc@2.0.3:
     resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==}
     engines: {node: '>=8'}
 
+  ee-first@1.1.1:
+    resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
+
+  encodeurl@1.0.2:
+    resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
+    engines: {node: '>= 0.8'}
+
+  encodeurl@2.0.0:
+    resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==}
+    engines: {node: '>= 0.8'}
+
   end-of-stream@1.4.4:
     resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
 
+  es-define-property@1.0.0:
+    resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==}
+    engines: {node: '>= 0.4'}
+
+  es-errors@1.3.0:
+    resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}
+    engines: {node: '>= 0.4'}
+
+  escape-html@1.0.3:
+    resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}
+
+  etag@1.8.1:
+    resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
+    engines: {node: '>= 0.6'}
+
   expand-template@2.0.3:
     resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}
     engines: {node: '>=6'}
 
+  express@4.21.1:
+    resolution: {integrity: sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==}
+    engines: {node: '>= 0.10.0'}
+
   file-uri-to-path@1.0.0:
     resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
 
+  finalhandler@1.3.1:
+    resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==}
+    engines: {node: '>= 0.8'}
+
+  forwarded@0.2.0:
+    resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
+    engines: {node: '>= 0.6'}
+
+  fresh@0.5.2:
+    resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}
+    engines: {node: '>= 0.6'}
+
   fs-constants@1.0.0:
     resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
 
+  function-bind@1.1.2:
+    resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
+
+  get-intrinsic@1.2.4:
+    resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
+    engines: {node: '>= 0.4'}
+
   github-from-package@0.0.0:
     resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}
 
+  gopd@1.0.1:
+    resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
+
+  has-property-descriptors@1.0.2:
+    resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==}
+
+  has-proto@1.0.3:
+    resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==}
+    engines: {node: '>= 0.4'}
+
+  has-symbols@1.0.3:
+    resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==}
+    engines: {node: '>= 0.4'}
+
+  hasown@2.0.2:
+    resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
+    engines: {node: '>= 0.4'}
+
+  http-errors@2.0.0:
+    resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
+    engines: {node: '>= 0.8'}
+
+  iconv-lite@0.4.24:
+    resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}
+    engines: {node: '>=0.10.0'}
+
   ieee754@1.2.1:
     resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
 
@@ -165,10 +297,38 @@ packages:
   ini@1.3.8:
     resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
 
+  ipaddr.js@1.9.1:
+    resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
+    engines: {node: '>= 0.10'}
+
   lz4@0.6.5:
     resolution: {integrity: sha512-KSZcJU49QZOlJSItaeIU3p8WoAvkTmD9fJqeahQXNu1iQ/kR0/mQLdbrK8JY9MY8f6AhJoMrihp1nu1xDbscSQ==}
     engines: {node: '>= 0.10'}
 
+  media-typer@0.3.0:
+    resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
+    engines: {node: '>= 0.6'}
+
+  merge-descriptors@1.0.3:
+    resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==}
+
+  methods@1.1.2:
+    resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
+    engines: {node: '>= 0.6'}
+
+  mime-db@1.52.0:
+    resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
+    engines: {node: '>= 0.6'}
+
+  mime-types@2.1.35:
+    resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
+    engines: {node: '>= 0.6'}
+
+  mime@1.6.0:
+    resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==}
+    engines: {node: '>=4'}
+    hasBin: true
+
   mimic-response@3.1.0:
     resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
     engines: {node: '>=10'}
@@ -179,9 +339,15 @@ packages:
   mkdirp-classic@0.5.3:
     resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
 
+  ms@2.0.0:
+    resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}
+
   ms@2.1.2:
     resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
 
+  ms@2.1.3:
+    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+
   nan@2.20.0:
     resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==}
 
@@ -194,6 +360,10 @@ packages:
   napi-macros@2.2.2:
     resolution: {integrity: sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==}
 
+  negotiator@0.6.3:
+    resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==}
+    engines: {node: '>= 0.6'}
+
   node-abi@3.68.0:
     resolution: {integrity: sha512-7vbj10trelExNjFSBm5kTvZXXa7pZyKWx9RCKIyqe6I9Ev3IzGpQoqBP3a+cOdxY+pWj6VkP28n/2wWysBHD/A==}
     engines: {node: '>=10'}
@@ -206,9 +376,24 @@ packages:
     resolution: {integrity: sha512-IRUxE4BVsHWXkV/SFOut4qTlagw2aM8T5/vnTsmrHJvVoKueJHRc/JaFND7QDDc61kLYUJ6qlZM3sqTSyx2dTw==}
     hasBin: true
 
+  object-inspect@1.13.2:
+    resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==}
+    engines: {node: '>= 0.4'}
+
+  on-finished@2.4.1:
+    resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}
+    engines: {node: '>= 0.8'}
+
   once@1.4.0:
     resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
 
+  parseurl@1.3.3:
+    resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==}
+    engines: {node: '>= 0.8'}
+
+  path-to-regexp@0.1.10:
+    resolution: {integrity: sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==}
+
   port-get@1.0.4:
     resolution: {integrity: sha512-B8RcNfc8Ld+7C31DPaKIQz2aO9dqIs+4sUjhxJ2TSjEaidwyxu05WBbm08FJe+qkVvLiQqPbEAfNw1rB7JbjtA==}
 
@@ -217,9 +402,25 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
 
+  proxy-addr@2.0.7:
+    resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
+    engines: {node: '>= 0.10'}
+
   pump@3.0.2:
     resolution: {integrity: sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==}
 
+  qs@6.13.0:
+    resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==}
+    engines: {node: '>=0.6'}
+
+  range-parser@1.2.1:
+    resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
+    engines: {node: '>= 0.6'}
+
+  raw-body@2.5.2:
+    resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==}
+    engines: {node: '>= 0.8'}
+
   rc@1.2.8:
     resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==}
     hasBin: true
@@ -231,17 +432,43 @@ packages:
   safe-buffer@5.2.1:
     resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
 
+  safer-buffer@2.1.2:
+    resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==}
+
   semver@7.6.3:
     resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==}
     engines: {node: '>=10'}
     hasBin: true
 
+  send@0.19.0:
+    resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==}
+    engines: {node: '>= 0.8.0'}
+
+  serve-static@1.16.2:
+    resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==}
+    engines: {node: '>= 0.8.0'}
+
+  set-function-length@1.2.2:
+    resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==}
+    engines: {node: '>= 0.4'}
+
+  setprototypeof@1.2.0:
+    resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
+
+  side-channel@1.0.6:
+    resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==}
+    engines: {node: '>= 0.4'}
+
   simple-concat@1.0.1:
     resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
 
   simple-get@4.0.1:
     resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
 
+  statuses@2.0.1:
+    resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
+    engines: {node: '>= 0.8'}
+
   string_decoder@1.3.0:
     resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
 
@@ -256,17 +483,37 @@ packages:
     resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
     engines: {node: '>=6'}
 
+  toidentifier@1.0.1:
+    resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
+    engines: {node: '>=0.6'}
+
   tunnel-agent@0.6.0:
     resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
 
+  type-is@1.6.18:
+    resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==}
+    engines: {node: '>= 0.6'}
+
   typescript@5.6.3:
     resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==}
     engines: {node: '>=14.17'}
     hasBin: true
 
+  unpipe@1.0.0:
+    resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==}
+    engines: {node: '>= 0.8'}
+
   util-deprecate@1.0.2:
     resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
 
+  utils-merge@1.0.1:
+    resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
+    engines: {node: '>= 0.4.0'}
+
+  vary@1.1.2:
+    resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}
+    engines: {node: '>= 0.8'}
+
   websocket-sftp@0.8.2:
     resolution: {integrity: sha512-LHUzwTSQNB+Rud+OTe+lt9ojdWzE5nHQ2rnBs1JwQIXnh3bc0NVXk9pY7TMmFLkLYoBUKypa7Kl2CPfuxKLE4g==}
     engines: {node: '>=0.16.0'}
@@ -318,6 +565,13 @@ snapshots:
       node-addon-api: 8.1.0
       prebuild-install: 7.1.2
 
+  accepts@1.3.8:
+    dependencies:
+      mime-types: 2.1.35
+      negotiator: 0.6.3
+
+  array-flatten@1.1.1: {}
+
   awaiting@3.0.0: {}
 
   base64-js@1.5.1: {}
@@ -334,19 +588,60 @@ snapshots:
       inherits: 2.0.4
       readable-stream: 3.6.2
 
+  body-parser@1.20.3:
+    dependencies:
+      bytes: 3.1.2
+      content-type: 1.0.5
+      debug: 2.6.9
+      depd: 2.0.0
+      destroy: 1.2.0
+      http-errors: 2.0.0
+      iconv-lite: 0.4.24
+      on-finished: 2.4.1
+      qs: 6.13.0
+      raw-body: 2.5.2
+      type-is: 1.6.18
+      unpipe: 1.0.0
+    transitivePeerDependencies:
+      - supports-color
+
   buffer@5.7.1:
     dependencies:
       base64-js: 1.5.1
       ieee754: 1.2.1
 
+  bytes@3.1.2: {}
+
+  call-bind@1.0.7:
+    dependencies:
+      es-define-property: 1.0.0
+      es-errors: 1.3.0
+      function-bind: 1.1.2
+      get-intrinsic: 1.2.4
+      set-function-length: 1.2.2
+
   chownr@1.1.4: {}
 
+  content-disposition@0.5.4:
+    dependencies:
+      safe-buffer: 5.2.1
+
+  content-type@1.0.5: {}
+
+  cookie-signature@1.0.6: {}
+
   cookie@0.5.0: {}
 
+  cookie@0.7.1: {}
+
   cookie@1.0.0: {}
 
   cuint@0.2.2: {}
 
+  debug@2.6.9:
+    dependencies:
+      ms: 2.0.0
+
   debug@4.3.4:
     dependencies:
       ms: 2.1.2
@@ -357,26 +652,144 @@ snapshots:
 
   deep-extend@0.6.0: {}
 
+  define-data-property@1.1.4:
+    dependencies:
+      es-define-property: 1.0.0
+      es-errors: 1.3.0
+      gopd: 1.0.1
+
+  depd@2.0.0: {}
+
+  destroy@1.2.0: {}
+
   detect-libc@2.0.3: {}
 
+  ee-first@1.1.1: {}
+
+  encodeurl@1.0.2: {}
+
+  encodeurl@2.0.0: {}
+
   end-of-stream@1.4.4:
     dependencies:
       once: 1.4.0
 
+  es-define-property@1.0.0:
+    dependencies:
+      get-intrinsic: 1.2.4
+
+  es-errors@1.3.0: {}
+
+  escape-html@1.0.3: {}
+
+  etag@1.8.1: {}
+
   expand-template@2.0.3: {}
 
+  express@4.21.1:
+    dependencies:
+      accepts: 1.3.8
+      array-flatten: 1.1.1
+      body-parser: 1.20.3
+      content-disposition: 0.5.4
+      content-type: 1.0.5
+      cookie: 0.7.1
+      cookie-signature: 1.0.6
+      debug: 2.6.9
+      depd: 2.0.0
+      encodeurl: 2.0.0
+      escape-html: 1.0.3
+      etag: 1.8.1
+      finalhandler: 1.3.1
+      fresh: 0.5.2
+      http-errors: 2.0.0
+      merge-descriptors: 1.0.3
+      methods: 1.1.2
+      on-finished: 2.4.1
+      parseurl: 1.3.3
+      path-to-regexp: 0.1.10
+      proxy-addr: 2.0.7
+      qs: 6.13.0
+      range-parser: 1.2.1
+      safe-buffer: 5.2.1
+      send: 0.19.0
+      serve-static: 1.16.2
+      setprototypeof: 1.2.0
+      statuses: 2.0.1
+      type-is: 1.6.18
+      utils-merge: 1.0.1
+      vary: 1.1.2
+    transitivePeerDependencies:
+      - supports-color
+
   file-uri-to-path@1.0.0: {}
 
+  finalhandler@1.3.1:
+    dependencies:
+      debug: 2.6.9
+      encodeurl: 2.0.0
+      escape-html: 1.0.3
+      on-finished: 2.4.1
+      parseurl: 1.3.3
+      statuses: 2.0.1
+      unpipe: 1.0.0
+    transitivePeerDependencies:
+      - supports-color
+
+  forwarded@0.2.0: {}
+
+  fresh@0.5.2: {}
+
   fs-constants@1.0.0: {}
 
+  function-bind@1.1.2: {}
+
+  get-intrinsic@1.2.4:
+    dependencies:
+      es-errors: 1.3.0
+      function-bind: 1.1.2
+      has-proto: 1.0.3
+      has-symbols: 1.0.3
+      hasown: 2.0.2
+
   github-from-package@0.0.0: {}
 
+  gopd@1.0.1:
+    dependencies:
+      get-intrinsic: 1.2.4
+
+  has-property-descriptors@1.0.2:
+    dependencies:
+      es-define-property: 1.0.0
+
+  has-proto@1.0.3: {}
+
+  has-symbols@1.0.3: {}
+
+  hasown@2.0.2:
+    dependencies:
+      function-bind: 1.1.2
+
+  http-errors@2.0.0:
+    dependencies:
+      depd: 2.0.0
+      inherits: 2.0.4
+      setprototypeof: 1.2.0
+      statuses: 2.0.1
+      toidentifier: 1.0.1
+
+  iconv-lite@0.4.24:
+    dependencies:
+      safer-buffer: 2.1.2
+
   ieee754@1.2.1: {}
 
   inherits@2.0.4: {}
 
   ini@1.3.8: {}
 
+  ipaddr.js@1.9.1: {}
+
   lz4@0.6.5:
     dependencies:
       buffer: 5.7.1
@@ -384,14 +797,32 @@ snapshots:
       nan: 2.20.0
       xxhashjs: 0.2.2
 
+  media-typer@0.3.0: {}
+
+  merge-descriptors@1.0.3: {}
+
+  methods@1.1.2: {}
+
+  mime-db@1.52.0: {}
+
+  mime-types@2.1.35:
+    dependencies:
+      mime-db: 1.52.0
+
+  mime@1.6.0: {}
+
   mimic-response@3.1.0: {}
 
   minimist@1.2.8: {}
 
   mkdirp-classic@0.5.3: {}
 
+  ms@2.0.0: {}
+
   ms@2.1.2: {}
 
+  ms@2.1.3: {}
+
   nan@2.20.0: {}
 
   nanoresource@1.3.0:
@@ -402,6 +833,8 @@ snapshots:
 
   napi-macros@2.2.2: {}
 
+  negotiator@0.6.3: {}
+
   node-abi@3.68.0:
     dependencies:
       semver: 7.6.3
@@ -410,10 +843,20 @@ snapshots:
 
   node-gyp-build@4.8.2: {}
 
+  object-inspect@1.13.2: {}
+
+  on-finished@2.4.1:
+    dependencies:
+      ee-first: 1.1.1
+
   once@1.4.0:
     dependencies:
       wrappy: 1.0.2
 
+  parseurl@1.3.3: {}
+
+  path-to-regexp@0.1.10: {}
+
   port-get@1.0.4: {}
 
   prebuild-install@7.1.2:
@@ -431,11 +874,29 @@ snapshots:
       tar-fs: 2.1.1
       tunnel-agent: 0.6.0
 
+  proxy-addr@2.0.7:
+    dependencies:
+      forwarded: 0.2.0
+      ipaddr.js: 1.9.1
+
   pump@3.0.2:
     dependencies:
       end-of-stream: 1.4.4
       once: 1.4.0
 
+  qs@6.13.0:
+    dependencies:
+      side-channel: 1.0.6
+
+  range-parser@1.2.1: {}
+
+  raw-body@2.5.2:
+    dependencies:
+      bytes: 3.1.2
+      http-errors: 2.0.0
+      iconv-lite: 0.4.24
+      unpipe: 1.0.0
+
   rc@1.2.8:
     dependencies:
       deep-extend: 0.6.0
@@ -451,8 +912,55 @@ snapshots:
 
   safe-buffer@5.2.1: {}
 
+  safer-buffer@2.1.2: {}
+
   semver@7.6.3: {}
 
+  send@0.19.0:
+    dependencies:
+      debug: 2.6.9
+      depd: 2.0.0
+      destroy: 1.2.0
+      encodeurl: 1.0.2
+      escape-html: 1.0.3
+      etag: 1.8.1
+      fresh: 0.5.2
+      http-errors: 2.0.0
+      mime: 1.6.0
+      ms: 2.1.3
+      on-finished: 2.4.1
+      range-parser: 1.2.1
+      statuses: 2.0.1
+    transitivePeerDependencies:
+      - supports-color
+
+  serve-static@1.16.2:
+    dependencies:
+      encodeurl: 2.0.0
+      escape-html: 1.0.3
+      parseurl: 1.3.3
+      send: 0.19.0
+    transitivePeerDependencies:
+      - supports-color
+
+  set-function-length@1.2.2:
+    dependencies:
+      define-data-property: 1.1.4
+      es-errors: 1.3.0
+      function-bind: 1.1.2
+      get-intrinsic: 1.2.4
+      gopd: 1.0.1
+      has-property-descriptors: 1.0.2
+
+  setprototypeof@1.2.0: {}
+
+  side-channel@1.0.6:
+    dependencies:
+      call-bind: 1.0.7
+      es-errors: 1.3.0
+      get-intrinsic: 1.2.4
+      object-inspect: 1.13.2
+
   simple-concat@1.0.1: {}
 
   simple-get@4.0.1:
@@ -461,6 +969,8 @@ snapshots:
       once: 1.4.0
       simple-concat: 1.0.1
 
+  statuses@2.0.1: {}
+
   string_decoder@1.3.0:
     dependencies:
       safe-buffer: 5.2.1
@@ -482,14 +992,27 @@ snapshots:
       inherits: 2.0.4
       readable-stream: 3.6.2
 
+  toidentifier@1.0.1: {}
+
   tunnel-agent@0.6.0:
     dependencies:
       safe-buffer: 5.2.1
 
+  type-is@1.6.18:
+    dependencies:
+      media-typer: 0.3.0
+      mime-types: 2.1.35
+
   typescript@5.6.3: {}
 
+  unpipe@1.0.0: {}
+
   util-deprecate@1.0.2: {}
 
+  utils-merge@1.0.1: {}
+
+  vary@1.1.2: {}
+
   websocket-sftp@0.8.2:
     dependencies:
       '@wwa/statvfs': 1.1.18
diff --git a/src/compute/static b/src/compute/static
new file mode 120000
index 0000000000..d4c106f6c1
--- /dev/null
+++ b/src/compute/static
@@ -0,0 +1 @@
+../packages/static/
\ No newline at end of file
diff --git a/src/package.json b/src/package.json
index 1cd544feec..944058d68f 100644
--- a/src/package.json
+++ b/src/package.json
@@ -14,7 +14,7 @@
     "database": "cd dev/project && ./start_postgres.py",
     "database-remove-locks": "./scripts/database-remove-locks",
     "c": "LOGS=/tmp/ DEBUG='cocalc:*' ./scripts/c",
-    "version-check": "pip3 install typing_extensions mypy  || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
+    "version-check": "pip3 install typing_extensions mypy 2>/dev/null || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
     "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test",
     "test": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r test",
     "prettier-all": "cd packages/"

From 3a334d9059b659f6c7ec19092cfbeb04f80ac58d Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Tue, 15 Oct 2024 13:10:33 +0000
Subject: [PATCH 02/35] compute servrer: serve actual cocalc frontend app code.

- also found a cache header bug in hub express server (only impacts
  cocalc-docker so not a real issue).
---
 src/compute/compute/lib/http-server.ts  | 20 +++++++++++++-
 src/compute/compute/package.json        |  2 ++
 src/compute/pnpm-lock.yaml              | 31 ++++++++++++++++++++++
 src/compute/pnpm-workspace.yaml         |  1 +
 src/packages/hub/servers/express-app.ts | 32 ++--------------------
 src/packages/pnpm-lock.yaml             | 35 +++++++++++--------------
 src/packages/util/http-caching.ts       | 33 +++++++++++++++++++++++
 src/packages/util/package.json          | 19 +++++---------
 8 files changed, 109 insertions(+), 64 deletions(-)
 create mode 100644 src/packages/util/http-caching.ts

diff --git a/src/compute/compute/lib/http-server.ts b/src/compute/compute/lib/http-server.ts
index 8ede516018..ccc7fcbbe5 100644
--- a/src/compute/compute/lib/http-server.ts
+++ b/src/compute/compute/lib/http-server.ts
@@ -7,9 +7,14 @@ import express from "express";
 import { createServer } from "http";
 import { getLogger } from "@cocalc/backend/logger";
 import type { Manager } from "./manager";
+import { path as STATIC_PATH } from "@cocalc/static";
+import { join } from "path";
+import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
 
 const logger = getLogger("compute:http-server");
 
+const ENTRY_POINT = "app.html";
+
 export function initHttpServer({
   port = 5004,
   host = "localhost",
@@ -26,9 +31,22 @@ export function initHttpServer({
 
   app.get("/", (_req, res) => {
     const files = manager.getOpenFiles();
-    res.send(`<h1>Compute Server</h1>  Open Files: ${files.join(", ")}`);
+    res.send(
+      `<h1>Compute Server</h1>  <a href="${join("/static", ENTRY_POINT)}">CoCalc App</a> <br/><br/> Open Files: ${files.join(", ")}`,
+    );
   });
 
+  app.use(
+    join("/static", ENTRY_POINT),
+    express.static(join(STATIC_PATH, ENTRY_POINT), {
+      setHeaders: cacheShortTerm,
+    }),
+  );
+  app.use(
+    "/static",
+    express.static(STATIC_PATH, { setHeaders: cacheLongTerm }),
+  );
+
   server.listen(port, host, () => {
     logger.info(`Server listening http://${host}:${port}`);
   });
diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index 6c8366a196..f635dc4d3e 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -32,11 +32,13 @@
     "@cocalc/backend": "workspace:*",
     "@cocalc/compute": "link:",
     "@cocalc/jupyter": "workspace:*",
+    "@cocalc/static": "workspace:*",
     "@cocalc/sync": "workspace:*",
     "@cocalc/sync-client": "workspace:*",
     "@cocalc/sync-fs": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
+    "@types/ms": "^0.7.34",
     "@types/ws": "^8.5.9",
     "awaiting": "^3.0.0",
     "cookie": "^1.0.0",
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index 029b410b4f..039797180b 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -6,6 +6,12 @@ settings:
 
 importers:
 
+  api-client: {}
+
+  backend: {}
+
+  comm: {}
+
   compute:
     dependencies:
       '@cocalc/api-client':
@@ -20,6 +26,9 @@ importers:
       '@cocalc/jupyter':
         specifier: workspace:*
         version: link:../jupyter
+      '@cocalc/static':
+        specifier: workspace:*
+        version: link:../static
       '@cocalc/sync':
         specifier: workspace:*
         version: link:../sync
@@ -35,6 +44,9 @@ importers:
       '@cocalc/util':
         specifier: workspace:*
         version: link:../util
+      '@types/ms':
+        specifier: ^0.7.34
+        version: 0.7.34
       '@types/ws':
         specifier: ^8.5.9
         version: 8.5.9
@@ -67,6 +79,20 @@ importers:
         specifier: ^5.6.3
         version: 5.6.3
 
+  jupyter: {}
+
+  static: {}
+
+  sync: {}
+
+  sync-client: {}
+
+  sync-fs: {}
+
+  terminal: {}
+
+  util: {}
+
 packages:
 
   '@cocalc/fuse-native@2.4.1':
@@ -80,6 +106,9 @@ packages:
   '@types/cookie@0.6.0':
     resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==}
 
+  '@types/ms@0.7.34':
+    resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==}
+
   '@types/node@18.16.14':
     resolution: {integrity: sha512-+ImzUB3mw2c5ISJUq0punjDilUQ5GnUim0ZRvchHIWJmOC0G+p0kzhXBqj6cDjK0QdPFwzrHWgrJp3RPvCG5qg==}
 
@@ -553,6 +582,8 @@ snapshots:
 
   '@types/cookie@0.6.0': {}
 
+  '@types/ms@0.7.34': {}
+
   '@types/node@18.16.14': {}
 
   '@types/ws@8.5.9':
diff --git a/src/compute/pnpm-workspace.yaml b/src/compute/pnpm-workspace.yaml
index 9f84f9cb8f..45a7342538 100644
--- a/src/compute/pnpm-workspace.yaml
+++ b/src/compute/pnpm-workspace.yaml
@@ -9,3 +9,4 @@ packages:
   - "util"
   - "terminal"
   - "compute"
+  - "static"
diff --git a/src/packages/hub/servers/express-app.ts b/src/packages/hub/servers/express-app.ts
index fcbbb4aded..690954737a 100644
--- a/src/packages/hub/servers/express-app.ts
+++ b/src/packages/hub/servers/express-app.ts
@@ -5,7 +5,6 @@ The main hub express app.
 import compression from "compression";
 import cookieParser from "cookie-parser";
 import express from "express";
-import ms from "ms";
 import { join } from "path";
 import { parse as parseURL } from "url";
 import webpackDevMiddleware from "webpack-dev-middleware";
@@ -32,10 +31,7 @@ import initStripeWebhook from "./app/webhooks/stripe";
 import { database } from "./database";
 import initHttpServer from "./http";
 import initRobots from "./robots";
-
-// Used for longterm caching of files. This should be in units of seconds.
-const MAX_AGE = Math.round(ms("10 days") / 1000);
-const SHORT_AGE = Math.round(ms("10 seconds") / 1000);
+import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
 
 interface Options {
   projectControl;
@@ -166,30 +162,6 @@ export default async function init(opts: Options): Promise<{
   return { httpServer, router };
 }
 
-function cacheShortTerm(res) {
-  res.setHeader(
-    "Cache-Control",
-    `public, max-age=${SHORT_AGE}, must-revalidate`,
-  );
-  res.setHeader(
-    "Expires",
-    new Date(Date.now().valueOf() + SHORT_AGE).toUTCString(),
-  );
-}
-
-// Various files such as the webpack static content should be cached long-term,
-// and we use this function to set appropriate headers at various points below.
-function cacheLongTerm(res) {
-  res.setHeader(
-    "Cache-Control",
-    `public, max-age=${MAX_AGE}, must-revalidate'`,
-  );
-  res.setHeader(
-    "Expires",
-    new Date(Date.now().valueOf() + MAX_AGE).toUTCString(),
-  );
-}
-
 async function initStatic(router) {
   let compiler: any = null;
   if (
@@ -215,7 +187,7 @@ async function initStatic(router) {
     router.use("/static", webpackHotMiddleware(compiler, {}));
   } else {
     router.use(
-      join("/static", STATIC_PATH, "app.html"),
+      join("/static", "app.html"),
       express.static(join(STATIC_PATH, "app.html"), {
         setHeaders: cacheShortTerm,
       }),
diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml
index 9c4545b815..e9efdf65d2 100644
--- a/src/packages/pnpm-lock.yaml
+++ b/src/packages/pnpm-lock.yaml
@@ -1963,6 +1963,9 @@ importers:
       '@types/debug':
         specifier: ^4.1.12
         version: 4.1.12
+      '@types/ms':
+        specifier: ^0.7.31
+        version: 0.7.34
       async:
         specifier: ^1.5.2
         version: 1.5.2
@@ -1999,6 +2002,9 @@ importers:
       lru-cache:
         specifier: ^7.18.3
         version: 7.18.3
+      ms:
+        specifier: 2.1.2
+        version: 2.1.2
       prop-types:
         specifier: ^15.7.2
         version: 15.8.1
@@ -4184,9 +4190,6 @@ packages:
   '@types/mocha@10.0.8':
     resolution: {integrity: sha512-HfMcUmy9hTMJh66VNcmeC9iVErIZJli2bszuXc6julh5YGuRb/W5OnkHjwLNYdFlMis0sY3If5SEAp+PktdJjw==}
 
-  '@types/ms@0.7.31':
-    resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
-
   '@types/ms@0.7.34':
     resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==}
 
@@ -12582,7 +12585,7 @@ snapshots:
       awaiting: 3.0.0
       cheerio: 1.0.0-rc.12
       csv-parse: 5.5.6
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color
 
@@ -13581,7 +13584,7 @@ snapshots:
       '@types/xml-encryption': 1.2.4
       '@types/xml2js': 0.4.14
       '@xmldom/xmldom': 0.8.10
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       xml-crypto: 3.2.0
       xml-encryption: 3.0.2
       xml2js: 0.5.0
@@ -14311,7 +14314,7 @@ snapshots:
 
   '@types/debug@4.1.12':
     dependencies:
-      '@types/ms': 0.7.31
+      '@types/ms': 0.7.34
 
   '@types/dot-object@2.1.6': {}
 
@@ -14460,8 +14463,6 @@ snapshots:
 
   '@types/mocha@10.0.8': {}
 
-  '@types/ms@0.7.31': {}
-
   '@types/ms@0.7.34': {}
 
   '@types/node-cleanup@2.1.5': {}
@@ -14920,13 +14921,13 @@ snapshots:
 
   agent-base@6.0.2:
     dependencies:
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color
 
   agent-base@7.1.1:
     dependencies:
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color
 
@@ -15239,7 +15240,7 @@ snapshots:
 
   axios@1.7.7:
     dependencies:
-      follow-redirects: 1.15.6
+      follow-redirects: 1.15.6(debug@4.3.7)
       form-data: 4.0.0
       proxy-from-env: 1.1.0
     transitivePeerDependencies:
@@ -16418,10 +16419,6 @@ snapshots:
     dependencies:
       ms: 2.1.2
 
-  debug@4.3.7:
-    dependencies:
-      ms: 2.1.3
-
   debug@4.3.7(supports-color@8.1.1):
     dependencies:
       ms: 2.1.3
@@ -17408,8 +17405,6 @@ snapshots:
     transitivePeerDependencies:
       - encoding
 
-  follow-redirects@1.15.6: {}
-
   follow-redirects@1.15.6(debug@4.3.7):
     optionalDependencies:
       debug: 4.3.7(supports-color@8.1.1)
@@ -18172,7 +18167,7 @@ snapshots:
     dependencies:
       '@tootallnate/once': 2.0.0
       agent-base: 6.0.2
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color
 
@@ -18199,14 +18194,14 @@ snapshots:
   https-proxy-agent@5.0.1:
     dependencies:
       agent-base: 6.0.2
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color
 
   https-proxy-agent@7.0.2:
     dependencies:
       agent-base: 7.1.1
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color
 
diff --git a/src/packages/util/http-caching.ts b/src/packages/util/http-caching.ts
new file mode 100644
index 0000000000..4b6a9e3cac
--- /dev/null
+++ b/src/packages/util/http-caching.ts
@@ -0,0 +1,33 @@
+/*
+Helper functions for caching static files when using express
+*/
+
+import ms from "ms";
+
+// Used for longterm caching of files. This should be in units of seconds.
+const MAX_AGE = Math.round(ms("10 days") / 1000);
+const SHORT_AGE = Math.round(ms("10 seconds") / 1000);
+
+export function cacheShortTerm(res) {
+  res.setHeader(
+    "Cache-Control",
+    `public, max-age=${SHORT_AGE}, must-revalidate`,
+  );
+  res.setHeader(
+    "Expires",
+    new Date(Date.now().valueOf() + SHORT_AGE).toUTCString(),
+  );
+}
+
+// Various files such as the webpack static content should be cached long-term,
+// and we use this function to set appropriate headers at various points below.
+export function cacheLongTerm(res) {
+  res.setHeader(
+    "Cache-Control",
+    `public, max-age=${MAX_AGE}, must-revalidate'`,
+  );
+  res.setHeader(
+    "Expires",
+    new Date(Date.now().valueOf() + MAX_AGE).toUTCString(),
+  );
+}
diff --git a/src/packages/util/package.json b/src/packages/util/package.json
index bb4f13a1e1..386f7a87d1 100644
--- a/src/packages/util/package.json
+++ b/src/packages/util/package.json
@@ -12,7 +12,8 @@
     "./sync/table": "./dist/sync/table/index.js",
     "./sync/editor/db": "./dist/sync/editor/db/index.js",
     "./licenses/purchase/*": "./dist/licenses/purchase/*.js",
-    "./redux/*": "./dist/redux/*.js"
+    "./redux/*": "./dist/redux/*.js",
+    "./http-caching": "./dist/http-caching.js"
   },
   "scripts": {
     "preinstall": "npx only-allow pnpm",
@@ -21,24 +22,15 @@
     "test": "pnpm exec jest",
     "prepublishOnly": "pnpm test"
   },
-  "files": [
-    "dist/**",
-    "bin/**",
-    "README.md",
-    "package.json"
-  ],
+  "files": ["dist/**", "bin/**", "README.md", "package.json"],
   "author": "SageMath, Inc.",
-  "keywords": [
-    "utilities",
-    "mathjax",
-    "markdown",
-    "cocalc"
-  ],
+  "keywords": ["utilities", "mathjax", "markdown", "cocalc"],
   "license": "SEE LICENSE.md",
   "dependencies": {
     "@ant-design/colors": "^6.0.0",
     "@cocalc/util": "workspace:*",
     "@types/debug": "^4.1.12",
+    "@types/ms": "^0.7.31",
     "async": "^1.5.2",
     "awaiting": "^3.0.0",
     "dayjs": "^1.11.11",
@@ -51,6 +43,7 @@
     "jsonic": "^1.0.1",
     "lodash": "^4.17.21",
     "lru-cache": "^7.18.3",
+    "ms": "2.1.2",
     "prop-types": "^15.7.2",
     "react-intl": "^6.7.0",
     "redux": "^4.2.1",

From 409ef637a24acab5117ee3d1688d074ac29516ed Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Tue, 15 Oct 2024 22:40:18 +0000
Subject: [PATCH 03/35] compute express server: add new entry point

---
 src/compute/compute/lib/http-server.ts        |  6 ++-
 .../frontend/account/account-page.tsx         |  5 +++
 src/packages/frontend/app-framework/index.ts  |  6 +++
 src/packages/frontend/compute/entry-point.ts  | 44 +++++++++++++++++++
 src/packages/frontend/embed/index.ts          |  4 +-
 src/packages/frontend/entry-point.ts          |  3 +-
 src/packages/static/package.json              |  2 +-
 src/packages/static/src/plugins/app-loader.ts | 11 +++++
 src/packages/static/src/rspack.config.ts      |  6 +++
 src/packages/static/src/webapp-compute.ts     | 11 +++++
 10 files changed, 94 insertions(+), 4 deletions(-)
 create mode 100644 src/packages/frontend/compute/entry-point.ts
 create mode 100644 src/packages/static/src/webapp-compute.ts

diff --git a/src/compute/compute/lib/http-server.ts b/src/compute/compute/lib/http-server.ts
index ccc7fcbbe5..81227f68e7 100644
--- a/src/compute/compute/lib/http-server.ts
+++ b/src/compute/compute/lib/http-server.ts
@@ -13,7 +13,7 @@ import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
 
 const logger = getLogger("compute:http-server");
 
-const ENTRY_POINT = "app.html";
+const ENTRY_POINT = "compute.html";
 
 export function initHttpServer({
   port = 5004,
@@ -47,6 +47,10 @@ export function initHttpServer({
     express.static(STATIC_PATH, { setHeaders: cacheLongTerm }),
   );
 
+  app.get("/customize", (_req, res) => {
+    res.json({ configuration: {}, registration: false });
+  });
+
   server.listen(port, host, () => {
     logger.info(`Server listening http://${host}:${port}`);
   });
diff --git a/src/packages/frontend/account/account-page.tsx b/src/packages/frontend/account/account-page.tsx
index f64dcf6737..c9dda55960 100644
--- a/src/packages/frontend/account/account-page.tsx
+++ b/src/packages/frontend/account/account-page.tsx
@@ -16,6 +16,7 @@ import { useIntl } from "react-intl";
 import { SignOut } from "@cocalc/frontend/account/sign-out";
 import { AntdTabItem, Col, Row, Tabs } from "@cocalc/frontend/antd-bootstrap";
 import {
+  entryPoint,
   React,
   redux,
   useIsMountedRef,
@@ -259,6 +260,10 @@ export const AccountPage: React.FC = () => {
 
 function RedirectToNextApp({}) {
   const isMountedRef = useIsMountedRef();
+  if (entryPoint == "compute") {
+    // no login page for compute cocalc app
+    return;
+  }
 
   useEffect(() => {
     const f = () => {
diff --git a/src/packages/frontend/app-framework/index.ts b/src/packages/frontend/app-framework/index.ts
index 132b936755..3fdf2a2aff 100644
--- a/src/packages/frontend/app-framework/index.ts
+++ b/src/packages/frontend/app-framework/index.ts
@@ -3,6 +3,12 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
+let entryPoint = "app"; // default
+export { entryPoint };
+export function setEntryPoint(x) {
+  entryPoint = x;
+}
+
 // Not sure where this should go...
 declare global {
   interface Window {
diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts
new file mode 100644
index 0000000000..f22ce6b7f8
--- /dev/null
+++ b/src/packages/frontend/compute/entry-point.ts
@@ -0,0 +1,44 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+/*
+Entry point for compute server version of CoCalc...
+*/
+
+// Load/initialize Redux-based react functionality
+import "@cocalc/frontend/client/client";
+import { redux, setEntryPoint } from "../app-framework";
+import "../jquery-plugins";
+// Initialize app stores, actions, etc.
+import { init as initAccount } from "../account";
+import { init as initApp } from "../app/init";
+import { init as initProjects } from "../projects";
+import { init as initMarkdown } from "../markdown/markdown-input/main";
+import { init as initCrashBanner } from "../crash-banner";
+
+// Do not delete this without first looking at https://github.com/sagemathinc/cocalc/issues/5390
+// This import of codemirror forces the initial full load of codemirror
+// as part of the main webpack entry point.
+import "codemirror";
+
+import { init as initLast } from "../last";
+import { render } from "../app/render";
+
+export async function init() {
+  setEntryPoint("compute");
+  initAccount(redux);
+  initApp();
+  initProjects();
+  initMarkdown();
+  initLast();
+  try {
+    await render();
+  } finally {
+    // don't insert the crash banner until the main app has rendered,
+    // or user would see the banner for a moment.
+    initCrashBanner();
+  }
+  console.log("Loaded Compute Server Entry Point");
+}
diff --git a/src/packages/frontend/embed/index.ts b/src/packages/frontend/embed/index.ts
index 3e492c8d3f..8dc77f4a48 100644
--- a/src/packages/frontend/embed/index.ts
+++ b/src/packages/frontend/embed/index.ts
@@ -12,7 +12,8 @@ console.log("Embed mode");
 
 // Load/initialize Redux-based react functionality
 import "@cocalc/frontend/client/client";
-import { redux } from "../app-framework";
+import { redux, setEntryPoint } from "../app-framework";
+
 import "../jquery-plugins";
 
 // Initialize app stores, actions, etc.
@@ -32,6 +33,7 @@ import { init as initLast } from "../last";
 import { render } from "../app/render";
 
 export async function init() {
+  setEntryPoint("embed");
   initAccount(redux);
   initApp();
   initProjects();
diff --git a/src/packages/frontend/entry-point.ts b/src/packages/frontend/entry-point.ts
index 21a9db783e..bf786f3392 100644
--- a/src/packages/frontend/entry-point.ts
+++ b/src/packages/frontend/entry-point.ts
@@ -13,7 +13,7 @@ debug.log = console.log.bind(console); // see https://github.com/debug-js/debug#
 import { COCALC_MINIMAL } from "./fullscreen";
 
 // Load/initialize Redux-based react functionality
-import { redux } from "./app-framework";
+import { redux, setEntryPoint } from "./app-framework";
 
 // Systemwide notifications that are broadcast to all users (and set by admins)
 import "./system-notifications";
@@ -50,6 +50,7 @@ import { init as initLast } from "./last";
 import { render } from "./app/render";
 
 export async function init() {
+  setEntryPoint("app");
   initAccount(redux);
   initApp();
   initProjects();
diff --git a/src/packages/static/package.json b/src/packages/static/package.json
index 50cc113292..a2cf0dd6dc 100644
--- a/src/packages/static/package.json
+++ b/src/packages/static/package.json
@@ -18,7 +18,7 @@
     "build0": "pnpm run copy-css && cd src && ../../node_modules/.bin/tsc --build",
     "build": "pnpm run build0 && ./production-build.py",
     "build-dev": "pnpm run build0 && NODE_ENV=development pnpm rspack build",
-    "watch": "NODE_ENV=development pnpm rspack build -w",
+    "watch": "pnpm tsc && NODE_ENV=development pnpm rspack build -w",
     "test": "pnpm exec jest",
     "prepublishOnly": "pnpm test"
   },
diff --git a/src/packages/static/src/plugins/app-loader.ts b/src/packages/static/src/plugins/app-loader.ts
index f65eb15313..f18f9678ad 100644
--- a/src/packages/static/src/plugins/app-loader.ts
+++ b/src/packages/static/src/plugins/app-loader.ts
@@ -27,4 +27,15 @@ export default function appLoaderPlugin(
       chunks: ["load", "embed"],
     }),
   );
+
+  registerPlugin(
+    "Compute -- generates the compute.html file",
+    new rspack.HtmlRspackPlugin({
+      title,
+      filename: "compute.html",
+      template: resolve(__dirname, "../app.html"),
+      hash: PRODMODE,
+      chunks: ["load", "compute"],
+    }),
+  );
 }
diff --git a/src/packages/static/src/rspack.config.ts b/src/packages/static/src/rspack.config.ts
index 203ee2bf47..cde0d5fe04 100644
--- a/src/packages/static/src/rspack.config.ts
+++ b/src/packages/static/src/rspack.config.ts
@@ -168,6 +168,12 @@ export default function getConfig({ middleware }: Options = {}) {
         ]),
         dependOn: "load",
       },
+      compute: {
+        import: insertHotMiddlewareUrl([
+          resolve("dist-ts/src/webapp-compute.js"),
+        ]),
+        dependOn: "load",
+      },
     },
     /* Why chunkhash below, rather than contenthash? This says contenthash is a special
      thing for css and other text files only (??):
diff --git a/src/packages/static/src/webapp-compute.ts b/src/packages/static/src/webapp-compute.ts
new file mode 100644
index 0000000000..429053595f
--- /dev/null
+++ b/src/packages/static/src/webapp-compute.ts
@@ -0,0 +1,11 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+import "./webapp-libraries";
+import { init } from "@cocalc/frontend/compute/entry-point";
+import { startedUp } from "./webapp-error";
+
+init();
+startedUp();

From cde9a06416265662480a2f94b04fad85bde56aee Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Tue, 15 Oct 2024 22:49:40 +0000
Subject: [PATCH 04/35] http compression

---
 src/compute/compute/lib/http-server.ts       |   3 +
 src/compute/compute/package.json             |   2 +
 src/compute/pnpm-lock.yaml                   | 126 +++++++++++++++++++
 src/packages/frontend/compute/entry-point.ts |   7 +-
 4 files changed, 132 insertions(+), 6 deletions(-)

diff --git a/src/compute/compute/lib/http-server.ts b/src/compute/compute/lib/http-server.ts
index 81227f68e7..b7514452d0 100644
--- a/src/compute/compute/lib/http-server.ts
+++ b/src/compute/compute/lib/http-server.ts
@@ -3,6 +3,7 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
+import compression from "compression";
 import express from "express";
 import { createServer } from "http";
 import { getLogger } from "@cocalc/backend/logger";
@@ -29,6 +30,8 @@ export function initHttpServer({
   const app = express();
   const server = createServer(app);
 
+  app.use(compression());
+
   app.get("/", (_req, res) => {
     const files = manager.getOpenFiles();
     res.send(
diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index f635dc4d3e..f43b183eb9 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -38,9 +38,11 @@
     "@cocalc/sync-fs": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
+    "@types/compression": "^1.7.5",
     "@types/ms": "^0.7.34",
     "@types/ws": "^8.5.9",
     "awaiting": "^3.0.0",
+    "compression": "^1.7.4",
     "cookie": "^1.0.0",
     "debug": "^4.3.2",
     "express": "^4.20.0",
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index 039797180b..7ae175986b 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -44,6 +44,9 @@ importers:
       '@cocalc/util':
         specifier: workspace:*
         version: link:../util
+      '@types/compression':
+        specifier: ^1.7.5
+        version: 1.7.5
       '@types/ms':
         specifier: ^0.7.34
         version: 0.7.34
@@ -53,6 +56,9 @@ importers:
       awaiting:
         specifier: ^3.0.0
         version: 3.0.0
+      compression:
+        specifier: ^1.7.4
+        version: 1.7.4
       cookie:
         specifier: ^1.0.0
         version: 1.0.0
@@ -103,15 +109,48 @@ packages:
     resolution: {integrity: sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==}
     engines: {node: '>=12'}
 
+  '@types/body-parser@1.19.5':
+    resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==}
+
+  '@types/compression@1.7.5':
+    resolution: {integrity: sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==}
+
+  '@types/connect@3.4.38':
+    resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==}
+
   '@types/cookie@0.6.0':
     resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==}
 
+  '@types/express-serve-static-core@5.0.0':
+    resolution: {integrity: sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==}
+
+  '@types/express@5.0.0':
+    resolution: {integrity: sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==}
+
+  '@types/http-errors@2.0.4':
+    resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==}
+
+  '@types/mime@1.3.5':
+    resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==}
+
   '@types/ms@0.7.34':
     resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==}
 
   '@types/node@18.16.14':
     resolution: {integrity: sha512-+ImzUB3mw2c5ISJUq0punjDilUQ5GnUim0ZRvchHIWJmOC0G+p0kzhXBqj6cDjK0QdPFwzrHWgrJp3RPvCG5qg==}
 
+  '@types/qs@6.9.16':
+    resolution: {integrity: sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==}
+
+  '@types/range-parser@1.2.7':
+    resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==}
+
+  '@types/send@0.17.4':
+    resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==}
+
+  '@types/serve-static@1.15.7':
+    resolution: {integrity: sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==}
+
   '@types/ws@8.5.9':
     resolution: {integrity: sha512-jbdrY0a8lxfdTp/+r7Z4CkycbOFN8WX+IOchLJr3juT/xzbJ8URyTVSJ/hvNdadTgM1mnedb47n+Y31GsFnQlg==}
 
@@ -148,6 +187,10 @@ packages:
   buffer@5.7.1:
     resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
 
+  bytes@3.0.0:
+    resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==}
+    engines: {node: '>= 0.8'}
+
   bytes@3.1.2:
     resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}
     engines: {node: '>= 0.8'}
@@ -159,6 +202,14 @@ packages:
   chownr@1.1.4:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
 
+  compressible@2.0.18:
+    resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==}
+    engines: {node: '>= 0.6'}
+
+  compression@1.7.4:
+    resolution: {integrity: sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==}
+    engines: {node: '>= 0.8.0'}
+
   content-disposition@0.5.4:
     resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
     engines: {node: '>= 0.6'}
@@ -413,6 +464,10 @@ packages:
     resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}
     engines: {node: '>= 0.8'}
 
+  on-headers@1.0.2:
+    resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==}
+    engines: {node: '>= 0.8'}
+
   once@1.4.0:
     resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
 
@@ -458,6 +513,9 @@ packages:
     resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
     engines: {node: '>= 6'}
 
+  safe-buffer@5.1.2:
+    resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==}
+
   safe-buffer@5.2.1:
     resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
 
@@ -580,12 +638,58 @@ snapshots:
 
   '@isaacs/ttlcache@1.4.1': {}
 
+  '@types/body-parser@1.19.5':
+    dependencies:
+      '@types/connect': 3.4.38
+      '@types/node': 18.16.14
+
+  '@types/compression@1.7.5':
+    dependencies:
+      '@types/express': 5.0.0
+
+  '@types/connect@3.4.38':
+    dependencies:
+      '@types/node': 18.16.14
+
   '@types/cookie@0.6.0': {}
 
+  '@types/express-serve-static-core@5.0.0':
+    dependencies:
+      '@types/node': 18.16.14
+      '@types/qs': 6.9.16
+      '@types/range-parser': 1.2.7
+      '@types/send': 0.17.4
+
+  '@types/express@5.0.0':
+    dependencies:
+      '@types/body-parser': 1.19.5
+      '@types/express-serve-static-core': 5.0.0
+      '@types/qs': 6.9.16
+      '@types/serve-static': 1.15.7
+
+  '@types/http-errors@2.0.4': {}
+
+  '@types/mime@1.3.5': {}
+
   '@types/ms@0.7.34': {}
 
   '@types/node@18.16.14': {}
 
+  '@types/qs@6.9.16': {}
+
+  '@types/range-parser@1.2.7': {}
+
+  '@types/send@0.17.4':
+    dependencies:
+      '@types/mime': 1.3.5
+      '@types/node': 18.16.14
+
+  '@types/serve-static@1.15.7':
+    dependencies:
+      '@types/http-errors': 2.0.4
+      '@types/node': 18.16.14
+      '@types/send': 0.17.4
+
   '@types/ws@8.5.9':
     dependencies:
       '@types/node': 18.16.14
@@ -641,6 +745,8 @@ snapshots:
       base64-js: 1.5.1
       ieee754: 1.2.1
 
+  bytes@3.0.0: {}
+
   bytes@3.1.2: {}
 
   call-bind@1.0.7:
@@ -653,6 +759,22 @@ snapshots:
 
   chownr@1.1.4: {}
 
+  compressible@2.0.18:
+    dependencies:
+      mime-db: 1.52.0
+
+  compression@1.7.4:
+    dependencies:
+      accepts: 1.3.8
+      bytes: 3.0.0
+      compressible: 2.0.18
+      debug: 2.6.9
+      on-headers: 1.0.2
+      safe-buffer: 5.1.2
+      vary: 1.1.2
+    transitivePeerDependencies:
+      - supports-color
+
   content-disposition@0.5.4:
     dependencies:
       safe-buffer: 5.2.1
@@ -880,6 +1002,8 @@ snapshots:
     dependencies:
       ee-first: 1.1.1
 
+  on-headers@1.0.2: {}
+
   once@1.4.0:
     dependencies:
       wrappy: 1.0.2
@@ -941,6 +1065,8 @@ snapshots:
       string_decoder: 1.3.0
       util-deprecate: 1.0.2
 
+  safe-buffer@5.1.2: {}
+
   safe-buffer@5.2.1: {}
 
   safer-buffer@2.1.2: {}
diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts
index f22ce6b7f8..6b21e490bc 100644
--- a/src/packages/frontend/compute/entry-point.ts
+++ b/src/packages/frontend/compute/entry-point.ts
@@ -11,18 +11,13 @@ Entry point for compute server version of CoCalc...
 import "@cocalc/frontend/client/client";
 import { redux, setEntryPoint } from "../app-framework";
 import "../jquery-plugins";
-// Initialize app stores, actions, etc.
+
 import { init as initAccount } from "../account";
 import { init as initApp } from "../app/init";
 import { init as initProjects } from "../projects";
 import { init as initMarkdown } from "../markdown/markdown-input/main";
 import { init as initCrashBanner } from "../crash-banner";
-
-// Do not delete this without first looking at https://github.com/sagemathinc/cocalc/issues/5390
-// This import of codemirror forces the initial full load of codemirror
-// as part of the main webpack entry point.
 import "codemirror";
-
 import { init as initLast } from "../last";
 import { render } from "../app/render";
 

From 47b209bb4ded6b922204b1b85d6458352eedbef3 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Tue, 15 Oct 2024 23:39:39 +0000
Subject: [PATCH 05/35] compute http server: steps towards websocket connection

---
 src/compute/compute/lib/http-server.ts       | 11 ++++++++-
 src/compute/compute/lib/manager.ts           |  4 ++--
 src/packages/frontend/compute/entry-point.ts | 24 ++++++++++++++++++--
 src/packages/frontend/customize.tsx          |  4 ++++
 src/packages/frontend/projects/store.ts      | 11 ++++++++-
 src/packages/static/package.json             |  2 +-
 6 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/src/compute/compute/lib/http-server.ts b/src/compute/compute/lib/http-server.ts
index b7514452d0..5d8edc610f 100644
--- a/src/compute/compute/lib/http-server.ts
+++ b/src/compute/compute/lib/http-server.ts
@@ -39,6 +39,10 @@ export function initHttpServer({
     );
   });
 
+  app.get("/settings", (_req, res) => {
+    res.send(`<h1><a href="${join("/static", ENTRY_POINT)}">CoCalc App</a>`);
+  });
+
   app.use(
     join("/static", ENTRY_POINT),
     express.static(join(STATIC_PATH, ENTRY_POINT), {
@@ -51,7 +55,12 @@ export function initHttpServer({
   );
 
   app.get("/customize", (_req, res) => {
-    res.json({ configuration: {}, registration: false });
+    res.json({
+      configuration: {
+        compute_server: { project_id: manager.project_id },
+      },
+      registration: false,
+    });
   });
 
   server.listen(port, host, () => {
diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts
index 6fff12ae63..227c476b49 100644
--- a/src/compute/compute/lib/manager.ts
+++ b/src/compute/compute/lib/manager.ts
@@ -82,12 +82,12 @@ export function manager(opts: Options) {
 export class Manager {
   private state: "new" | "init" | "ready" = "new";
   private sync_db;
-  private project_id: string;
+  public project_id: string;
   private home: string;
   private host?: string;
   private port?: number;
   private waitHomeFilesystemType?: string;
-  private compute_server_id: number;
+  public compute_server_id: number;
   private connections: { [path: string]: any } = {};
   private websocket;
   private client;
diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts
index 6b21e490bc..291d1a0154 100644
--- a/src/packages/frontend/compute/entry-point.ts
+++ b/src/packages/frontend/compute/entry-point.ts
@@ -11,7 +11,6 @@ Entry point for compute server version of CoCalc...
 import "@cocalc/frontend/client/client";
 import { redux, setEntryPoint } from "../app-framework";
 import "../jquery-plugins";
-
 import { init as initAccount } from "../account";
 import { init as initApp } from "../app/init";
 import { init as initProjects } from "../projects";
@@ -28,6 +27,7 @@ export async function init() {
   initProjects();
   initMarkdown();
   initLast();
+  initEntryPointState();
   try {
     await render();
   } finally {
@@ -35,5 +35,25 @@ export async function init() {
     // or user would see the banner for a moment.
     initCrashBanner();
   }
-  console.log("Loaded Compute Server Entry Point");
+  console.log("Loaded Compute Server Entry Point.");
+}
+
+import { fromJS } from "immutable";
+async function initEntryPointState() {
+  console.log("initEntryPointState");
+  const customizeStore = redux.getStore("customize");
+  await customizeStore.async_wait({
+    until: () => customizeStore.get("compute_server"),
+  });
+  const project_id = customizeStore.getIn([
+    "compute_server",
+    "project_id",
+  ]) as string;
+  const project_map = fromJS({
+    [project_id]: {
+      title: "Compute Server Project (TODO)",
+      state: { time: new Date(), state: "running" },
+    },
+  }) as any;
+  redux.getActions("projects").setState({ project_map });
 }
diff --git a/src/packages/frontend/customize.tsx b/src/packages/frontend/customize.tsx
index 129046db46..61ba24376a 100644
--- a/src/packages/frontend/customize.tsx
+++ b/src/packages/frontend/customize.tsx
@@ -180,6 +180,10 @@ export interface CustomizeState {
   insecure_test_mode?: boolean;
 
   i18n?: List<Locale>;
+
+  // in case we're connecting directly to a compute server,
+  // this has info about the configuration of that server.
+  compute_server?: TypedMap<{ project_id: string }>;
 }
 
 export class CustomizeStore extends Store<CustomizeState> {
diff --git a/src/packages/frontend/projects/store.ts b/src/packages/frontend/projects/store.ts
index e9d5b400bf..3ac21bf757 100644
--- a/src/packages/frontend/projects/store.ts
+++ b/src/packages/frontend/projects/store.ts
@@ -5,7 +5,12 @@
 import { List, Map, Set } from "immutable";
 import { fromPairs, isEmpty } from "lodash";
 import LRU from "lru-cache";
-import { redux, Store, TypedMap } from "@cocalc/frontend/app-framework";
+import {
+  entryPoint,
+  redux,
+  Store,
+  TypedMap,
+} from "@cocalc/frontend/app-framework";
 import { StudentProjectFunctionality } from "@cocalc/frontend/course/configuration/customize-student-project-functionality";
 import { CUSTOM_IMG_PREFIX } from "@cocalc/frontend/custom-software/util";
 import { WebsocketState } from "@cocalc/frontend/project/websocket/websocket-state";
@@ -243,6 +248,10 @@ export class ProjectsStore extends Store<ProjectsState> {
   'admin' - user is not owner/collaborator but is an admin, hence has rights.
   */
   public get_my_group(project_id: string): UserGroup | undefined {
+    if (entryPoint == "compute") {
+      // in compute server mode there is only one project.
+      return "owner";
+    }
     const account_store = redux.getStore("account");
     if (account_store == null) {
       return;
diff --git a/src/packages/static/package.json b/src/packages/static/package.json
index a2cf0dd6dc..e1bc853301 100644
--- a/src/packages/static/package.json
+++ b/src/packages/static/package.json
@@ -18,7 +18,7 @@
     "build0": "pnpm run copy-css && cd src && ../../node_modules/.bin/tsc --build",
     "build": "pnpm run build0 && ./production-build.py",
     "build-dev": "pnpm run build0 && NODE_ENV=development pnpm rspack build",
-    "watch": "pnpm tsc && NODE_ENV=development pnpm rspack build -w",
+    "watch": "../node_modules/.bin/tsc && NODE_ENV=development pnpm rspack build -w",
     "test": "pnpm exec jest",
     "prepublishOnly": "pnpm test"
   },

From cde91ee2258ccb6823be34ad1fe70563c6ec02f6 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 00:59:32 +0000
Subject: [PATCH 06/35] compute http server: add a primus websocket server
 (doesn't do anything yet)

---
 .../{http-server.ts => http-server/index.ts}  |  18 +-
 .../compute/lib/http-server/websocket.ts      |  49 +++
 src/compute/compute/lib/logger.ts             |   6 +
 src/compute/compute/package.json              |   3 +
 src/compute/pnpm-lock.yaml                    | 291 ++++++++++++++++--
 src/packages/frontend/client/project.ts       |   1 -
 .../frontend/project/websocket/connect.ts     |  24 +-
 7 files changed, 355 insertions(+), 37 deletions(-)
 rename src/compute/compute/lib/{http-server.ts => http-server/index.ts} (74%)
 create mode 100644 src/compute/compute/lib/http-server/websocket.ts
 create mode 100644 src/compute/compute/lib/logger.ts

diff --git a/src/compute/compute/lib/http-server.ts b/src/compute/compute/lib/http-server/index.ts
similarity index 74%
rename from src/compute/compute/lib/http-server.ts
rename to src/compute/compute/lib/http-server/index.ts
index 5d8edc610f..9735e45920 100644
--- a/src/compute/compute/lib/http-server.ts
+++ b/src/compute/compute/lib/http-server/index.ts
@@ -6,13 +6,14 @@
 import compression from "compression";
 import express from "express";
 import { createServer } from "http";
-import { getLogger } from "@cocalc/backend/logger";
-import type { Manager } from "./manager";
+import { getLogger } from "../logger";
+import type { Manager } from "../manager";
 import { path as STATIC_PATH } from "@cocalc/static";
 import { join } from "path";
 import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
+import initWebsocket from "./websocket";
 
-const logger = getLogger("compute:http-server");
+const logger = getLogger("http-server");
 
 const ENTRY_POINT = "compute.html";
 
@@ -30,6 +31,13 @@ export function initHttpServer({
   const app = express();
   const server = createServer(app);
 
+  // this is expected by the frontend code for where to find the project.
+  const projectBase = `/${manager.project_id}/raw/`;
+  logger.info({ projectBase });
+
+  app.use(projectBase, initWebsocket(server, projectBase));
+
+  // CRITICAL: compression must be after websocket above!
   app.use(compression());
 
   app.get("/", (_req, res) => {
@@ -44,8 +52,8 @@ export function initHttpServer({
   });
 
   app.use(
-    join("/static", ENTRY_POINT),
-    express.static(join(STATIC_PATH, ENTRY_POINT), {
+    `/static/${ENTRY_POINT}`,
+    express.static(`/${STATIC_PATH}/${ENTRY_POINT}`, {
       setHeaders: cacheShortTerm,
     }),
   );
diff --git a/src/compute/compute/lib/http-server/websocket.ts b/src/compute/compute/lib/http-server/websocket.ts
new file mode 100644
index 0000000000..c6970df72e
--- /dev/null
+++ b/src/compute/compute/lib/http-server/websocket.ts
@@ -0,0 +1,49 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+/*
+Create the Primus realtime socket server.
+
+Similar to @cocalc/project/browser-websocket/server.ts
+*/
+
+import { join } from "path";
+import { Router } from "express";
+import { Server } from "http";
+import Primus from "primus";
+import type { PrimusWithChannels } from "@cocalc/terminal";
+import { getLogger } from "../logger";
+const logger = getLogger("websocket");
+
+export default function initWebsocket(
+  server: Server,
+  basePath: string,
+): Router {
+  const opts = {
+    pathname: join(basePath, ".smc", "ws"),
+    transformer: "websockets",
+  } as const;
+  logger.debug(`Initializing primus websocket server at "${opts.pathname}"...`);
+  const primus = new Primus(server, opts) as PrimusWithChannels;
+
+  // add multiplex to Primus so we have channels.
+  primus.plugin("multiplex", require("@cocalc/primus-multiplex"));
+  primus.plugin("responder", require("@cocalc/primus-responder"));
+
+  const router = Router();
+  const library: string = primus.library();
+  // See note above.
+  //UglifyJS.minify(primus.library()).code;
+
+  router.get("/.smc/primus.js", (_, res) => {
+    logger.debug("serving up primus.js to a specific client");
+    res.send(library);
+  });
+  logger.debug(
+    `waiting for clients to request primus.js (length=${library.length})...`,
+  );
+
+  return router;
+}
diff --git a/src/compute/compute/lib/logger.ts b/src/compute/compute/lib/logger.ts
new file mode 100644
index 0000000000..aee2a5a9f3
--- /dev/null
+++ b/src/compute/compute/lib/logger.ts
@@ -0,0 +1,6 @@
+import debug from "debug";
+
+export function getLogger(name) {
+  const d = debug(`cocalc:compute:${name}`);
+  return { info: d, debug: d };
+}
diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index f43b183eb9..95693621ca 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -32,6 +32,8 @@
     "@cocalc/backend": "workspace:*",
     "@cocalc/compute": "link:",
     "@cocalc/jupyter": "workspace:*",
+    "@cocalc/primus-multiplex": "^1.1.0",
+    "@cocalc/primus-responder": "^1.0.5",
     "@cocalc/static": "workspace:*",
     "@cocalc/sync": "workspace:*",
     "@cocalc/sync-client": "workspace:*",
@@ -46,6 +48,7 @@
     "cookie": "^1.0.0",
     "debug": "^4.3.2",
     "express": "^4.20.0",
+    "primus": "^8.0.7",
     "websocketfs": "^0.17.4",
     "ws": "^8.18.0"
   },
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index 7ae175986b..4ab726b838 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -6,12 +6,6 @@ settings:
 
 importers:
 
-  api-client: {}
-
-  backend: {}
-
-  comm: {}
-
   compute:
     dependencies:
       '@cocalc/api-client':
@@ -26,6 +20,12 @@ importers:
       '@cocalc/jupyter':
         specifier: workspace:*
         version: link:../jupyter
+      '@cocalc/primus-multiplex':
+        specifier: ^1.1.0
+        version: 1.1.0
+      '@cocalc/primus-responder':
+        specifier: ^1.0.5
+        version: 1.0.5
       '@cocalc/static':
         specifier: workspace:*
         version: link:../static
@@ -68,6 +68,9 @@ importers:
       express:
         specifier: ^4.20.0
         version: 4.21.1
+      primus:
+        specifier: ^8.0.7
+        version: 8.0.9
       websocketfs:
         specifier: ^0.17.4
         version: 0.17.4
@@ -85,26 +88,19 @@ importers:
         specifier: ^5.6.3
         version: 5.6.3
 
-  jupyter: {}
-
-  static: {}
-
-  sync: {}
-
-  sync-client: {}
-
-  sync-fs: {}
-
-  terminal: {}
-
-  util: {}
-
 packages:
 
   '@cocalc/fuse-native@2.4.1':
     resolution: {integrity: sha512-Z4mzgaJyz/vNIv5Z+8eK2D5n+xkcNbKZ4/yQI56685VD0SwOaD9WNRdvD7u6s1IiB2FoYqljdBjat/MwzNlb3g==}
     hasBin: true
 
+  '@cocalc/primus-multiplex@1.1.0':
+    resolution: {integrity: sha512-o8AOFVs996NQ2nvCjoSzNkBJRkPYi5XVPw0BbtMZUz+hnegSCfGawT59S++wta0CzGdFPWnyPtQqVJpobQlT4w==}
+
+  '@cocalc/primus-responder@1.0.5':
+    resolution: {integrity: sha512-nFHqPo9zxbPNRkMzFH/ZbVrrp3RGb0vAVWSRG67+j8dwt3MN9pqZSEUOfWUt9AURsRJd9IR+gLnwrczQTNu6Zg==}
+    engines: {node: '>=0.10.28'}
+
   '@isaacs/ttlcache@1.4.1':
     resolution: {integrity: sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==}
     engines: {node: '>=12'}
@@ -161,9 +157,17 @@ packages:
     resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==}
     engines: {node: '>= 0.6'}
 
+  access-control@1.0.1:
+    resolution: {integrity: sha512-H5aqjkogmFxfaOrfn/e42vyspHVXuJ8er63KuljJXpOyJ1ZO/U5CrHfO8BLKIy2w7mBM02L5quL0vbfQqrGQbA==}
+
   array-flatten@1.1.1:
     resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==}
 
+  asyncemit@3.0.1:
+    resolution: {integrity: sha512-sVGuQSrkDjPJrnuMvWvltg4+UIO85qkDPGuSzVmj1LojoDR5DZPCVTSPRjYk1V0UPaeypFeMMoj22ZOG8egdyw==}
+    peerDependencies:
+      eventemitter3: '>=1.1.0'
+
   awaiting@3.0.0:
     resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==}
     engines: {node: '>=7.6.x'}
@@ -202,6 +206,27 @@ packages:
   chownr@1.1.4:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
 
+  color-convert@1.9.3:
+    resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==}
+
+  color-name@1.1.3:
+    resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==}
+
+  color-name@1.1.4:
+    resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
+
+  color-string@1.9.1:
+    resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==}
+
+  color@3.2.1:
+    resolution: {integrity: sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==}
+
+  colornames@1.1.1:
+    resolution: {integrity: sha512-/pyV40IrsdulWv+wFPmERh9k/mjsPZ64yUMDmWrtj/k1nmgrzzIENWKdaVKyBbvFdQWqkcaRxr+polCo3VMe7A==}
+
+  colorspace@1.1.4:
+    resolution: {integrity: sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==}
+
   compressible@2.0.18:
     resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==}
     engines: {node: '>= 0.6'}
@@ -210,6 +235,9 @@ packages:
     resolution: {integrity: sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==}
     engines: {node: '>= 0.8.0'}
 
+  connected@0.0.2:
+    resolution: {integrity: sha512-J8DB7618GkIYjc1RCxSdG3vffhhYRwHNEckjOGfwAbabQIMgKsL5c54IaWtisulEwoOEbEODEUak4kyJP2GJ/Q==}
+
   content-disposition@0.5.4:
     resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
     engines: {node: '>= 0.6'}
@@ -233,6 +261,9 @@ packages:
     resolution: {integrity: sha512-bsSztFoaR8bw9MlFCrTHzc1wOKCUKOBsbgFdoDilZDkETAOOjKSqV7L+EQLbTaylwvZasd9vM4MGKotJaUfSpA==}
     engines: {node: '>=18'}
 
+  create-server@1.0.2:
+    resolution: {integrity: sha512-hie+Kyero+jxt6dwKhLKtN23qSNiMn8mNIEjTjwzaZwH2y4tr4nYloeFrpadqV+ZqV9jQ15t3AKotaK8dOo45w==}
+
   cuint@0.2.2:
     resolution: {integrity: sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==}
 
@@ -277,9 +308,24 @@ packages:
     resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==}
     engines: {node: '>=8'}
 
+  diagnostics@1.1.1:
+    resolution: {integrity: sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==}
+
+  diagnostics@2.0.2:
+    resolution: {integrity: sha512-gvnlQHwkWTOeSM1iRNEwPcUuUwlhovzbuQzalKrTbcJhI5cvhtkRVZZqomwZt4pCl2dvbsugD6yyu+66rtMy3Q==}
+
   ee-first@1.1.1:
     resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
 
+  emits@3.0.0:
+    resolution: {integrity: sha512-WJSCMaN/qjIkzWy5Ayu0MDENFltcu4zTPPnWqdFPOVBtsENVTN+A3d76G61yuiVALsMK+76MejdPrwmccv/wag==}
+
+  enabled@1.0.2:
+    resolution: {integrity: sha512-nnzgVSpB35qKrUN8358SjO1bYAmxoThECTWw9s3J0x5G8A9hokKHVDFzBjVpCoSryo6MhN8woVyascN5jheaNA==}
+
+  enabled@2.0.0:
+    resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==}
+
   encodeurl@1.0.2:
     resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
     engines: {node: '>= 0.8'}
@@ -291,6 +337,9 @@ packages:
   end-of-stream@1.4.4:
     resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
 
+  env-variable@0.0.6:
+    resolution: {integrity: sha512-bHz59NlBbtS0NhftmR8+ExBEekE7br0e01jw+kk0NDro7TtZzBYZ5ScGPs3OmwnpyfHTHOtr1Y6uedCdrIldtg==}
+
   es-define-property@1.0.0:
     resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==}
     engines: {node: '>= 0.4'}
@@ -302,10 +351,20 @@ packages:
   escape-html@1.0.3:
     resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}
 
+  escape-string-regexp@4.0.0:
+    resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
+    engines: {node: '>=10'}
+
   etag@1.8.1:
     resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
     engines: {node: '>= 0.6'}
 
+  eventemitter3@4.0.7:
+    resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
+
+  eventemitter3@5.0.1:
+    resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}
+
   expand-template@2.0.3:
     resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}
     engines: {node: '>=6'}
@@ -314,6 +373,9 @@ packages:
     resolution: {integrity: sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==}
     engines: {node: '>= 0.10.0'}
 
+  extendible@0.1.1:
+    resolution: {integrity: sha512-AglckQA0TJV8/ZmhQcNmaaFcFFPXFIoZbfuoQOlGDK7Jh/roWotYzJ7ik1FBBCHBr8n7CgTR8lXXPAN8Rfb7rw==}
+
   file-uri-to-path@1.0.0:
     resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
 
@@ -321,6 +383,9 @@ packages:
     resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==}
     engines: {node: '>= 0.8'}
 
+  forwarded-for@1.1.0:
+    resolution: {integrity: sha512-1Yam9ht7GyMXMBvuwJfUYqpdtLVodtT5ee5JMBzGiSwVVeh37ZN8LuOWkNHd6ho2zUxpSZCHuQrt1Vjl2AxDNA==}
+
   forwarded@0.2.0:
     resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
     engines: {node: '>= 0.6'}
@@ -335,6 +400,9 @@ packages:
   function-bind@1.1.2:
     resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
 
+  fusing@1.0.0:
+    resolution: {integrity: sha512-gc2uPkiQy4zeMmLmdiCQXPe5zs+knBaoxGGe7R5Ve7fGEbrnnSOWiTKxq0elaO/RMBuTiZ1hzSGQa2+Sdl2h5w==}
+
   get-intrinsic@1.2.4:
     resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
     engines: {node: '>= 0.4'}
@@ -381,6 +449,15 @@ packages:
     resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
     engines: {node: '>= 0.10'}
 
+  is-arrayish@0.3.2:
+    resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
+
+  kuler@1.0.1:
+    resolution: {integrity: sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==}
+
+  kuler@2.0.0:
+    resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==}
+
   lz4@0.6.5:
     resolution: {integrity: sha512-KSZcJU49QZOlJSItaeIU3p8WoAvkTmD9fJqeahQXNu1iQ/kR0/mQLdbrK8JY9MY8f6AhJoMrihp1nu1xDbscSQ==}
     engines: {node: '>= 0.10'}
@@ -396,6 +473,9 @@ packages:
     resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
     engines: {node: '>= 0.6'}
 
+  millisecond@0.1.2:
+    resolution: {integrity: sha512-BJ8XtxY+woL+5TkP6uS6XvOArm0JVrX2otkgtWZseHpIax0oOOPW3cnwhOjRqbEJg7YRO/BDF7fO/PTWNT3T9Q==}
+
   mime-db@1.52.0:
     resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
     engines: {node: '>= 0.6'}
@@ -431,6 +511,11 @@ packages:
   nan@2.20.0:
     resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==}
 
+  nanoid@3.3.7:
+    resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==}
+    engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
+    hasBin: true
+
   nanoresource@1.3.0:
     resolution: {integrity: sha512-OI5dswqipmlYfyL3k/YMm7mbERlh4Bd1KuKdMHpeoVD1iVxqxaTMKleB4qaA2mbQZ6/zMNSxCXv9M9P/YbqTuQ==}
 
@@ -456,6 +541,11 @@ packages:
     resolution: {integrity: sha512-IRUxE4BVsHWXkV/SFOut4qTlagw2aM8T5/vnTsmrHJvVoKueJHRc/JaFND7QDDc61kLYUJ6qlZM3sqTSyx2dTw==}
     hasBin: true
 
+  node-uuid@1.4.8:
+    resolution: {integrity: sha512-TkCET/3rr9mUuRp+CpO7qfgT++aAxfDRaalQhwPFzI9BY/2rCDn6OfpZOVggi1AXfTPpfkTrg5f5WQx5G1uLxA==}
+    deprecated: Use uuid module instead
+    hasBin: true
+
   object-inspect@1.13.2:
     resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==}
     engines: {node: '>= 0.4'}
@@ -486,6 +576,12 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
 
+  predefine@0.1.3:
+    resolution: {integrity: sha512-Nq6APFC5OtQRl5TmMk6RlGwl6UOCtEqa+5ZTbKFp6tMw4wdMUa7Rief0UNE3fV5BgQahJ70QmDgeOog8RE9FMw==}
+
+  primus@8.0.9:
+    resolution: {integrity: sha512-gWsd6pWHAHGfyArl6DQU9iCAp4bAgFrintDpFbyA2r0wdzJ2n9SsffSaFqOKYZeE9wqKcBepnwBGoFKzNybqMA==}
+
   proxy-addr@2.0.7:
     resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
     engines: {node: '>= 0.10'}
@@ -539,6 +635,9 @@ packages:
     resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==}
     engines: {node: '>= 0.4'}
 
+  setheader@1.0.2:
+    resolution: {integrity: sha512-A704nIwzqGed0CnJZIqDE+0udMPS839ocgf1R9OJ8aq8vw4U980HWeNaD9ec8VnmBni9lyGEWDedOWXT/C5kxA==}
+
   setprototypeof@1.2.0:
     resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
 
@@ -552,10 +651,16 @@ packages:
   simple-get@4.0.1:
     resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
 
+  simple-swizzle@0.2.2:
+    resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
+
   statuses@2.0.1:
     resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
     engines: {node: '>= 0.8'}
 
+  storage-engine@3.0.7:
+    resolution: {integrity: sha512-V/jJykpPdsyDImLwu19syIAWn/Tb41tBDikQS+aQPH2h2OgqdLxwOg7wI9nPH3Y0Mh1ce566JZl2u+4eH1nAsg==}
+
   string_decoder@1.3.0:
     resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
 
@@ -570,6 +675,9 @@ packages:
     resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
     engines: {node: '>=6'}
 
+  text-hex@1.0.0:
+    resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==}
+
   toidentifier@1.0.1:
     resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
     engines: {node: '>=0.6'}
@@ -586,6 +694,9 @@ packages:
     engines: {node: '>=14.17'}
     hasBin: true
 
+  ultron@1.1.1:
+    resolution: {integrity: sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==}
+
   unpipe@1.0.0:
     resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==}
     engines: {node: '>= 0.8'}
@@ -636,6 +747,19 @@ snapshots:
       napi-macros: 2.2.2
       node-gyp-build: 4.8.2
 
+  '@cocalc/primus-multiplex@1.1.0':
+    dependencies:
+      escape-string-regexp: 4.0.0
+      eventemitter3: 5.0.1
+      predefine: 0.1.3
+
+  '@cocalc/primus-responder@1.0.5':
+    dependencies:
+      debug: 4.3.4
+      node-uuid: 1.4.8
+    transitivePeerDependencies:
+      - supports-color
+
   '@isaacs/ttlcache@1.4.1': {}
 
   '@types/body-parser@1.19.5':
@@ -705,8 +829,18 @@ snapshots:
       mime-types: 2.1.35
       negotiator: 0.6.3
 
+  access-control@1.0.1:
+    dependencies:
+      millisecond: 0.1.2
+      setheader: 1.0.2
+      vary: 1.1.2
+
   array-flatten@1.1.1: {}
 
+  asyncemit@3.0.1(eventemitter3@5.0.1):
+    dependencies:
+      eventemitter3: 5.0.1
+
   awaiting@3.0.0: {}
 
   base64-js@1.5.1: {}
@@ -759,6 +893,31 @@ snapshots:
 
   chownr@1.1.4: {}
 
+  color-convert@1.9.3:
+    dependencies:
+      color-name: 1.1.3
+
+  color-name@1.1.3: {}
+
+  color-name@1.1.4: {}
+
+  color-string@1.9.1:
+    dependencies:
+      color-name: 1.1.4
+      simple-swizzle: 0.2.2
+
+  color@3.2.1:
+    dependencies:
+      color-convert: 1.9.3
+      color-string: 1.9.1
+
+  colornames@1.1.1: {}
+
+  colorspace@1.1.4:
+    dependencies:
+      color: 3.2.1
+      text-hex: 1.0.0
+
   compressible@2.0.18:
     dependencies:
       mime-db: 1.52.0
@@ -775,6 +934,8 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  connected@0.0.2: {}
+
   content-disposition@0.5.4:
     dependencies:
       safe-buffer: 5.2.1
@@ -789,6 +950,10 @@ snapshots:
 
   cookie@1.0.0: {}
 
+  create-server@1.0.2:
+    dependencies:
+      connected: 0.0.2
+
   cuint@0.2.2: {}
 
   debug@2.6.9:
@@ -817,8 +982,29 @@ snapshots:
 
   detect-libc@2.0.3: {}
 
+  diagnostics@1.1.1:
+    dependencies:
+      colorspace: 1.1.4
+      enabled: 1.0.2
+      kuler: 1.0.1
+
+  diagnostics@2.0.2:
+    dependencies:
+      colorspace: 1.1.4
+      enabled: 2.0.0
+      kuler: 2.0.0
+      storage-engine: 3.0.7
+
   ee-first@1.1.1: {}
 
+  emits@3.0.0: {}
+
+  enabled@1.0.2:
+    dependencies:
+      env-variable: 0.0.6
+
+  enabled@2.0.0: {}
+
   encodeurl@1.0.2: {}
 
   encodeurl@2.0.0: {}
@@ -827,6 +1013,8 @@ snapshots:
     dependencies:
       once: 1.4.0
 
+  env-variable@0.0.6: {}
+
   es-define-property@1.0.0:
     dependencies:
       get-intrinsic: 1.2.4
@@ -835,8 +1023,14 @@ snapshots:
 
   escape-html@1.0.3: {}
 
+  escape-string-regexp@4.0.0: {}
+
   etag@1.8.1: {}
 
+  eventemitter3@4.0.7: {}
+
+  eventemitter3@5.0.1: {}
+
   expand-template@2.0.3: {}
 
   express@4.21.1:
@@ -875,6 +1069,8 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  extendible@0.1.1: {}
+
   file-uri-to-path@1.0.0: {}
 
   finalhandler@1.3.1:
@@ -889,6 +1085,8 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  forwarded-for@1.1.0: {}
+
   forwarded@0.2.0: {}
 
   fresh@0.5.2: {}
@@ -897,6 +1095,11 @@ snapshots:
 
   function-bind@1.1.2: {}
 
+  fusing@1.0.0:
+    dependencies:
+      emits: 3.0.0
+      predefine: 0.1.3
+
   get-intrinsic@1.2.4:
     dependencies:
       es-errors: 1.3.0
@@ -943,6 +1146,14 @@ snapshots:
 
   ipaddr.js@1.9.1: {}
 
+  is-arrayish@0.3.2: {}
+
+  kuler@1.0.1:
+    dependencies:
+      colornames: 1.1.1
+
+  kuler@2.0.0: {}
+
   lz4@0.6.5:
     dependencies:
       buffer: 5.7.1
@@ -956,6 +1167,8 @@ snapshots:
 
   methods@1.1.2: {}
 
+  millisecond@0.1.2: {}
+
   mime-db@1.52.0: {}
 
   mime-types@2.1.35:
@@ -978,6 +1191,8 @@ snapshots:
 
   nan@2.20.0: {}
 
+  nanoid@3.3.7: {}
+
   nanoresource@1.3.0:
     dependencies:
       inherits: 2.0.4
@@ -996,6 +1211,8 @@ snapshots:
 
   node-gyp-build@4.8.2: {}
 
+  node-uuid@1.4.8: {}
+
   object-inspect@1.13.2: {}
 
   on-finished@2.4.1:
@@ -1029,6 +1246,23 @@ snapshots:
       tar-fs: 2.1.1
       tunnel-agent: 0.6.0
 
+  predefine@0.1.3:
+    dependencies:
+      extendible: 0.1.1
+
+  primus@8.0.9:
+    dependencies:
+      access-control: 1.0.1
+      asyncemit: 3.0.1(eventemitter3@5.0.1)
+      create-server: 1.0.2
+      diagnostics: 2.0.2
+      eventemitter3: 5.0.1
+      forwarded-for: 1.1.0
+      fusing: 1.0.0
+      nanoid: 3.3.7
+      setheader: 1.0.2
+      ultron: 1.1.1
+
   proxy-addr@2.0.7:
     dependencies:
       forwarded: 0.2.0
@@ -1109,6 +1343,10 @@ snapshots:
       gopd: 1.0.1
       has-property-descriptors: 1.0.2
 
+  setheader@1.0.2:
+    dependencies:
+      diagnostics: 1.1.1
+
   setprototypeof@1.2.0: {}
 
   side-channel@1.0.6:
@@ -1126,8 +1364,17 @@ snapshots:
       once: 1.4.0
       simple-concat: 1.0.1
 
+  simple-swizzle@0.2.2:
+    dependencies:
+      is-arrayish: 0.3.2
+
   statuses@2.0.1: {}
 
+  storage-engine@3.0.7:
+    dependencies:
+      enabled: 2.0.0
+      eventemitter3: 4.0.7
+
   string_decoder@1.3.0:
     dependencies:
       safe-buffer: 5.2.1
@@ -1149,6 +1396,8 @@ snapshots:
       inherits: 2.0.4
       readable-stream: 3.6.2
 
+  text-hex@1.0.0: {}
+
   toidentifier@1.0.1: {}
 
   tunnel-agent@0.6.0:
@@ -1162,6 +1411,8 @@ snapshots:
 
   typescript@5.6.3: {}
 
+  ultron@1.1.1: {}
+
   unpipe@1.0.0: {}
 
   util-deprecate@1.0.2: {}
diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts
index 092196bf74..732564e96b 100644
--- a/src/packages/frontend/client/project.ts
+++ b/src/packages/frontend/client/project.ts
@@ -8,7 +8,6 @@ Functionality that mainly involves working with a specific project.
 */
 
 import { join } from "path";
-
 import { redux } from "@cocalc/frontend/app-framework";
 import computeServers from "@cocalc/frontend/compute/manager";
 import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
diff --git a/src/packages/frontend/project/websocket/connect.ts b/src/packages/frontend/project/websocket/connect.ts
index 2b04449565..d009a9fe33 100644
--- a/src/packages/frontend/project/websocket/connect.ts
+++ b/src/packages/frontend/project/websocket/connect.ts
@@ -16,8 +16,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { callback, delay } from "awaiting";
 import { ajax, globalEval } from "jquery";
 import { join } from "path";
-
-import { redux } from "@cocalc/frontend/app-framework";
+import { entryPoint, redux } from "@cocalc/frontend/app-framework";
 import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
 import { webapp_client } from "@cocalc/frontend/webapp-client";
 import { allow_project_to_run } from "../client-side-throttle";
@@ -88,17 +87,20 @@ async function connection_to_project0(project_id: string): Promise<any> {
           throw Error("currently reading one already");
         }
 
-        if (!webapp_client.is_signed_in()) {
-          // At least wait until main client is signed in, since nothing
-          // will work until that is the case anyways.
-          await once(webapp_client, "signed_in");
-        }
+        // for compute entry point, no sign in needed and project is assumed running.
+        if (entryPoint != "compute") {
+          if (!webapp_client.is_signed_in()) {
+            // At least wait until main client is signed in, since nothing
+            // will work until that is the case anyways.
+            await once(webapp_client, "signed_in");
+          }
 
-        log("wait_for_project_to_start...");
-        await wait_for_project_to_start(project_id);
-        log("wait_for_project_to_start: done");
+          log("wait_for_project_to_start...");
+          await wait_for_project_to_start(project_id);
+          log("wait_for_project_to_start: done");
+        }
 
-        // Now project is thought to be running, so maybe this will work:
+        // Now user is signed in and project is thought to be running, so maybe this will work:
         try {
           if (do_eval) {
             READING_PRIMUS_JS = true;

From 3291875dd4314ef96304d0805ab860e835249684 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 01:50:37 +0000
Subject: [PATCH 07/35] working on listing api

---
 .../compute/lib/http-server/websocket-api.ts  | 59 +++++++++++++++++++
 .../compute/lib/http-server/websocket.ts      |  6 +-
 src/compute/compute/package.json              | 13 +---
 src/compute/pnpm-lock.yaml                    |  3 +
 .../frontend/project/websocket/api.ts         |  2 +-
 src/packages/sync-client/lib/api.ts           |  2 +-
 6 files changed, 71 insertions(+), 14 deletions(-)
 create mode 100644 src/compute/compute/lib/http-server/websocket-api.ts

diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
new file mode 100644
index 0000000000..03b4a5380a
--- /dev/null
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -0,0 +1,59 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+// Similar to @cocalc/project/browser-websocket/api.ts
+// It might make sense to refactor this with that -- not sure yet.
+
+import { getLogger } from "../logger";
+import { version } from "@cocalc/util/smc-version";
+import type { Mesg } from "@cocalc/comm/websocket/types";
+
+const log = getLogger("websocket-api");
+
+let primus: any = undefined;
+export function initWebsocketApi(primus0): void {
+  primus = primus0;
+
+  primus.on("connection", function (spark) {
+    log.debug(`new connection from ${spark.address.ip} -- ${spark.id}`);
+
+    spark.on("request", async (data, done) => {
+      log.debug("primus-api", "request", data, "REQUEST");
+      const t0 = Date.now();
+      try {
+        const resp = await handleApiCall(data, spark);
+        done(resp);
+      } catch (err) {
+        // console.trace(); log.debug("primus-api error stacktrack", err.stack, err);
+        done({ error: err.toString(), status: "error" });
+      }
+      log.debug(
+        "primus-api",
+        "request",
+        data,
+        `FINISHED: time=${Date.now() - t0}ms`,
+      );
+    });
+  });
+
+  primus.on("disconnection", function (spark) {
+    log.debug(
+      "primus-api",
+      `end connection from ${spark.address.ip} -- ${spark.id}`,
+    );
+  });
+}
+
+async function handleApiCall(data: Mesg, _spark): Promise<any> {
+  switch (data.cmd) {
+    case "version":
+      return version;
+    case "listing":
+      // see packages/sync-fs/lib/index.ts
+      throw Error("todo");
+    default:
+      throw Error(`command "${(data as any).cmd}" not implemented`);
+  }
+}
diff --git a/src/compute/compute/lib/http-server/websocket.ts b/src/compute/compute/lib/http-server/websocket.ts
index c6970df72e..3d40797a00 100644
--- a/src/compute/compute/lib/http-server/websocket.ts
+++ b/src/compute/compute/lib/http-server/websocket.ts
@@ -14,7 +14,9 @@ import { Router } from "express";
 import { Server } from "http";
 import Primus from "primus";
 import type { PrimusWithChannels } from "@cocalc/terminal";
+import { initWebsocketApi } from "./websocket-api";
 import { getLogger } from "../logger";
+
 const logger = getLogger("websocket");
 
 export default function initWebsocket(
@@ -31,11 +33,11 @@ export default function initWebsocket(
   // add multiplex to Primus so we have channels.
   primus.plugin("multiplex", require("@cocalc/primus-multiplex"));
   primus.plugin("responder", require("@cocalc/primus-responder"));
+  
+  initWebsocketApi(primus);
 
   const router = Router();
   const library: string = primus.library();
-  // See note above.
-  //UglifyJS.minify(primus.library()).code;
 
   router.get("/.smc/primus.js", (_, res) => {
     logger.debug("serving up primus.js to a specific client");
diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index 95693621ca..51fe5ca968 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -15,22 +15,15 @@
   "bin": {
     "cocalc-compute-start": "./bin/start.js"
   },
-  "files": [
-    "dist/**",
-    "bin/**",
-    "README.md",
-    "package.json"
-  ],
+  "files": ["dist/**", "bin/**", "README.md", "package.json"],
   "author": "SageMath, Inc.",
-  "keywords": [
-    "cocalc",
-    "jupyter"
-  ],
+  "keywords": ["cocalc", "jupyter"],
   "license": "SEE LICENSE.md",
   "dependencies": {
     "@cocalc/api-client": "workspace:*",
     "@cocalc/backend": "workspace:*",
     "@cocalc/compute": "link:",
+    "@cocalc/comm": "workspace:*",
     "@cocalc/jupyter": "workspace:*",
     "@cocalc/primus-multiplex": "^1.1.0",
     "@cocalc/primus-responder": "^1.0.5",
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index 4ab726b838..b60e77f3f4 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -14,6 +14,9 @@ importers:
       '@cocalc/backend':
         specifier: workspace:*
         version: link:../backend
+      '@cocalc/comm':
+        specifier: workspace:*
+        version: link:../comm
       '@cocalc/compute':
         specifier: 'link:'
         version: 'link:'
diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts
index ed9712c5bf..53ca9d29a1 100644
--- a/src/packages/frontend/project/websocket/api.ts
+++ b/src/packages/frontend/project/websocket/api.ts
@@ -52,7 +52,7 @@ export class API {
     try {
       this.cachedVersion = await this.call({ cmd: "version" }, 15000);
     } catch (err) {
-      if (err.message.includes('command "version" not implemented')) {
+      if (err.message?.includes('command "version" not implemented')) {
         this.cachedVersion = 0;
       } else {
         throw err;
diff --git a/src/packages/sync-client/lib/api.ts b/src/packages/sync-client/lib/api.ts
index d3823e74e3..86721528fd 100644
--- a/src/packages/sync-client/lib/api.ts
+++ b/src/packages/sync-client/lib/api.ts
@@ -32,7 +32,7 @@ export default class API implements API_Interface {
     try {
       this.cachedVersion = await this.call({ cmd: "version" }, 15000);
     } catch (err) {
-      if (err.message.includes('command "version" not implemented')) {
+      if (err.message?.includes('command "version" not implemented')) {
         this.cachedVersion = 0;
       } else {
         throw err;

From abd8a18a582fb1177d60c7676d75665d49617ebf Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 04:21:36 +0000
Subject: [PATCH 08/35] listing and exec api calls

---
 src/compute/compute/lib/http-server/index.ts  |  2 +-
 .../compute/lib/http-server/websocket-api.ts  | 19 +++++++++++++++----
 .../compute/lib/http-server/websocket.ts      | 19 ++++++++++++-------
 src/compute/compute/lib/manager.ts            |  2 +-
 src/packages/backend/execute-code.ts          | 11 ++++++++++-
 src/packages/sync-fs/lib/index.ts             | 16 +++++-----------
 src/packages/util/types/execute-code.ts       |  2 ++
 7 files changed, 46 insertions(+), 25 deletions(-)

diff --git a/src/compute/compute/lib/http-server/index.ts b/src/compute/compute/lib/http-server/index.ts
index 9735e45920..d469a1eb5f 100644
--- a/src/compute/compute/lib/http-server/index.ts
+++ b/src/compute/compute/lib/http-server/index.ts
@@ -35,7 +35,7 @@ export function initHttpServer({
   const projectBase = `/${manager.project_id}/raw/`;
   logger.info({ projectBase });
 
-  app.use(projectBase, initWebsocket(server, projectBase));
+  app.use(projectBase, initWebsocket({ server, projectBase, manager }));
 
   // CRITICAL: compression must be after websocket above!
   app.use(compression());
diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index 03b4a5380a..28de6803a7 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -9,11 +9,13 @@
 import { getLogger } from "../logger";
 import { version } from "@cocalc/util/smc-version";
 import type { Mesg } from "@cocalc/comm/websocket/types";
+import getListing from "@cocalc/backend/get-listing";
+import { executeCode } from "@cocalc/backend/execute-code";
 
 const log = getLogger("websocket-api");
 
 let primus: any = undefined;
-export function initWebsocketApi(primus0): void {
+export function initWebsocketApi({ primus: primus0, manager }): void {
   primus = primus0;
 
   primus.on("connection", function (spark) {
@@ -23,7 +25,7 @@ export function initWebsocketApi(primus0): void {
       log.debug("primus-api", "request", data, "REQUEST");
       const t0 = Date.now();
       try {
-        const resp = await handleApiCall(data, spark);
+        const resp = await handleApiCall(data, spark, manager);
         done(resp);
       } catch (err) {
         // console.trace(); log.debug("primus-api error stacktrack", err.stack, err);
@@ -46,13 +48,22 @@ export function initWebsocketApi(primus0): void {
   });
 }
 
-async function handleApiCall(data: Mesg, _spark): Promise<any> {
+async function handleApiCall(data: Mesg, _spark, manager): Promise<any> {
   switch (data.cmd) {
     case "version":
       return version;
     case "listing":
       // see packages/sync-fs/lib/index.ts
-      throw Error("todo");
+      return await getListing(data.path, data.hidden, manager.home);
+    case "exec":
+      if (data.opts == null) {
+        throw Error("opts must not be null");
+      }
+      return await executeCode({
+        ...data.opts,
+        home: manager.home,
+        ccNewFile: true,
+      });
     default:
       throw Error(`command "${(data as any).cmd}" not implemented`);
   }
diff --git a/src/compute/compute/lib/http-server/websocket.ts b/src/compute/compute/lib/http-server/websocket.ts
index 3d40797a00..ec8c53c166 100644
--- a/src/compute/compute/lib/http-server/websocket.ts
+++ b/src/compute/compute/lib/http-server/websocket.ts
@@ -19,12 +19,17 @@ import { getLogger } from "../logger";
 
 const logger = getLogger("websocket");
 
-export default function initWebsocket(
-  server: Server,
-  basePath: string,
-): Router {
+export default function initWebsocket({
+  server,
+  projectBase,
+  manager,
+}: {
+  server: Server;
+  projectBase: string;
+  manager;
+}): Router {
   const opts = {
-    pathname: join(basePath, ".smc", "ws"),
+    pathname: join(projectBase, ".smc", "ws"),
     transformer: "websockets",
   } as const;
   logger.debug(`Initializing primus websocket server at "${opts.pathname}"...`);
@@ -33,8 +38,8 @@ export default function initWebsocket(
   // add multiplex to Primus so we have channels.
   primus.plugin("multiplex", require("@cocalc/primus-multiplex"));
   primus.plugin("responder", require("@cocalc/primus-responder"));
-  
-  initWebsocketApi(primus);
+
+  initWebsocketApi({ primus, manager });
 
   const router = Router();
   const library: string = primus.library();
diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts
index 227c476b49..c1956928f2 100644
--- a/src/compute/compute/lib/manager.ts
+++ b/src/compute/compute/lib/manager.ts
@@ -83,7 +83,7 @@ export class Manager {
   private state: "new" | "init" | "ready" = "new";
   private sync_db;
   public project_id: string;
-  private home: string;
+  public home: string;
   private host?: string;
   private port?: number;
   private waitHomeFilesystemType?: string;
diff --git a/src/packages/backend/execute-code.ts b/src/packages/backend/execute-code.ts
index f1208eee1e..e74eccb6bd 100644
--- a/src/packages/backend/execute-code.ts
+++ b/src/packages/backend/execute-code.ts
@@ -17,7 +17,6 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { EventEmitter } from "node:stream";
 import shellEscape from "shell-escape";
-
 import getLogger from "@cocalc/backend/logger";
 import { envToInt } from "@cocalc/backend/misc/env-to-number";
 import { aggregate } from "@cocalc/util/aggregate";
@@ -37,6 +36,7 @@ import {
 import { Processes } from "@cocalc/util/types/project-info/types";
 import { envForSpawn } from "./misc";
 import { ProcessStats } from "./process-stats";
+import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists";
 
 const log = getLogger("execute-code");
 
@@ -143,6 +143,15 @@ async function executeCodeNoAggregate(
     } else {
       throw new Error(`Async operation '${key}' does not exist.`);
     }
+  } else if (opts.ccNewFile && opts.command == "cc-new-file") {
+    // so we don't have to depend on having our cc-new-file script
+    // installed.  We just don't support templates on compute server.
+    for (const path of opts.args ?? []) {
+      const target = join(opts.home ?? process.env.HOME ?? "", path);
+      await ensureContainingDirectoryExists(target);
+      await writeFile(target, "");
+    }
+    return { exit_code: 0, stdout: "", stderr: "", type: "blocking" };
   }
 
   opts.args ??= [];
diff --git a/src/packages/sync-fs/lib/index.ts b/src/packages/sync-fs/lib/index.ts
index 75af6f3858..2083aacd2d 100644
--- a/src/packages/sync-fs/lib/index.ts
+++ b/src/packages/sync-fs/lib/index.ts
@@ -318,17 +318,11 @@ class SyncFS {
         return await getListing(data.path, data.hidden, this.mount);
 
       case "exec":
-        if (data.opts.command == "cc-new-file") {
-          // so we don't have to depend on having our cc-new-file script
-          // installed.  We just don't support templates on compute server.
-          for (const path of data.opts.args ?? []) {
-            const target = join(this.mount, path);
-            await ensureContainingDirectoryExists(target);
-            await writeFile(target, "");
-          }
-          return { status: 0, stdout: "", stderr: "" };
-        }
-        return await executeCode({ ...data.opts, home: this.mount });
+        return await executeCode({
+          ...data.opts,
+          home: this.mount,
+          ccNewFile: true,
+        });
 
       case "delete_files":
         return await delete_files(data.paths, this.mount);
diff --git a/src/packages/util/types/execute-code.ts b/src/packages/util/types/execute-code.ts
index d6c3085250..1b59b95b0a 100644
--- a/src/packages/util/types/execute-code.ts
+++ b/src/packages/util/types/execute-code.ts
@@ -48,6 +48,8 @@ export interface ExecuteCodeOptions {
   aggregate?: string | number; // if given, aggregates multiple calls with same sequence number into one -- see @cocalc/util/aggregate; typically make this a timestamp for compiling code (e.g., latex).
   verbose?: boolean; // default true -- impacts amount of logging
   async_call?: boolean; // default false -- if true, return right after the process started (to get the PID) or when it fails.
+  // if ccNewFile is true, cc-new-file [args...] makes blank files if they don't exist (no template). ONLY supported for async version of executeCode (not callback)
+  ccNewFile?: boolean;
 }
 
 export interface ExecuteCodeOptionsAsyncGet {

From af27be73b2cdd413044e40349e5cbddba31005cc Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 04:34:58 +0000
Subject: [PATCH 09/35] compute websocket server: more api calls; list all and
 include query

---
 .../compute/lib/http-server/websocket-api.ts  | 45 +++++++++++++++++++
 src/compute/compute/lib/manager.ts            |  2 +-
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index 28de6803a7..57665e93b5 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -11,6 +11,7 @@ import { version } from "@cocalc/util/smc-version";
 import type { Mesg } from "@cocalc/comm/websocket/types";
 import getListing from "@cocalc/backend/get-listing";
 import { executeCode } from "@cocalc/backend/execute-code";
+import { callback2 } from "@cocalc/util/async-utils";
 
 const log = getLogger("websocket-api");
 
@@ -55,6 +56,19 @@ async function handleApiCall(data: Mesg, _spark, manager): Promise<any> {
     case "listing":
       // see packages/sync-fs/lib/index.ts
       return await getListing(data.path, data.hidden, manager.home);
+
+    // TODO
+    case "delete_files":
+    case "move_files":
+    case "rename_file":
+    case "canonical_paths":
+    case "configuration":
+    case "prettier": // deprecated
+    case "formatter":
+    case "prettier_string": // deprecated
+    case "formatter_string":
+      throw Error(`command "${(data as any).cmd}" not implemented`);
+
     case "exec":
       if (data.opts == null) {
         throw Error("opts must not be null");
@@ -64,6 +78,37 @@ async function handleApiCall(data: Mesg, _spark, manager): Promise<any> {
         home: manager.home,
         ccNewFile: true,
       });
+
+    case "query":
+      if (data.opts?.changes) {
+        throw Error("changefeeds are not supported for api queries");
+      }
+      return await callback2(
+        manager.client.query.bind(manager.client),
+        data.opts,
+      );
+
+    // TODO
+    case "eval_code":
+    case "terminal":
+    case "lean":
+    case "jupyter_strip_notebook":
+    case "jupyter_nbconvert":
+    case "jupyter_run_notebook":
+    case "lean_channel":
+    case "x11_channel":
+    case "synctable_channel":
+    case "syncdoc_call":
+    case "symmetric_channel":
+    case "realpath":
+    case "project_info":
+    case "compute_filesystem_cache":
+    case "sync_fs":
+    case "compute_server_sync_register":
+    case "compute_server_compute_register":
+    case "compute_server_sync_request":
+    case "copy_from_project_to_compute_server":
+    case "copy_from_compute_server_to_project":
     default:
       throw Error(`command "${(data as any).cmd}" not implemented`);
   }
diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts
index c1956928f2..3f8b45453d 100644
--- a/src/compute/compute/lib/manager.ts
+++ b/src/compute/compute/lib/manager.ts
@@ -90,7 +90,7 @@ export class Manager {
   public compute_server_id: number;
   private connections: { [path: string]: any } = {};
   private websocket;
-  private client;
+  public client;
 
   constructor({
     project_id,

From a24b29de587d7a95942d952a6b5cb50b3b18face Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 04:38:25 +0000
Subject: [PATCH 10/35] fix typescript error

---
 src/packages/sync-fs/lib/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/packages/sync-fs/lib/index.ts b/src/packages/sync-fs/lib/index.ts
index 2083aacd2d..0235cecc12 100644
--- a/src/packages/sync-fs/lib/index.ts
+++ b/src/packages/sync-fs/lib/index.ts
@@ -28,7 +28,6 @@ import { executeCode } from "@cocalc/backend/execute-code";
 import { delete_files } from "@cocalc/backend/files/delete-files";
 import { move_files } from "@cocalc/backend/files/move-files";
 import { rename_file } from "@cocalc/backend/files/rename-file";
-import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists";
 
 const EXPLICIT_HIDDEN_EXCLUDES = [".cache", ".local"];
 

From 49405b0af981ee4e9dbe62d57fe3bb159eee7308 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 16:02:07 +0000
Subject: [PATCH 11/35] jupyter.html no longer needed (deprecated)

---
 src/packages/frontend/jupyter.html | 163 -----------------------------
 1 file changed, 163 deletions(-)
 delete mode 100644 src/packages/frontend/jupyter.html

diff --git a/src/packages/frontend/jupyter.html b/src/packages/frontend/jupyter.html
deleted file mode 100644
index c9c366ecd1..0000000000
--- a/src/packages/frontend/jupyter.html
+++ /dev/null
@@ -1,163 +0,0 @@
-<!--
-  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
-  License: MS-RSL – see LICENSE.md for details
- -->
-
-<div class="smc-jupyter-templates hide">
-    <div class="smc-jupyter-cursor"><pre class="smc-jupyter-cursor-label"></pre><div class="smc-jupyter-cursor-inside">&nbsp;&nbsp;&nbsp;</div></div>
-
-    <div class="smc-jupyter-notebook smc-vfill">
-        <div class="smc-jupyter-startup-message alert alert-info" role='alert' style="width: 94%; margin: 3%;">
-            Loading...
-        </div>
-        <div class="hide smc-jupyter-notebook-buttons smc-jupyter-notebook-buttons-hidden-xs hidden-xs">
-            <span class="smc-jupyter-notebook-status-messages lighten"> </span>
-            <span class="hide smc-jupyter-notebook-danger">DANGER: Users on this VM could connect unless you stop your ipython-notebook server (they would have to know secret internal project-id).</span>
-            <span class="btn-group">
-                <a href="#undo"
-                   class="btn btn-default webapp-editor-write-only"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Undo">
-                    <i class="fa fa-undo"></i>
-                </a>
-                <a href="#redo"
-                   class="btn btn-default webapp-editor-write-only"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Redo">
-                    <i class="fa fa-repeat"></i>
-                </a>
-                <a href="#font-size-decrease"
-                   class="btn btn-default"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Decrease font-size">
-                    <i class="fa fa-font" style="font-size:7pt"></i>
-                </a>
-                <a href="#font-size-increase"
-                   class="btn btn-default"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Increase font-size">
-                    <i class="fa fa-font" style="font-size:11pt"></i>
-                </a>
-            </span>
-            <span class="btn-group">
-                <a href="#save" class="btn btn-success"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Save .ipynb file to disk (file is constantly sync'd with server).">
-                  <i class="fa fa-save primary-icon"></i>
-                  <i class="fa fa-cocalc-ring hide"></i>
-                  <span class="hidden-sm">Save</span>
-                </a>
-                <a href="#history" class="btn btn-info webapp-editor-history-button"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="View the history of this file.">
-                       <i class="fa fa-history"></i> <span class="hidden-sm">TimeTravel</span>
-                </a>
-                <a href="#publish" class="btn btn-default"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Publish this notebook for anybody to see">
-                     <i class="fa fa-refresh fa-spin hide"></i>
-                     <i class="fa fa-share-square"></i>
-                     <span class="hidden-sm">Publish</span>
-                </a>
-                <a href="#info" class="btn btn-default"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Extra information about the Jupyter notebook">
-                       <i class="fa fa-question-circle"></i> <span class="hidden-sm">About</span>
-                </a>
-                <a href="#refresh" class="btn btn-default"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Reload the notebook">
-                       <i class="fa fa-refresh"></i> <span class="hidden-sm">Refresh</span>
-                </a>
-                <a href="#modern" class="btn btn-default"
-                   data-toggle="tooltip" data-placement="bottom"
-                   title="Reload the notebook">
-                       <i class="fa fa-exchange"></i> <span class="hidden-sm">CoCalc Jupyter Notebook...</span>
-                </a>
-            </span>
-        </div>
-
-        <div class="smc-jupyter-notebook-buttons visible-xs">
-            <span class="btn-group">
-                <a href="#save" class="btn btn-success">
-                  <i class="fa fa-save primary-icon"></i>
-                  <i class="fa fa-cocalc-ring hide"></i>
-                  Save
-                </a>
-                <a href="#publish" class="btn btn-default"> <i class="fa fa-share-square"></i> Publish</a>
-                <a href="#info" class="btn  btn-default"> <i class="fa fa-info-circle"></i> About</a>
-            </span>
-            <span class="btn-group webapp-editor-codemirror-worksheet-buttons hide">
-                <a href="#execute" class="btn btn-success btn-lg"> <i class="fa fa-play"></i>
-                    <span>Run</span>
-                </a>
-                <a href="#interrupt" class="btn btn-warning btn-lg"> <i class="fa fa-stop"></i>
-                    <span>Stop</span>
-                </a>
-                <a href="#tab" class="btn btn-default btn-lg"> <i class="fa fa-info-circle"></i>
-                    <span>Tab</span>
-                </a>
-            </span>
-        </div>
-
-        <h3 class="smc-jupyter-notebook-connecting hide" style="margin-left:2em">Opening...</h3>
-        <div class="smc-jupyter-notebook-notebook smc-vfill"></div>
-    </div>
-
-
-    <!-- Jupyter Notebook Viewer -->
-    <div class="smc-jupyter-nbviewer" style="height:100%">
-        <div style="margin-left:1em">
-            <span>
-                <a href="#copy" class="btn btn-default btn-sm"><i class="fa fa-paper-plane"></i> Copy Jupyter Notebook to your Project...</a>
-            </span>
-            <span>
-                <a href="#download" class="btn btn-default btn-sm"><i class="fa fa-cloud-download"></i> Download</a>
-            </span>
-        </div>
-        <div class="smc-jupyter-nbviewer-content"  style="height:100%">
-            <iframe style="width:100%;height:100%;border:1px solid lightgray; border-radius:4px;" frameborder="0" scrolling="yes">
-            </iframe>
-        </div>
-    </div>
-
-
-    <div class="modal smc-jupyter-publish-dialog" data-backdrop="static" tabindex="-1" role="dialog" aria-hidden="true">
-        <div class="modal-dialog">
-            <div class="modal-content">
-                <div class="modal-header">
-                    <button type="button" class="close btn-close" aria-hidden="true">
-                        <span style="font-size:20pt;">×</span>
-                    </button>
-                    <h3><i class="fa fa-share-square"> </i>  Publishing IPython Notebook</h3>
-                </div>
-                <div class="well">
-                    <div class="smc-jupyter-publish-dialog-publishing">Publishing...
-                        <span class="smc-jupyter-publish-status"></span>
-                        <br>
-                        <div class="progress">
-                            <div class="progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0"
-                                 aria-valuemax="100" style="width: 0%;">
-                                0%
-                            </div>
-                        </div>
-                    </div>
-                    <div class="smc-jupyter-publish-dialog-fail hide"><b>Error:</b> <span></span></div>
-                    <div class="smc-jupyter-publish-dialog-success hide"><br>Share the following link:
-                        <br><br>
-                        <input class="form-control smc-jupyter-publish-url"><br>
-                        <span class="lighten">Note: Anybody may read this public notebook and only collaborators may change it.<br/>The published notebook does not update automatically. You have to click publish again after saving it.</span>
-                    </div>
-
-                </div>
-                <div class="modal-footer">
-                    <button class="btn btn-close pull-left btn-default btn-lg">Close</button>
-                </div>
-            </div>
-        </div>
-    </div>
-
-
-
-
-</div>

From 42899b4552af5cd031229c0afabe10269989291c Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 16:02:56 +0000
Subject: [PATCH 12/35] fix version consistency issue

---
 src/compute/compute/package.json |  9 ++++----
 src/compute/pnpm-lock.yaml       | 39 +++++++++++++++++---------------
 2 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index 51fe5ca968..758ea3e271 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -33,14 +33,11 @@
     "@cocalc/sync-fs": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
-    "@types/compression": "^1.7.5",
-    "@types/ms": "^0.7.34",
-    "@types/ws": "^8.5.9",
     "awaiting": "^3.0.0",
     "compression": "^1.7.4",
     "cookie": "^1.0.0",
     "debug": "^4.3.2",
-    "express": "^4.20.0",
+    "express": "^4.21.1",
     "primus": "^8.0.7",
     "websocketfs": "^0.17.4",
     "ws": "^8.18.0"
@@ -51,8 +48,12 @@
     "url": "https://github.com/sagemathinc/cocalc"
   },
   "devDependencies": {
+    "@types/compression": "^1.7.5",
     "@types/cookie": "^0.6.0",
+    "@types/express": "^4.17.21",
+    "@types/ms": "^0.7.34",
     "@types/node": "^18.16.14",
+    "@types/ws": "^8.5.9",
     "typescript": "^5.6.3"
   }
 }
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index b60e77f3f4..2e8b589b93 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -47,15 +47,6 @@ importers:
       '@cocalc/util':
         specifier: workspace:*
         version: link:../util
-      '@types/compression':
-        specifier: ^1.7.5
-        version: 1.7.5
-      '@types/ms':
-        specifier: ^0.7.34
-        version: 0.7.34
-      '@types/ws':
-        specifier: ^8.5.9
-        version: 8.5.9
       awaiting:
         specifier: ^3.0.0
         version: 3.0.0
@@ -69,7 +60,7 @@ importers:
         specifier: ^4.3.2
         version: 4.3.4
       express:
-        specifier: ^4.20.0
+        specifier: ^4.21.1
         version: 4.21.1
       primus:
         specifier: ^8.0.7
@@ -81,12 +72,24 @@ importers:
         specifier: ^8.18.0
         version: 8.18.0
     devDependencies:
+      '@types/compression':
+        specifier: ^1.7.5
+        version: 1.7.5
       '@types/cookie':
         specifier: ^0.6.0
         version: 0.6.0
+      '@types/express':
+        specifier: ^4.17.21
+        version: 4.17.21
+      '@types/ms':
+        specifier: ^0.7.34
+        version: 0.7.34
       '@types/node':
         specifier: ^18.16.14
         version: 18.16.14
+      '@types/ws':
+        specifier: ^8.5.9
+        version: 8.5.9
       typescript:
         specifier: ^5.6.3
         version: 5.6.3
@@ -120,11 +123,11 @@ packages:
   '@types/cookie@0.6.0':
     resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==}
 
-  '@types/express-serve-static-core@5.0.0':
-    resolution: {integrity: sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==}
+  '@types/express-serve-static-core@4.19.6':
+    resolution: {integrity: sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==}
 
-  '@types/express@5.0.0':
-    resolution: {integrity: sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==}
+  '@types/express@4.17.21':
+    resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==}
 
   '@types/http-errors@2.0.4':
     resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==}
@@ -772,7 +775,7 @@ snapshots:
 
   '@types/compression@1.7.5':
     dependencies:
-      '@types/express': 5.0.0
+      '@types/express': 4.17.21
 
   '@types/connect@3.4.38':
     dependencies:
@@ -780,17 +783,17 @@ snapshots:
 
   '@types/cookie@0.6.0': {}
 
-  '@types/express-serve-static-core@5.0.0':
+  '@types/express-serve-static-core@4.19.6':
     dependencies:
       '@types/node': 18.16.14
       '@types/qs': 6.9.16
       '@types/range-parser': 1.2.7
       '@types/send': 0.17.4
 
-  '@types/express@5.0.0':
+  '@types/express@4.17.21':
     dependencies:
       '@types/body-parser': 1.19.5
-      '@types/express-serve-static-core': 5.0.0
+      '@types/express-serve-static-core': 4.19.6
       '@types/qs': 6.9.16
       '@types/serve-static': 1.15.7
 

From 47d13f7020c5ad0855eb71a1a9b2a28e62cdb6b7 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 18:45:55 +0000
Subject: [PATCH 13/35] http compute server ui - work in progress...

---
 src/compute/compute/lib/http-server/index.ts  |  2 +-
 .../compute/lib/http-server/websocket-api.ts  | 31 +++++++++++++------
 .../eval-code.ts                              |  0
 .../browser-websocket => backend}/realpath.ts | 12 ++++---
 src/packages/frontend/app/active-content.tsx  | 13 ++++++--
 src/packages/frontend/app/page.tsx            |  8 ++++-
 src/packages/frontend/compute/entry-point.ts  |  2 +-
 src/packages/project/browser-websocket/api.ts |  4 +--
 8 files changed, 52 insertions(+), 20 deletions(-)
 rename src/packages/{project/browser-websocket => backend}/eval-code.ts (100%)
 rename src/packages/{project/browser-websocket => backend}/realpath.ts (73%)

diff --git a/src/compute/compute/lib/http-server/index.ts b/src/compute/compute/lib/http-server/index.ts
index d469a1eb5f..7757744fe8 100644
--- a/src/compute/compute/lib/http-server/index.ts
+++ b/src/compute/compute/lib/http-server/index.ts
@@ -48,7 +48,7 @@ export function initHttpServer({
   });
 
   app.get("/settings", (_req, res) => {
-    res.send(`<h1><a href="${join("/static", ENTRY_POINT)}">CoCalc App</a>`);
+    res.redirect(join("/static", ENTRY_POINT));
   });
 
   app.use(
diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index 57665e93b5..4b591ad31c 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -12,13 +12,13 @@ import type { Mesg } from "@cocalc/comm/websocket/types";
 import getListing from "@cocalc/backend/get-listing";
 import { executeCode } from "@cocalc/backend/execute-code";
 import { callback2 } from "@cocalc/util/async-utils";
+import { terminal } from "@cocalc/terminal";
+import realpath from "@cocalc/backend/realpath";
+import { eval_code } from "@cocalc/backend/eval-code";
 
 const log = getLogger("websocket-api");
 
-let primus: any = undefined;
-export function initWebsocketApi({ primus: primus0, manager }): void {
-  primus = primus0;
-
+export function initWebsocketApi({ primus: primus, manager }): void {
   primus.on("connection", function (spark) {
     log.debug(`new connection from ${spark.address.ip} -- ${spark.id}`);
 
@@ -26,7 +26,7 @@ export function initWebsocketApi({ primus: primus0, manager }): void {
       log.debug("primus-api", "request", data, "REQUEST");
       const t0 = Date.now();
       try {
-        const resp = await handleApiCall(data, spark, manager);
+        const resp = await handleApiCall(data, spark, manager, primus);
         done(resp);
       } catch (err) {
         // console.trace(); log.debug("primus-api error stacktrack", err.stack, err);
@@ -49,7 +49,12 @@ export function initWebsocketApi({ primus: primus0, manager }): void {
   });
 }
 
-async function handleApiCall(data: Mesg, _spark, manager): Promise<any> {
+async function handleApiCall(
+  data: Mesg,
+  _spark,
+  manager,
+  primus,
+): Promise<any> {
   switch (data.cmd) {
     case "version":
       return version;
@@ -88,9 +93,18 @@ async function handleApiCall(data: Mesg, _spark, manager): Promise<any> {
         data.opts,
       );
 
-    // TODO
-    case "eval_code":
     case "terminal":
+      // this might work but be TOTALLY WRONG (?)... or require
+      // some thought about who "hosts" the terminal.
+      return await terminal(primus, data.path, data.options);
+
+    case "eval_code":
+      return await eval_code(data.code);
+
+    case "realpath":
+      return realpath(data.path, manager.home);
+
+    // TODO
     case "lean":
     case "jupyter_strip_notebook":
     case "jupyter_nbconvert":
@@ -100,7 +114,6 @@ async function handleApiCall(data: Mesg, _spark, manager): Promise<any> {
     case "synctable_channel":
     case "syncdoc_call":
     case "symmetric_channel":
-    case "realpath":
     case "project_info":
     case "compute_filesystem_cache":
     case "sync_fs":
diff --git a/src/packages/project/browser-websocket/eval-code.ts b/src/packages/backend/eval-code.ts
similarity index 100%
rename from src/packages/project/browser-websocket/eval-code.ts
rename to src/packages/backend/eval-code.ts
diff --git a/src/packages/project/browser-websocket/realpath.ts b/src/packages/backend/realpath.ts
similarity index 73%
rename from src/packages/project/browser-websocket/realpath.ts
rename to src/packages/backend/realpath.ts
index 176b5d8b94..7a819e206c 100644
--- a/src/packages/project/browser-websocket/realpath.ts
+++ b/src/packages/backend/realpath.ts
@@ -14,11 +14,15 @@ import { realpath as fs_realpath } from "node:fs/promises";
 const HOME: string =
   process.env.SMC_LOCAL_HUB_HOME ?? process.env.HOME ?? "/home/user";
 
-export async function realpath(path: string): Promise<string> {
-  const fullpath = HOME + "/" + path;
+export default async function realpath(
+  path: string,
+  home?: string,
+): Promise<string> {
+  home = home ?? HOME;
+  const fullpath = home + "/" + path;
   const rpath = await fs_realpath(fullpath);
-  if (rpath == fullpath || !rpath.startsWith(HOME + "/")) {
+  if (rpath == fullpath || !rpath.startsWith(home + "/")) {
     return path;
   }
-  return rpath.slice((HOME + "/").length);
+  return rpath.slice((home + "/").length);
 }
diff --git a/src/packages/frontend/app/active-content.tsx b/src/packages/frontend/app/active-content.tsx
index 654ce667d8..34c2c77845 100644
--- a/src/packages/frontend/app/active-content.tsx
+++ b/src/packages/frontend/app/active-content.tsx
@@ -8,7 +8,9 @@ import { AdminPage } from "@cocalc/frontend/admin";
 import { Alert } from "@cocalc/frontend/antd-bootstrap";
 import {
   CSS,
+  entryPoint,
   React,
+  redux,
   useActions,
   useTypedRedux,
 } from "@cocalc/frontend/app-framework";
@@ -64,10 +66,17 @@ export const ActiveContent: React.FC = React.memo(() => {
     v.push(
       <div key={project_id} className={cls}>
         {x}
-      </div>
+      </div>,
     );
   });
 
+  if (entryPoint == "compute") {
+    const project_id = redux
+      .getStore("customize")
+      .getIn(["compute_server", "project_id"]) as string;
+    return <ProjectPage project_id={project_id} is_active={true} />;
+  }
+
   if (get_api_key) {
     // Only render the account page which has the message for allowing api access:
     return <AccountPage key={"account"} />;
@@ -100,7 +109,7 @@ export const ActiveContent: React.FC = React.memo(() => {
             </A>
             .
           </Alert>
-        </div>
+        </div>,
       );
     }
   }
diff --git a/src/packages/frontend/app/page.tsx b/src/packages/frontend/app/page.tsx
index 3572b67fde..626633ce2b 100644
--- a/src/packages/frontend/app/page.tsx
+++ b/src/packages/frontend/app/page.tsx
@@ -16,6 +16,7 @@ import { alert_message } from "@cocalc/frontend/alerts";
 import { Button } from "@cocalc/frontend/antd-bootstrap";
 import {
   CSS,
+  entryPoint,
   React,
   useActions,
   useEffect,
@@ -200,7 +201,12 @@ export const Page: React.FC = () => {
   }
 
   function render_sign_in_tab(): JSX.Element | null {
-    if (is_logged_in) return null;
+    if (is_logged_in) {
+      return null;
+    }
+    if (entryPoint == "compute") {
+      return null;
+    }
 
     let style: CSS | undefined = undefined;
     if (active_top_tab !== "account") {
diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts
index 291d1a0154..854b90e815 100644
--- a/src/packages/frontend/compute/entry-point.ts
+++ b/src/packages/frontend/compute/entry-point.ts
@@ -27,8 +27,8 @@ export async function init() {
   initProjects();
   initMarkdown();
   initLast();
-  initEntryPointState();
   try {
+    await initEntryPointState();
     await render();
   } finally {
     // don't insert the crash banner until the main app has rendered,
diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts
index bf45024e51..822646ca55 100644
--- a/src/packages/project/browser-websocket/api.ts
+++ b/src/packages/project/browser-websocket/api.ts
@@ -27,11 +27,11 @@ import { terminal } from "@cocalc/terminal";
 import { x11_channel } from "../x11/server";
 import { canonical_paths } from "./canonical-path";
 import { delete_files } from "@cocalc/backend/files/delete-files";
-import { eval_code } from "./eval-code";
+import { eval_code } from "@cocalc/backend/eval-code";
 import computeFilesystemCache from "./compute-filesystem-cache";
 import { move_files } from "@cocalc/backend/files/move-files";
 import { rename_file } from "@cocalc/backend/files/rename-file";
-import { realpath } from "./realpath";
+import realpath from "@cocalc/backend/realpath";
 import { project_info_ws } from "../project-info";
 import query from "./query";
 import { browser_symmetric_channel } from "./symmetric_channel";

From f6d2743cb322d68a74bef9def26b0e178606c684 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 21:26:35 +0000
Subject: [PATCH 14/35] compute-http-server: adding basic api/v2/
 infrastructure

---
 .../compute/lib/http-server/http-next-api.ts  | 52 +++++++++++++++++++
 src/compute/compute/lib/http-server/index.ts  |  3 ++
 .../compute/lib/http-server/websocket-api.ts  |  2 +-
 src/packages/frontend/compute/entry-point.ts  | 20 +++----
 .../frontend/jupyter/use-kernels-info.ts      |  1 -
 .../frontend/project/new/jupyter-buttons.tsx  |  5 +-
 .../page/home-page/ai-generate-document.tsx   |  1 -
 .../project/page/project-status-hook.ts       |  4 +-
 src/packages/frontend/user-tracking.ts        |  8 ++-
 9 files changed, 79 insertions(+), 17 deletions(-)
 create mode 100644 src/compute/compute/lib/http-server/http-next-api.ts

diff --git a/src/compute/compute/lib/http-server/http-next-api.ts b/src/compute/compute/lib/http-server/http-next-api.ts
new file mode 100644
index 0000000000..05590347fd
--- /dev/null
+++ b/src/compute/compute/lib/http-server/http-next-api.ts
@@ -0,0 +1,52 @@
+/*
+HTTP Next API
+
+- this is whatever we need from cocalc/src/packages/next/pages/api/v2 specifically
+  for working with one project.
+
+*/
+
+import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data";
+import { Router } from "express";
+import { getLogger } from "../logger";
+import type { Manager } from "../manager";
+
+const logger = getLogger("http-next-api");
+
+export default function initHttpNextApi({ manager }: { manager }): Router {
+  logger.info("initHttpNextApi");
+  const router = Router();
+
+  for (const path in HANDLERS) {
+    router.post("/" + path, handler(manager, HANDLERS[path]));
+    router.get("/" + path, handler(manager, HANDLERS[path]));
+  }
+
+  router.post("*", (req, res) => {
+    res.json({ error: `api endpoint '${req.path}' is not implemented` });
+  });
+  router.get("*", (req, res) => {
+    res.json({ error: `api endpoint '${req.path}' is not implemented` });
+  });
+
+  return router;
+}
+
+function handler(
+  manager,
+  f: (x: { req; res; manager: Manager }) => Promise<object>,
+) {
+  return async (req, res) => {
+    try {
+      res.json({ success: true, ...(await f({ req, res, manager })) });
+    } catch (err) {
+      res.json({ error: `${err}` });
+    }
+  };
+}
+
+const HANDLERS = {
+  "jupyter/kernels": async () => {
+    return { kernels: await get_kernel_data() };
+  },
+};
diff --git a/src/compute/compute/lib/http-server/index.ts b/src/compute/compute/lib/http-server/index.ts
index 7757744fe8..e0223a6c1f 100644
--- a/src/compute/compute/lib/http-server/index.ts
+++ b/src/compute/compute/lib/http-server/index.ts
@@ -12,6 +12,7 @@ import { path as STATIC_PATH } from "@cocalc/static";
 import { join } from "path";
 import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
 import initWebsocket from "./websocket";
+import initHttpNextApi from "./http-next-api";
 
 const logger = getLogger("http-server");
 
@@ -71,6 +72,8 @@ export function initHttpServer({
     });
   });
 
+  app.use("/api/v2", initHttpNextApi({ manager }));
+
   server.listen(port, host, () => {
     logger.info(`Server listening http://${host}:${port}`);
   });
diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index 4b591ad31c..1b99e0802e 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -18,7 +18,7 @@ import { eval_code } from "@cocalc/backend/eval-code";
 
 const log = getLogger("websocket-api");
 
-export function initWebsocketApi({ primus: primus, manager }): void {
+export function initWebsocketApi({ primus, manager }): void {
   primus.on("connection", function (spark) {
     log.debug(`new connection from ${spark.address.ip} -- ${spark.id}`);
 
diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts
index 854b90e815..588e5535f1 100644
--- a/src/packages/frontend/compute/entry-point.ts
+++ b/src/packages/frontend/compute/entry-point.ts
@@ -9,21 +9,23 @@ Entry point for compute server version of CoCalc...
 
 // Load/initialize Redux-based react functionality
 import "@cocalc/frontend/client/client";
-import { redux, setEntryPoint } from "../app-framework";
-import "../jquery-plugins";
-import { init as initAccount } from "../account";
-import { init as initApp } from "../app/init";
-import { init as initProjects } from "../projects";
-import { init as initMarkdown } from "../markdown/markdown-input/main";
-import { init as initCrashBanner } from "../crash-banner";
+import { redux, setEntryPoint } from "@cocalc/frontend/app-framework";
+import "@cocalc/frontend/jquery-plugins";
+import { init as initAccount } from "@cocalc/frontend/account";
+import { init as initApp } from "@cocalc/frontend/app/init";
+import { init as initProjects } from "@cocalc/frontend/projects";
+import { init as initFileUse } from "@cocalc/frontend/file-use/init";
+import { init as initMarkdown } from "@cocalc/frontend/markdown/markdown-input/main";
+import { init as initCrashBanner } from "@cocalc/frontend/crash-banner";
 import "codemirror";
-import { init as initLast } from "../last";
-import { render } from "../app/render";
+import { init as initLast } from "@cocalc/frontend/last";
+import { render } from "@cocalc/frontend/app/render";
 
 export async function init() {
   setEntryPoint("compute");
   initAccount(redux);
   initApp();
+  initFileUse();
   initProjects();
   initMarkdown();
   initLast();
diff --git a/src/packages/frontend/jupyter/use-kernels-info.ts b/src/packages/frontend/jupyter/use-kernels-info.ts
index 70c5662be6..9b51a3ed9d 100644
--- a/src/packages/frontend/jupyter/use-kernels-info.ts
+++ b/src/packages/frontend/jupyter/use-kernels-info.ts
@@ -6,7 +6,6 @@
 import { fromJS } from "immutable";
 import { useEffect, useMemo, useState } from "react";
 import useAsyncEffect from "use-async-effect";
-
 import { getKernelInfo } from "@cocalc/frontend/components/run-button/kernel-info";
 import { useProjectContext } from "@cocalc/frontend/project/context";
 import {
diff --git a/src/packages/frontend/project/new/jupyter-buttons.tsx b/src/packages/frontend/project/new/jupyter-buttons.tsx
index 2a443f9e43..4f0ae7fa16 100644
--- a/src/packages/frontend/project/new/jupyter-buttons.tsx
+++ b/src/packages/frontend/project/new/jupyter-buttons.tsx
@@ -87,7 +87,6 @@ export function JupyterNotebookButtons({
   // SEE https://github.com/sagemathinc/cocalc/issues/7168
   // Sept 2024: adding "Sage Notebook", as part of deprecating "Sage Worksheet"
   const { error, kernel_selection, kernels_by_name } = useJupyterKernelsInfo();
-
   if (!availableFeatures.jupyter_notebook) {
     return null;
   }
@@ -235,7 +234,9 @@ export function JupyterNotebookButtons({
   }
 
   function renderLanguageSpecificButtons() {
-    if (kernel_selection == null || kernels_by_name == null) return null;
+    if (kernel_selection == null || kernels_by_name == null) {
+      return null;
+    }
 
     const langs = ["sage", "sagemath"] as const;
     const btns: { lang: string; btn: JSX.Element }[] = [];
diff --git a/src/packages/frontend/project/page/home-page/ai-generate-document.tsx b/src/packages/frontend/project/page/home-page/ai-generate-document.tsx
index 74a0743500..991352082c 100644
--- a/src/packages/frontend/project/page/home-page/ai-generate-document.tsx
+++ b/src/packages/frontend/project/page/home-page/ai-generate-document.tsx
@@ -25,7 +25,6 @@ import { delay } from "awaiting";
 import { debounce, isEmpty, throttle } from "lodash";
 import { useEffect, useRef, useState } from "react";
 import { FormattedMessage, useIntl } from "react-intl";
-
 import { useLanguageModelSetting } from "@cocalc/frontend/account/useLanguageModelSetting";
 import {
   CSS,
diff --git a/src/packages/frontend/project/page/project-status-hook.ts b/src/packages/frontend/project/page/project-status-hook.ts
index be13c434cd..56d8e4c296 100644
--- a/src/packages/frontend/project/page/project-status-hook.ts
+++ b/src/packages/frontend/project/page/project-status-hook.ts
@@ -23,7 +23,9 @@ export function useProjectStatus(actions?: ProjectActions): void {
   }
 
   function connect() {
-    if (project_id == null) return;
+    if (project_id == null) {
+      return;
+    }
     const status_sync = webapp_client.project_client.project_status(project_id);
     statusRef.current = status_sync;
     const update = () => {
diff --git a/src/packages/frontend/user-tracking.ts b/src/packages/frontend/user-tracking.ts
index 21bbc8b254..2ecb0e7a4c 100644
--- a/src/packages/frontend/user-tracking.ts
+++ b/src/packages/frontend/user-tracking.ts
@@ -9,7 +9,7 @@
 
 import { query, server_time } from "./frame-editors/generic/client";
 import { analytics_cookie_name as analytics, uuid } from "@cocalc/util/misc";
-import { redux } from "./app-framework";
+import { entryPoint, redux } from "./app-framework";
 import { version } from "@cocalc/util/smc-version";
 import { get_cookie } from "./misc";
 import { webapp_client } from "./webapp-client";
@@ -41,8 +41,12 @@ export async function log(eventName: string, payload: any): Promise<void> {
 // shows a warning in the console when it can't report to the backend.
 export default async function track(
   event: string,
-  value: object
+  value: object,
 ): Promise<void> {
+  if (entryPoint == "compute") {
+    // for now, we do no tracking of users when using compute servers directly.
+    return;
+  }
   // Replace all dashes with underscores in the event argument for consistency
   event = event.replace(/-/g, "_");
 

From 57dcb9c855ac4b4bcc710ae7fdc96a9918096c25 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Wed, 16 Oct 2024 22:58:54 +0000
Subject: [PATCH 15/35] writing something about what synctables are

motivation: I'm trying to decide what architecture to use for the
compute server's web server, and it would be really helpful to better
understand what we do now, in order to see the pros and cons.
---
 src/packages/sync/table/README.md | 117 +++++++++++++++++++++++++-----
 1 file changed, 97 insertions(+), 20 deletions(-)

diff --git a/src/packages/sync/table/README.md b/src/packages/sync/table/README.md
index bc77975f52..164d652cd2 100644
--- a/src/packages/sync/table/README.md
+++ b/src/packages/sync/table/README.md
@@ -1,4 +1,70 @@
-# SYNCHRONIZED TABLE --
+# Synchronized Tables aka SyncTables
+
+A synchronized table is basically a small in\-memory key:value store that is replicated across many web browsers and a cocalc project, and usually persisted in the database. They are _mostly_ defined by a PostgreSQL table and rules that grant a user permission to read or write to a defined subset of the table. There are also ephemeral synctables, which are not persisted to the database, e.g., the location of user cursors in a document.
+
+A web browser can directly manage a synctable via queries \(through a hub\) to the database, e.g., the account settings and list of user projects are dealt with this way.
+
+For file editing, the synctable is managed by the project, and all web browsers connect via a websocket to the project. The project ensures that the syntable state is eventually identical for all browsers to its own state, and is also responsible for persisting data longterm to the database. It does NOT setup a changefeed for file editing with the database, since all file editing goes through the project, hence it is the only one that changes the database table of file editing state.
+
+```mermaid
+graph TD;
+  A[Web Browsers...] -- WebSocket --> B[The Project Home Base];
+  D -- TCP Socket --> B;
+  A -- Queries --> D[Hubs...];
+  D -- SQL & LISTEN/NOTIFY --> C[(The PostgreSQL Database)];
+  G[Compute Servers...] -- WebSocket --> B;
+  A -- WebSocket --> G;
+
+  subgraph SyncTable Architecture
+    E((Ephemeral SyncTables))
+    F((Persistent SyncTables))
+    G
+    A
+    B
+  end
+
+  style E fill:#f9f,stroke:#333,stroke-width:2px
+  style F fill:#ff9,stroke:#333,stroke-width:2px
+  style B fill:#afa,stroke:#333,stroke-width:2px
+  style G fill:#f88,stroke:#333,stroke-width:2px
+  style C fill:#ccf,stroke:#333,stroke-width:2px
+  style D fill:#ffb,stroke:#333,stroke-width:2px
+```
+
+The direction of the arrow indicates which side _initiates_ the connection; in most cases data flows in both directions,
+possibly with a realtime push model (due to persistent connections).
+
+## SyncTables: Motivation and Broader Context
+
+Realtime synchronized text editing and editing complicated data structures like those in Jupyter notebooks is **not** done directly using synctables! Instead, synctables are a low level primitive on which we build those more complicated synchronized documents.
+
+### Global State
+
+We do use synctables to manage various aspects of user state that directly reflects the contents of the database, including:
+
+- all (or recent!) projects a user is a collaborator on,
+- the log of activity in an open project,
+- account settings for a user,
+- files that were recently active in some project you collaborate on
+
+These are all managed via a direct connection between the web browser and a backend hub, which connects to the database and uses PostgreSQL LISTEN/NOTIFY to get realtime updates whenever the database changes. It is not necessary to that if one web browser makes a change, e.g., to account settings, that everybody else sees that as quickly as possible; also, the volume of such traffic is relatively low.
+
+
+### File Editing State
+
+In contrast, when actively editing a file, the higher level SyncDoc data structures create a synctables that is being populated with new patches every second or two, and these must be made visible to
+all other clients as quickly as possible. Moreover, when evaluating code in a Jupyter notebook, a synctable changes to reflect that you want to evaluate code, and is then updated again with a new patch with the result of that code evaluation -- this must all happen quickly, and can't involve the database as part of the data flow. Instead, the browsers and home base communicate directly via a websocket, and the patches are persisted to the database (in big chunks) in the background.
+
+## The Algorithm Used to Synchronize SyncTables
+
+Synctables exist in three places:
+
+- Web Browsers
+- The Project Home Base
+- Compute Servers
+
+They are _eventually consistent_, in the sense that if everybody stops making changes and all network connections are working, then eventually all copies of the same table will be the same.
+
 
 ## Defined by an object query
 
@@ -8,21 +74,24 @@
 
 ## Methods
 
-- constructor(query): query = the name of a table (or a more complicated object)
+- constructor\(query\): query = the name of a table \(or a more complicated object\)
 
-- set(map): Set the given keys of map to their values; one key must be
-  the primary key for the table. NOTE: Computed primary keys will
-  get automatically filled in; these are keys in schema.coffee,
-  where the set query looks like this say:
-  (obj, db) -> db.sha1(obj.project_id, obj.path)
-- get(): Current value of the query, as an immutable.js Map from
+- set\(map\): Set the given keys of map to their values; one key must be
+  the primary key for the table. NOTE: Computed primary keys will get automatically filled in; these are keys in `packages/util/db-schema`
+  where the set query looks like this:
+  `(obj, db) -> db.sha1(obj.project_id, obj.path)`
+
+- get\(\): Current value of the query, as an immutable.js Map from
   the primary key to the records, which are also immutable.js Maps.
-- get(key): The record with given key, as an immutable Map.
-- get(keys): Immutable Map from given keys to the corresponding records.
-- get_one(): Returns one record as an immutable Map (useful if there
-  is only one record)
 
-- close(): Frees up resources, stops syncing, don't use object further
+- get\(key\): The record with given key, as an immutable Map.
+
+- get\(keys\): Immutable Map from given keys to the corresponding records.
+
+- get_one\(\): Returns one record as an immutable Map \(useful if there
+  is only one record\)
+
+- close\(\): Frees up resources, stops syncing, don't use object further
 
 ## Events
 
@@ -43,18 +112,26 @@
 
 A SyncTable is a finite state machine as follows:
 
-                          -------------------<------------------
-                         \|/                                   |
-    [connecting] --> [connected]  -->  [disconnected]  --> [reconnecting]
-
-Also, there is a final state called 'closed', that the SyncTable moves to when
+```mermaid
+stateDiagram-v2
+    [*] --> connecting
+    connecting --> connected
+    connected --> disconnected
+    disconnected --> reconnecting
+    reconnecting --> connecting
+
+    connecting --> closed
+    connected --> closed
+    disconnected --> closed
+    reconnecting --> closed
+```
+
+There is a final state called 'closed', that the SyncTable moves to when
 it will not be used further; this frees up all connections and used memory.
 The table can't be used after it is closed. The only way to get to the
 closed state is to explicitly call close() on the table; otherwise, the
 table will keep attempting to connect and work, until it works.
 
-    (anything)  --> [closed]
-
 - connecting -- connecting to the backend, and have never connected before.
 
 - connected -- successfully connected to the backend, initialized, and receiving updates.

From 3d919052bc1caa5e91b1a3a0585d627465471437 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Thu, 17 Oct 2024 00:09:54 +0000
Subject: [PATCH 16/35] working on documenting synctable algorithm

---
 src/packages/sync/table/README.md    | 84 ++++++++++++++++++++++++----
 src/packages/sync/table/synctable.ts | 11 +++-
 2 files changed, 81 insertions(+), 14 deletions(-)

diff --git a/src/packages/sync/table/README.md b/src/packages/sync/table/README.md
index 164d652cd2..6b9bf018cd 100644
--- a/src/packages/sync/table/README.md
+++ b/src/packages/sync/table/README.md
@@ -1,6 +1,6 @@
 # Synchronized Tables aka SyncTables
 
-A synchronized table is basically a small in\-memory key:value store that is replicated across many web browsers and a cocalc project, and usually persisted in the database. They are _mostly_ defined by a PostgreSQL table and rules that grant a user permission to read or write to a defined subset of the table. There are also ephemeral synctables, which are not persisted to the database, e.g., the location of user cursors in a document.
+A synchronized table is basically a small in\-memory key:value store that is replicated across many web browsers and a cocalc project, and usually persisted in the database. The keys are strings and the values are JSON-able objects. Some of our synctables are defined by a PostgreSQL table and rules that grant a user permission to read or write to a defined subset of the table. There are also ephemeral synctables, which are not persisted to the database, e.g., the location of user cursors in a document.
 
 A web browser can directly manage a synctable via queries \(through a hub\) to the database, e.g., the account settings and list of user projects are dealt with this way.
 
@@ -49,13 +49,12 @@ We do use synctables to manage various aspects of user state that directly refle
 
 These are all managed via a direct connection between the web browser and a backend hub, which connects to the database and uses PostgreSQL LISTEN/NOTIFY to get realtime updates whenever the database changes. It is not necessary to that if one web browser makes a change, e.g., to account settings, that everybody else sees that as quickly as possible; also, the volume of such traffic is relatively low.
 
-
 ### File Editing State
 
 In contrast, when actively editing a file, the higher level SyncDoc data structures create a synctables that is being populated with new patches every second or two, and these must be made visible to
 all other clients as quickly as possible. Moreover, when evaluating code in a Jupyter notebook, a synctable changes to reflect that you want to evaluate code, and is then updated again with a new patch with the result of that code evaluation -- this must all happen quickly, and can't involve the database as part of the data flow. Instead, the browsers and home base communicate directly via a websocket, and the patches are persisted to the database (in big chunks) in the background.
 
-## The Algorithm Used to Synchronize SyncTables
+## Protocol Used to Synchronize SyncTables
 
 Synctables exist in three places:
 
@@ -63,8 +62,68 @@ Synctables exist in three places:
 - The Project Home Base
 - Compute Servers
 
-They are _eventually consistent_, in the sense that if everybody stops making changes and all network connections are working, then eventually all copies of the same table will be the same.
+They are _eventually consistent_, in the sense that if everybody stops making changes and all network connections are working, then eventually all copies of the same table \(i.e., defined by the same query\) will be the same.
+
+There is no fundamental reason regarding the algorithm for not syncing between browsers directly. What is the algorithm though?
+
+### Database Backed Synctables
+
+For synctables that directly reflect the PostgreSQL database, the client writes changes to the hub until they succeed, and the hub listens using LISTEN/NOTIFY for changes to the relevant table in the database, and if they are relevant to the user, sends an update message. Doing this efficiently in general is probably impossible, but we have a pretty explicit list of queries and tables, and special data structures to make this efficient.
+
+The browser client writes changes to the hub until success.
+
+```mermaid
+graph TD;
+  A[Web Browsers...] <--Websocket--> B[Hub];
+  B <--SELECT/INSERT/UPDATE/LISTEN/NOTIFY--> C[(Database)]
+  D[Project Home Base] <--Websocket--> B;
+
+  subgraph Clients of Database Backed SyncTables
+    A
+    D
+    E[Compute Servers...] <--WebSocket-->D
+  end
+```
+
+### Project Backed Synctable Protocol
+
+As mentioned above, there is one master \(the home base\) and a bunch of clients. We do not currently do any browser\-to\-browser synchronization of synctables, so when collaboratively editing documents, all data goes from the browser back to the project home base, and is then broadcast back to the other browsers.
+
+```mermaid
+graph TD;
+  A[Web Browsers...] <---> B[Project Home Base];
+  C[Compute Servers...] <---> B;
+  B -.Non-Ephemeral Data.-> H[Hub] --> D[(Database)];
+  B -.Ephemeral Data.-> N[ /dev/null]
+
+  subgraph Clients of Project-Backed SyncTables
+    A
+    C
+  end
+
+```
+
+Consider a synctable $T_p$ in the project home base $p$ and the corresponding
+synctable $T_c$ in a web browser or compute server client $c$. These are key\-value stores, and either synctable may be modified at any time. There is a persistent websocket connection between the project and browser, but we assume it can go down at any time,
+but eventually comes back. Assume that all relevant data is stored in memory in
+both the project and web browser, since longterm persistent is not relevant to this protocol.
+
+Our goal is to periodically do a sync operation involving $T_p$ and $T_c$, which doesn't require blocking writes on either side, but after which the state of both will be the same if there are no changes to either side.  Moreover, when there are conflicts, we want "last change wins", where time is defined as follows.  We assume that $p$ and $c$ have synchronized their clocks within a second or so, which is fine, given our requirements \(ping times should be far less than 1 second\), and easy to do via a ping.   If between sync operations the same key has different values set, then we resolve the conflict by selecting the one that was changed most recently, with any tie being broken by the project winning.
 
+**NOTE:** This is not a distributed computing paper, it's a simple real life protocol.  Time exists.
+
+First we assume that the network connection is robust and explain how the protocol works.  Then we explain what we do to fix things when the network connection goes down and the client reconnects.  
+
+#### Robust Network
+
+Assume that the network is working perfectly.   Each client keeps track of the following:
+
+- **value:**  the key:value mapping \(we use immutable.js to implement this\)
+- **changes:** a map from each key to 
+
+#### Network Connection Breaks
+
+Next we consider the various situation where the network connection breaks.  During that downtime, we must assume that both every table may be getting changed, so of course we can't just reset everything. 
 
 ## Defined by an object query
 
@@ -74,24 +133,26 @@ They are _eventually consistent_, in the sense that if everybody stops making ch
 
 ## Methods
 
-- constructor\(query\): query = the name of a table \(or a more complicated object\)
+- **constructor\(query\):** query = the name of a table \(or a more complicated object\)
 
-- set\(map\): Set the given keys of map to their values; one key must be
+- **set\(map\):** Set the given keys of map to their values; one key must be
   the primary key for the table. NOTE: Computed primary keys will get automatically filled in; these are keys in `packages/util/db-schema`
   where the set query looks like this:
   `(obj, db) -> db.sha1(obj.project_id, obj.path)`
 
-- get\(\): Current value of the query, as an immutable.js Map from
+- **get\(\):** Current value of the query, as an immutable.js Map from
   the primary key to the records, which are also immutable.js Maps.
 
-- get\(key\): The record with given key, as an immutable Map.
+- **get\(key\):** The record with given key, as an immutable Map.
 
-- get\(keys\): Immutable Map from given keys to the corresponding records.
+- **get\(keys\):** Immutable Map from given keys to the corresponding records.
 
-- get_one\(\): Returns one record as an immutable Map \(useful if there
+- **get\_one\(\):** Returns one record as an immutable Map \(useful if there
   is only one record\)
 
-- close\(\): Frees up resources, stops syncing, don't use object further
+- **close\(\):** Frees up resources, stops syncing, don't use object further
+
+- **save\(\):** Sync all changes to the database or project home base
 
 ## Events
 
@@ -183,3 +244,4 @@ main thing this function deals with below.
 
 3. We use a stable version, since otherwise things will randomly break if the
    key is an object.
+
diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts
index 42b5ac47e2..32de972927 100644
--- a/src/packages/sync/table/synctable.ts
+++ b/src/packages/sync/table/synctable.ts
@@ -930,7 +930,7 @@ export class SyncTable extends EventEmitter {
                 need to be saved.
 
      If writing to the database results in an error (but not due to no network),
-     then an error state is set (which client can consult), an even is emitted,
+     then an error state is set (which client can consult), an event is emitted,
      and we do not try to write to the database again until that error
      state is cleared. One way it can be cleared is by changing the table.
   */
@@ -1362,8 +1362,13 @@ export class SyncTable extends EventEmitter {
     const received_keys = this.apply_changes_to_browser_client(changes);
     if (before != null) {
       before.forEach((_, key) => {
-        if (key == null || received_keys[key]) return; // received as part of init
-        if (this.changes[key] && this.versions[key] == 0) return; // not event sent yet
+        if (key == null || received_keys[key]) {
+          return; // received as part of init
+        }
+        if (this.changes[key] && this.versions[key] == 0) {
+          return;
+          // not event sent yet
+        }
         // This key was known and confirmed sent before init, but
         // didn't get sent back this time.  So it was lost somehow,
         // e.g., due to not getting saved to the database and the project

From 43df69fd85be950076ef40738d3231ac80d538c0 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Thu, 17 Oct 2024 03:58:53 +0000
Subject: [PATCH 17/35] more documentation of synctables

---
 src/packages/sync/table/README.md    | 61 +++++++++++++++++++++++++++-
 src/packages/sync/table/synctable.ts |  2 +-
 2 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/src/packages/sync/table/README.md b/src/packages/sync/table/README.md
index 6b9bf018cd..eede522dae 100644
--- a/src/packages/sync/table/README.md
+++ b/src/packages/sync/table/README.md
@@ -119,11 +119,68 @@ First we assume that the network connection is robust and explain how the protoc
 Assume that the network is working perfectly.   Each client keeps track of the following:
 
 - **value:**  the key:value mapping \(we use immutable.js to implement this\)
-- **changes:** a map from each key to 
+- **changes:** a map from keys that this client has changed but not yet saved to the timestamp when they  made the change. Initially this is empty.
+- **versions:** a map from all keys to when they were last changed \(or time when table was initialized\)
+
+When a participant _makes a change_ to their copy of the synctable, they do the following:
+
+- update **value**\[key\] \(which could include deleting the value\)
+
+- set **changes**\[key\] = time of change.  \(In the code we add 1ms enough times to the realtime so that changes\[key\] are locally distinct, but I don't see why that is needed.\)
+
+- The project assigns **versions**\[key\] and immediately broadcasts to all connected clients the new **value**\[key\] and its version.  The browser sets **versions**\[key\] to 0.
+
+When a browser saves their changes to the project, they send the following to the project for every key where **changes**\[key\] is set:
+
+- **value**\[key\]   \-\- the value.  This also determines the key since it the primary key.
+- **time = changes**\[key\] \-\- when browser made the change to value\[key\]
+
+The project sends a message with the above data, and assuming the send succeeds, it clears the **changes** variable, assuming that the data was sent.  
+
+**NOTE:** Our implementation does verify that the send happens \(write to the websocket succeeds\), but does not wait for an explicit acknowledgement back confirming that the data was received and processed by the project.   I think the idea is that every reason for the project to not receive the data would **also** result in the websocket connection breaking and everything resetting anyways.
+
+The project then receives a message with the above data and does the following \(see `apply_changes_from_browser_client`\):
+
+- if the **changes**\[key\] is $\geq$ **versions**\[key\], make the change, recording the new value, versions and changes values.
+
+We do the above for all keys, then broadcast a message to all connected clients with the new values and times.  The clients then receive those broadcast message and for each key, do the following:
+
+- if the **changes**\[key\] is $\geq$ **versions**\[key\], make the change, recording the new value and version.
+
+**NOTE:** We have not implemented deleting from a synctable yet, and there's code that awkwardly gets around this, e.g., by using a sentinel value.  For collab editing we don't need delete, since all we are saving is patches, which we never delete \(unless deleting everything\).   I don't see any reason why implementing delete would be hard though.
+
+**NOTE:**  Maybe the comparison should be $>$ instead of $\geq$ because in the case of a tie, the project is going to send out the version again, and then the other client with the same timestamp will receive and change their result. It works because the other client does change their result, but it's extra work and a bit nerve racking, and it would just be more efficient to break the tie in the other direction.
 
 #### Network Connection Breaks
 
-Next we consider the various situation where the network connection breaks.  During that downtime, we must assume that both every table may be getting changed, so of course we can't just reset everything. 
+Next we consider the various situation where the network connection breaks, the project restarts entirely, etc..  During that time, we assume that every table may be getting changed \(e.g., users are still typing away\), so we can't just reset everything.   
+
+After something breaks \(network, project, etc.\), a client connects to the project and the following happens:
+
+- the project sends the entire table, which includes key/value pairs, and also their versions.  For an ephemeral table, e.g., cursors, ipywidgets, etc., this is empty.  For a table backed by the database, e.g., directory listings, this would be the latest version of the relevant data from the database and the versions are all reset to the time the table was initialized again in the project. 
+- the client then goes through this table and for each key where the version is bigger $\geq$ to what they have, the value and version is updated \(as above\).
+- for each key that the client knows about and for which they did not change it when offline \(so it's not already marked to send out\), we set changes\[key\] to now, so that key will get sent to the server.    E.g., if we have an ephemeral table and the project side restarts, this makes it so everything in all clients gets sent to the project on the next save, and who wins is somewhat random.
+  - **NOTE**: For an ephemeral table, if one client changes a key when the project is down and nobody else does, then that one client would have the oldest timestamp for that change, and all the other clients, with an older version of that data, would overwrite the newer version.  _That's not optimal and we should probably change this._ Ephemeral tables are typically for things like cursors \(or ipywidgets\) when editing a document, so the impact of this might be a glitch for 1 second while reconnecting, and there's probably other much weirder stuff going on at the same time.
+
+**NOTE:** If the table is entirely reset to the database and the browser has made changes while waiting to connect, those would be lost, because the versions that the project assigns in this case are when the table is initialized again in the project.   There's probably applications where this is bad.  However, in cocalc it isn't a problem: for collaborative editing we never change entries in the table, instead just make new ones \(which can't clash\); for listings, those are only created by the project.
+
+### Compute Servers
+
+Our new goal is that compute servers can also act like the project.  In fact a large number of web browsers connect to a compute server, it manages sync with all of them, then periodically sends a big single update to the project.   This would allow for best security, scalability, etc., where a single project could have hundreds of notebooks run at once \(say\), across several compute servers, with realtime sync, and many clients.
+
+```mermaid
+graph TD;
+  A[Web Browsers...] <---> B[Project Home Base];
+  C[Compute Servers...] <---> B;
+  B -.Non-Ephemeral Data.-> H[Hub] --> D[(Database)];
+  B -.Ephemeral Data.-> N[ /dev/null]
+  A <---> C
+
+  subgraph Clients of Project-Backed SyncTables
+    A
+    C
+  end
+```
 
 ## Defined by an object query
 
diff --git a/src/packages/sync/table/synctable.ts b/src/packages/sync/table/synctable.ts
index 32de972927..b6baa1bffc 100644
--- a/src/packages/sync/table/synctable.ts
+++ b/src/packages/sync/table/synctable.ts
@@ -965,7 +965,7 @@ export class SyncTable extends EventEmitter {
     const changes = copy(this.changes);
     //console.log("_save: send ", changes);
     for (const key in this.changes) {
-      if (this.versions[key] === 0) {
+      if (this.versions[key] == 0) {
         proposed_keys[key] = true;
       }
       const x = this.value.get(key);

From 286dfc1b3a048a9946fb4c46e93be3405bf69ba9 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Thu, 17 Oct 2024 19:04:54 +0000
Subject: [PATCH 18/35] working on refactoring server status code (nothing much
 done yet)

---
 .../lib/http-server/synctable-channel.ts      | 18 ++++++++++
 .../compute/lib/http-server/websocket-api.ts  | 10 +++++-
 src/packages/project/project-status/server.ts | 24 +++++++-------
 src/packages/project/sync/project-status.ts   | 33 +++++++------------
 src/packages/project/sync/server.ts           |  1 -
 src/packages/util/db-schema/project-status.ts |  4 ++-
 6 files changed, 53 insertions(+), 37 deletions(-)
 create mode 100644 src/compute/compute/lib/http-server/synctable-channel.ts

diff --git a/src/compute/compute/lib/http-server/synctable-channel.ts b/src/compute/compute/lib/http-server/synctable-channel.ts
new file mode 100644
index 0000000000..32a24ceeff
--- /dev/null
+++ b/src/compute/compute/lib/http-server/synctable-channel.ts
@@ -0,0 +1,18 @@
+import { getLogger } from "../logger";
+
+const log = getLogger("synctable-channel");
+
+export default async function synctableChannel({
+  manager,
+  query,
+  options,
+  primus,
+}: {
+  manager;
+  query;
+  options;
+  primus;
+}) {
+  log.debug(JSON.stringify({ query, options }));
+  throw Error("not implemented");
+}
diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index 1b99e0802e..bd8a1ce1a0 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -15,6 +15,7 @@ import { callback2 } from "@cocalc/util/async-utils";
 import { terminal } from "@cocalc/terminal";
 import realpath from "@cocalc/backend/realpath";
 import { eval_code } from "@cocalc/backend/eval-code";
+import synctableChannel from "./synctable-channel";
 
 const log = getLogger("websocket-api");
 
@@ -104,6 +105,14 @@ async function handleApiCall(
     case "realpath":
       return realpath(data.path, manager.home);
 
+    case "synctable_channel":
+      return await synctableChannel({
+        manager,
+        query: data.query,
+        options: data.options,
+        primus,
+      });
+
     // TODO
     case "lean":
     case "jupyter_strip_notebook":
@@ -111,7 +120,6 @@ async function handleApiCall(
     case "jupyter_run_notebook":
     case "lean_channel":
     case "x11_channel":
-    case "synctable_channel":
     case "syncdoc_call":
     case "symmetric_channel":
     case "project_info":
diff --git a/src/packages/project/project-status/server.ts b/src/packages/project/project-status/server.ts
index 53a7c44703..e15b6104aa 100644
--- a/src/packages/project/project-status/server.ts
+++ b/src/packages/project/project-status/server.ts
@@ -8,14 +8,14 @@ Project status server, doing the heavy lifting of telling the client
 what's going on in the project, especially if there is a problem.
 
 Under the hood, it subscribes to the ProjectInfoServer, which updates
-various statistics at a high-frequency. Therefore, this here filters
+various statistics at a high-frequency. Therefore, this filters
 that information to a low-frequency low-volume stream of important
 status updates.
 
-Hence in particular, information like cpu, memory and disk are smoothed out and throttled.
+In particular, information like cpu, memory and disk are smoothed out and throttled.
 */
 
-import { getLogger } from "@cocalc/project/logger";
+import { getLogger } from "@cocalc/backend/logger";
 import { how_long_ago_m, round1 } from "@cocalc/util/misc";
 import { version as smcVersion } from "@cocalc/util/smc-version";
 import { delay } from "awaiting";
@@ -42,7 +42,7 @@ import { cgroup_stats } from "@cocalc/comm/project-status/utils";
 //  return next;
 //}
 
-const winston = getLogger("ProjectStatusServer");
+const logger = getLogger("project:project-status-server");
 
 function quantize(val, order) {
   const q = Math.round(Math.pow(10, order));
@@ -58,7 +58,6 @@ interface Elevated {
 }
 
 export class ProjectStatusServer extends EventEmitter {
-  private readonly dbg: Function;
   private running = false;
   private readonly testing: boolean;
   private readonly project_info: ProjectInfoServer;
@@ -83,14 +82,13 @@ export class ProjectStatusServer extends EventEmitter {
   constructor(testing = false) {
     super();
     this.testing = testing;
-    this.dbg = (...msg) => winston.debug(...msg);
     this.project_info = get_ProjectInfoServer();
   }
 
   private async init(): Promise<void> {
     this.project_info.start();
     this.project_info.on("info", (info) => {
-      //this.dbg(`got info timestamp=${info.timestamp}`);
+      //logger.debug(`got info timestamp=${info.timestamp}`);
       this.info = info;
       this.update();
       this.emitInfo();
@@ -100,14 +98,14 @@ export class ProjectStatusServer extends EventEmitter {
   // checks if there the current state (after update()) should be emitted
   private emitInfo(): void {
     if (this.lastEmit === 0) {
-      this.dbg("emitInfo[last=0]", this.status);
+      logger.debug("emitInfo[last=0]", this.status);
       this.doEmit();
       return;
     }
 
     // if alert changed, emit immediately
     if (!isEqual(this.last?.alerts, this.status?.alerts)) {
-      this.dbg("emitInfo[alert]", this.status);
+      logger.debug("emitInfo[alert]", this.status);
       this.doEmit();
     } else {
       // deep comparison check via lodash and we rate limit
@@ -115,7 +113,7 @@ export class ProjectStatusServer extends EventEmitter {
         this.lastEmit + 1000 * STATUS_UPDATES_INTERVAL_S > Date.now();
       const changed = !isEqual(this.status, this.last);
       if (!recent && changed) {
-        this.dbg("emitInfo[changed]", this.status);
+        logger.debug("emitInfo[changed]", this.status);
         this.doEmit();
       }
     }
@@ -202,7 +200,7 @@ export class ProjectStatusServer extends EventEmitter {
       }
     }
     pids.sort(); // to make this stable across iterations
-    //this.dbg("alert_cpu_processes", pids, ecp);
+    //logger.debug("alert_cpu_processes", pids, ecp);
     return pids;
   }
 
@@ -298,7 +296,7 @@ export class ProjectStatusServer extends EventEmitter {
 
   public async start(): Promise<void> {
     if (this.running) {
-      this.dbg(
+      logger.debug(
         "project-status/server: already running, cannot be started twice",
       );
     } else {
@@ -307,7 +305,7 @@ export class ProjectStatusServer extends EventEmitter {
   }
 
   private async _start(): Promise<void> {
-    this.dbg("start");
+    logger.debug("start");
     if (this.running) {
       throw Error("Cannot start ProjectStatusServer twice");
     }
diff --git a/src/packages/project/sync/project-status.ts b/src/packages/project/sync/project-status.ts
index 14fb1a73b8..2d4d205d29 100644
--- a/src/packages/project/sync/project-status.ts
+++ b/src/packages/project/sync/project-status.ts
@@ -10,25 +10,22 @@ import {
   get_ProjectStatusServer,
   ProjectStatusServer,
 } from "../project-status";
-import { ProjectStatus } from "@cocalc/comm/project-status/types";
+import type { ProjectStatus } from "@cocalc/comm/project-status/types";
+import { getLogger } from "@cocalc/backend/logger";
+
+const logger = getLogger("project:project-status");
 
 class ProjectStatusTable {
   private table: SyncTable;
-  private logger: { debug: Function };
   private project_id: string;
   private state: "ready" | "closed" = "ready";
   private readonly publish: (status: ProjectStatus) => Promise<void>;
   private readonly status_server: ProjectStatusServer;
 
-  constructor(
-    table: SyncTable,
-    logger: { debug: Function },
-    project_id: string,
-  ) {
+  constructor(table: SyncTable, project_id: string) {
     this.status_handler = this.status_handler.bind(this);
     this.project_id = project_id;
-    this.logger = logger;
-    this.log("register");
+    logger.debug("register");
     this.publish = reuseInFlight(this.publish_impl.bind(this));
     this.table = table;
     this.table.on("closed", () => this.close());
@@ -39,7 +36,7 @@ class ProjectStatusTable {
   }
 
   private status_handler(status): void {
-    this.log?.("status_server event 'status'", status.timestamp);
+    logger.debug("status_server event 'status'", status.timestamp);
     this.publish?.(status);
   }
 
@@ -50,10 +47,10 @@ class ProjectStatusTable {
       try {
         await this.table.save();
       } catch (err) {
-        this.log(`error saving ${err}`);
+        logger.debug(`error saving ${err}`);
       }
-    } else if (this.log != null) {
-      this.log(
+    } else {
+      logger.debug(
         `ProjectStatusTable '${
           this.state
         }' and table is ${this.table?.get_state()}`,
@@ -62,24 +59,18 @@ class ProjectStatusTable {
   }
 
   public close(): void {
-    this.log("close");
+    logger.debug("close");
     this.status_server?.off("status", this.status_handler);
     this.table?.close_no_async();
     close(this);
     this.state = "closed";
   }
-
-  private log(...args): void {
-    if (this.logger == null) return;
-    this.logger.debug("project_status", ...args);
-  }
 }
 
 let project_status_table: ProjectStatusTable | undefined = undefined;
 
 export function register_project_status_table(
   table: SyncTable,
-  logger: any,
   project_id: string,
 ): void {
   logger.debug("register_project_status_table");
@@ -89,7 +80,7 @@ export function register_project_status_table(
     );
     project_status_table.close();
   }
-  project_status_table = new ProjectStatusTable(table, logger, project_id);
+  project_status_table = new ProjectStatusTable(table, project_id);
 }
 
 export function get_project_status_table(): ProjectStatusTable | undefined {
diff --git a/src/packages/project/sync/server.ts b/src/packages/project/sync/server.ts
index 188118a899..d0706958e0 100644
--- a/src/packages/project/sync/server.ts
+++ b/src/packages/project/sync/server.ts
@@ -592,7 +592,6 @@ async function synctable_channel0(
     } else if (query?.project_status != null) {
       register_project_status_table(
         synctable_channels[name].get_synctable(),
-        logger,
         client.client_id(),
       );
     } else if (query?.usage_info != null) {
diff --git a/src/packages/util/db-schema/project-status.ts b/src/packages/util/db-schema/project-status.ts
index d1127d74ea..9d891a25da 100644
--- a/src/packages/util/db-schema/project-status.ts
+++ b/src/packages/util/db-schema/project-status.ts
@@ -8,7 +8,9 @@ This table contains the current overall status about a running project.
 This is the sister-table to "project-info".
 In contrast, this table provides much less frequently changed pieces of status information.
 For example, project version, certain "alerts", disk usage, etc.
-Its intended usage is to subscribe to it once you open a project and notify the user if certain alerts go off.
+Its intended use is to subscribe to it once you open a project and notify the user if certain alerts go off.
+
+It is an ephemeral table stored in memory only in the project and clients.
 */
 
 import { Table } from "./types";

From 08e05e098b096da3aaa71cef347e834541f7f7e3 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 00:08:39 +0000
Subject: [PATCH 19/35] working on compute http server again...

---
 .../lib/http-server/synctable-channel.ts      |  2 ++
 .../compute/lib/http-server/websocket-api.ts  | 26 +++++++++----------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/compute/compute/lib/http-server/synctable-channel.ts b/src/compute/compute/lib/http-server/synctable-channel.ts
index 32a24ceeff..656c262448 100644
--- a/src/compute/compute/lib/http-server/synctable-channel.ts
+++ b/src/compute/compute/lib/http-server/synctable-channel.ts
@@ -14,5 +14,7 @@ export default async function synctableChannel({
   primus;
 }) {
   log.debug(JSON.stringify({ query, options }));
+  const syncTable = manager.client.sync_client.sync_table(query, options);
+  log.debug("have our syncTable!", syncTable);
   throw Error("not implemented");
 }
diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index bd8a1ce1a0..b855784aee 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -31,6 +31,7 @@ export function initWebsocketApi({ primus, manager }): void {
         done(resp);
       } catch (err) {
         // console.trace(); log.debug("primus-api error stacktrack", err.stack, err);
+        console.log("primus-api", "request", data, "FAILED", err);
         done({ error: err.toString(), status: "error" });
       }
       log.debug(
@@ -63,18 +64,6 @@ async function handleApiCall(
       // see packages/sync-fs/lib/index.ts
       return await getListing(data.path, data.hidden, manager.home);
 
-    // TODO
-    case "delete_files":
-    case "move_files":
-    case "rename_file":
-    case "canonical_paths":
-    case "configuration":
-    case "prettier": // deprecated
-    case "formatter":
-    case "prettier_string": // deprecated
-    case "formatter_string":
-      throw Error(`command "${(data as any).cmd}" not implemented`);
-
     case "exec":
       if (data.opts == null) {
         throw Error("opts must not be null");
@@ -87,7 +76,9 @@ async function handleApiCall(
 
     case "query":
       if (data.opts?.changes) {
-        throw Error("changefeeds are not supported for api queries");
+        throw Error(
+          `changefeeds are not supported for api queries`,
+        );
       }
       return await callback2(
         manager.client.query.bind(manager.client),
@@ -114,6 +105,15 @@ async function handleApiCall(
       });
 
     // TODO
+    case "delete_files":
+    case "move_files":
+    case "rename_file":
+    case "canonical_paths":
+    case "configuration":
+    case "prettier": // deprecated
+    case "formatter":
+    case "prettier_string": // deprecated
+    case "formatter_string":
     case "lean":
     case "jupyter_strip_notebook":
     case "jupyter_nbconvert":

From 45d1084c8d663b68224209bf06fa366f45affd26 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 04:01:22 +0000
Subject: [PATCH 20/35] compute-http-server: working toward synctables

---
 src/compute/compute/dev/README.md                   | 13 +++++++++++++
 src/compute/compute/dev/start-compute.js            | 10 ++++++++++
 .../compute/lib/http-server/synctable-channel.ts    | 11 ++++++++---
 src/compute/compute/lib/listings.ts                 |  2 ++
 src/compute/compute/lib/manager.ts                  |  3 ++-
 5 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/src/compute/compute/dev/README.md b/src/compute/compute/dev/README.md
index 560147a228..96dc439476 100644
--- a/src/compute/compute/dev/README.md
+++ b/src/compute/compute/dev/README.md
@@ -47,3 +47,16 @@ For debugging set the DEBUG env variable to different things according to the de
 DEBUG_CONSOLE=yes DEBUG=* ./2-syncfs.sh
 ```
 
+### Console
+
+Once you have everything setup you can get an interactive console by doing this instead of [3\-compute.sh](http://3-compute.sh): 
+
+```sh
+~/cocalc/src/compute/compute/dev$ . env.sh
+~/cocalc/src/compute/compute/dev$ node
+Welcome to Node.js v18.17.1.
+Type ".help" for more information.
+> manager = require('./start-compute.js').manager
+
+```
+
diff --git a/src/compute/compute/dev/start-compute.js b/src/compute/compute/dev/start-compute.js
index d006d01463..5f35d93199 100644
--- a/src/compute/compute/dev/start-compute.js
+++ b/src/compute/compute/dev/start-compute.js
@@ -1,5 +1,15 @@
 #!/usr/bin/env node
 
+/*
+To get an interactive console with access to the manager:
+
+~/cocalc/src/compute/compute/dev$ node
+Welcome to Node.js v18.17.1.
+Type ".help" for more information.
+> manager = require('./start-compute.js').manager
+
+*/
+
 process.env.API_SERVER = process.env.API_SERVER ?? "https://cocalc.com";
 console.log("API_SERVER=", process.env.API_SERVER);
 
diff --git a/src/compute/compute/lib/http-server/synctable-channel.ts b/src/compute/compute/lib/http-server/synctable-channel.ts
index 656c262448..56f920f8d9 100644
--- a/src/compute/compute/lib/http-server/synctable-channel.ts
+++ b/src/compute/compute/lib/http-server/synctable-channel.ts
@@ -13,8 +13,13 @@ export default async function synctableChannel({
   options;
   primus;
 }) {
-  log.debug(JSON.stringify({ query, options }));
-  const syncTable = manager.client.sync_client.sync_table(query, options);
-  log.debug("have our syncTable!", syncTable);
+  log.debug("synctableChannel ", query, options);
+  console.log("synctableChannel", primus != null);
+  const table = await manager.client.synctable_project(
+    manager.project_id,
+    query,
+    options ?? [],
+  );
+  console.log("have our syncTable!", table.get()?.toJS());
   throw Error("not implemented");
 }
diff --git a/src/compute/compute/lib/listings.ts b/src/compute/compute/lib/listings.ts
index d7181edd37..2e2a1277c1 100644
--- a/src/compute/compute/lib/listings.ts
+++ b/src/compute/compute/lib/listings.ts
@@ -60,4 +60,6 @@ export async function initListings({
     existsSync,
     getLogger,
   });
+
+  return table;
 }
diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts
index 3f8b45453d..84463c694e 100644
--- a/src/compute/compute/lib/manager.ts
+++ b/src/compute/compute/lib/manager.ts
@@ -90,6 +90,7 @@ export class Manager {
   public compute_server_id: number;
   private connections: { [path: string]: any } = {};
   private websocket;
+  public listings;
   public client;
 
   constructor({
@@ -187,7 +188,7 @@ export class Manager {
   };
 
   private initListings = async () => {
-    await initListings({
+    this.listings = await initListings({
       client: this.client,
       project_id: this.project_id,
       compute_server_id: this.compute_server_id,

From f1b4cc3b1f8c7fac12ad4943937d74c1d172f9ca Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 13:53:36 +0000
Subject: [PATCH 21/35] sync refactoring: move the syncdocs manager to
 @cocalc/sync/server

motivation is so it can be used in a compute server too; also this helps
better organize and clean up code.  more to come.
---
 src/packages/project/browser-websocket/api.ts |   4 +-
 src/packages/project/client.ts                |   6 +-
 .../sync/compute-server-open-file-tracking.ts |   4 +-
 src/packages/project/sync/listings.ts         |   4 +-
 src/packages/project/sync/server.ts           |  26 +++-
 src/packages/sync/client/types.ts             |   2 +-
 src/packages/sync/server/README.md            |  12 ++
 .../sync => sync/server}/open-synctables.ts   |   0
 .../server/syncdocs-manager.ts}               | 122 +++++++++++-------
 .../sync/table/synctable-no-changefeed.ts     |   2 +
 .../sync/table/synctable-no-database.ts       |   2 +
 src/packages/sync/table/test/client-test.ts   |   3 +
 12 files changed, 122 insertions(+), 65 deletions(-)
 create mode 100644 src/packages/sync/server/README.md
 rename src/packages/{project/sync => sync/server}/open-synctables.ts (100%)
 rename src/packages/{project/sync/sync-doc.ts => sync/server/syncdocs-manager.ts} (68%)

diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts
index 822646ca55..23e5a601b0 100644
--- a/src/packages/project/browser-websocket/api.ts
+++ b/src/packages/project/browser-websocket/api.ts
@@ -22,7 +22,7 @@ import { lean, lean_channel } from "../lean/server";
 import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse";
 import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run";
 import { synctable_channel } from "../sync/server";
-import { syncdoc_call } from "../sync/sync-doc";
+import { callSyncDoc } from "@cocalc/sync/server/syncdocs-manager";
 import { terminal } from "@cocalc/terminal";
 import { x11_channel } from "../x11/server";
 import { canonical_paths } from "./canonical-path";
@@ -167,7 +167,7 @@ async function handleApiCall(data: Mesg, spark): Promise<any> {
         data.options,
       );
     case "syncdoc_call":
-      return await syncdoc_call(data.path, data.mesg);
+      return await callSyncDoc(data.path, data.mesg);
     case "symmetric_channel":
       return await browser_symmetric_channel(client, primus, log, data.name);
     case "realpath":
diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts
index f3cab0492c..83af21f584 100644
--- a/src/packages/project/client.ts
+++ b/src/packages/project/client.ts
@@ -46,8 +46,8 @@ import * as kucalc from "./kucalc";
 import { getLogger } from "./logger";
 import * as sage_session from "./sage_session";
 import { getListingsTable } from "@cocalc/project/sync/listings";
-import { get_synctable } from "./sync/open-synctables";
-import { get_syncdoc } from "./sync/sync-doc";
+import { get_synctable } from "@cocalc/sync/server/open-synctables";
+import { getSyncDoc } from "@cocalc/sync/server/syncdocs-manager";
 
 const winston = getLogger("client");
 
@@ -506,7 +506,7 @@ export class Client extends EventEmitter implements ProjectClientInterface {
   // Get the synchronized doc with the given path.  Returns undefined
   // if currently no such sync-doc.
   public syncdoc({ path }: { path: string }): SyncDoc | undefined {
-    return get_syncdoc(path);
+    return getSyncDoc(path);
   }
 
   public symmetric_channel(name) {
diff --git a/src/packages/project/sync/compute-server-open-file-tracking.ts b/src/packages/project/sync/compute-server-open-file-tracking.ts
index 4ebb773a59..6d74a6849a 100644
--- a/src/packages/project/sync/compute-server-open-file-tracking.ts
+++ b/src/packages/project/sync/compute-server-open-file-tracking.ts
@@ -1,10 +1,10 @@
 /*
 Manage the state of open files in the compute servers syncdb sync'd file.
 
-TODO: terminals aren't handled at all here, since they don't have a syncdoc.
+NOTE: terminals aren't handled at all here, since they don't have a syncdoc.
 */
 
-import type { SyncDocs } from "./sync-doc";
+import type { SyncDocs } from "@cocalc/sync/server/syncdocs-manager";
 import type { SyncDB } from "@cocalc/sync/editor/db/sync";
 import { once } from "@cocalc/util/async-utils";
 import { meta_file, auxFileToOriginal } from "@cocalc/util/misc";
diff --git a/src/packages/project/sync/listings.ts b/src/packages/project/sync/listings.ts
index 796f0a01a6..4a5ebf2c58 100644
--- a/src/packages/project/sync/listings.ts
+++ b/src/packages/project/sync/listings.ts
@@ -9,7 +9,7 @@ import {
 } from "@cocalc/sync/listings";
 import getListing from "@cocalc/backend/get-listing";
 import { Watcher } from "@cocalc/backend/path-watcher";
-import { close_all_syncdocs_in_tree } from "./sync-doc";
+import { closeAllSyncDocsInTree } from "@cocalc/sync/server/syncdocs-manager";
 import { getLogger } from "@cocalc/backend/logger";
 import { existsSync } from "fs";
 
@@ -24,7 +24,7 @@ export function registerListingsTable(table, query): void {
   const onDeletePath = async (path) => {
     // Also we need to close *all* syncdocs that are going to be deleted,
     // and wait until closing is done before we return.
-    await close_all_syncdocs_in_tree(path);
+    await closeAllSyncDocsInTree(path);
   };
 
   const createWatcher = (path: string, debounce: number) =>
diff --git a/src/packages/project/sync/server.ts b/src/packages/project/sync/server.ts
index d0706958e0..dce5ab6d72 100644
--- a/src/packages/project/sync/server.ts
+++ b/src/packages/project/sync/server.ts
@@ -10,7 +10,7 @@ between project and browser client.
 TODO:
 
 - [ ] If initial query fails, need to raise exception.  Right now it gets
-silently swallowed in persistent mode...
+      silently swallowed in persistent mode...
 */
 
 // How long to wait from when we hit 0 clients until closing this channel.
@@ -51,8 +51,7 @@ import {
 // @ts-ignore -- typescript nonsense.
 const _ = set_debug;
 
-import { init_syncdoc, getSyncDocFromSyncTable } from "./sync-doc";
-import { key, register_synctable } from "./open-synctables";
+import { key, register_synctable } from "@cocalc/sync/server/open-synctables";
 import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { once } from "@cocalc/util/async-utils";
 import { delay } from "awaiting";
@@ -63,7 +62,24 @@ import { register_project_status_table } from "./project-status";
 import { register_usage_info_table } from "./usage-info";
 import type { MergeType } from "@cocalc/sync/table/synctable";
 import Client from "@cocalc/sync-client";
-import { getJupyterRedux } from "@cocalc/jupyter/kernel";
+import {
+  getJupyterRedux,
+  initJupyterRedux,
+  removeJupyterRedux,
+} from "@cocalc/jupyter/kernel";
+import {
+  initSyncDoc,
+  getSyncDocFromSyncTable,
+  initSyncDocsManager,
+} from "@cocalc/sync/server/syncdocs-manager";
+
+import computeServerOpenFileTracking from "./compute-server-open-file-tracking";
+import { getLogger } from "@cocalc/backend/logger";
+initSyncDocsManager({
+  logger: getLogger("project:sync:sync-doc"),
+  computeServerOpenFileTracking,
+  jupyter: { initJupyterRedux, removeJupyterRedux },
+});
 
 type Query = { [key: string]: any };
 
@@ -237,7 +253,7 @@ class SyncTableChannel {
     }
     if (this.synctable.table === "syncstrings") {
       this.log("init_synctable -- syncstrings: also initialize syncdoc...");
-      init_syncdoc(this.client, this.synctable);
+      initSyncDoc(this.client, this.synctable);
     }
 
     this.synctable.on(
diff --git a/src/packages/sync/client/types.ts b/src/packages/sync/client/types.ts
index af6fa87b9f..12e15a89f7 100644
--- a/src/packages/sync/client/types.ts
+++ b/src/packages/sync/client/types.ts
@@ -17,6 +17,7 @@ export interface Client extends EventEmitter {
   touch_project: (project_id: string) => void;
   set_connected?: Function;
   is_deleted: (path: string, project_id: string) => true | false | undefined;
+  client_id: () => string | undefined;
 }
 
 export interface ClientFs extends Client {
@@ -35,7 +36,6 @@ export interface ClientFs extends Client {
     debounce?: number;
   }) => any;
   server_time: () => Date;
-  client_id: () => string | undefined;
 }
 
 export interface Channel {
diff --git a/src/packages/sync/server/README.md b/src/packages/sync/server/README.md
new file mode 100644
index 0000000000..2997e75041
--- /dev/null
+++ b/src/packages/sync/server/README.md
@@ -0,0 +1,12 @@
+# Sync Server
+
+The project home base @cocalc/project serves synctables to browser clients
+and compute servers. Compute servers themselves also server synctables
+to browsers, so that browsers can connect directly to compute servers
+for reduced latency.
+
+All connections work over WebSockets, and the code here is meant to manage
+all of this. It makes sure that there is only one actual synctable with
+given defining parameters and so on.
+
+**TODO**
\ No newline at end of file
diff --git a/src/packages/project/sync/open-synctables.ts b/src/packages/sync/server/open-synctables.ts
similarity index 100%
rename from src/packages/project/sync/open-synctables.ts
rename to src/packages/sync/server/open-synctables.ts
diff --git a/src/packages/project/sync/sync-doc.ts b/src/packages/sync/server/syncdocs-manager.ts
similarity index 68%
rename from src/packages/project/sync/sync-doc.ts
rename to src/packages/sync/server/syncdocs-manager.ts
index 660fd44407..78f17b8568 100644
--- a/src/packages/project/sync/sync-doc.ts
+++ b/src/packages/sync/server/syncdocs-manager.ts
@@ -4,7 +4,13 @@
  */
 
 /*
-Backend project support for using syncdocs.
+Backend support for using syncdocs.  If a client opens a synctable with
+table='syncstrings', then a corresponding SyncDoc gets created here, so
+that its possible to edit the actual file on disk that corresponds to
+that entry in the syncstrings table.  This is done automatically, rather
+than requiring the frontend client to somehow configure this.
+
+---
 
 This is mainly responsible for:
 
@@ -16,16 +22,26 @@ This is mainly responsible for:
 import { SyncTable } from "@cocalc/sync/table";
 import { SyncDB } from "@cocalc/sync/editor/db/sync";
 import { SyncString } from "@cocalc/sync/editor/string/sync";
-import type Client from "@cocalc/sync-client";
+import type { Client } from "@cocalc/sync/client/types";
 import { once } from "@cocalc/util/async-utils";
 import { filename_extension, original_path } from "@cocalc/util/misc";
-import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel";
 import { EventEmitter } from "events";
 import { COMPUTER_SERVER_DB_NAME } from "@cocalc/util/compute/manager";
-import computeServerOpenFileTracking from "./compute-server-open-file-tracking";
-import { getLogger } from "@cocalc/backend/logger";
 
-const logger = getLogger("project:sync:sync-doc");
+// This must be called externally to initialize the logger and open file tracker,
+// if you need that functionality
+let logger: undefined | { debug: Function } = undefined;
+let computeServerOpenFileTracking: undefined | Function = undefined;
+let jupyter: undefined | { initJupyterRedux; removeJupyterRedux } = undefined;
+export function initSyncDocsManager(opts: {
+  logger?;
+  computeServerOpenFileTracking?;
+  jupyter?;
+}) {
+  logger = opts.logger;
+  computeServerOpenFileTracking = opts.computeServerOpenFileTracking;
+  jupyter = opts.jupyter;
+}
 
 type SyncDoc = SyncDB | SyncString;
 
@@ -39,11 +55,11 @@ export class SyncDocs extends EventEmitter {
   async close(path: string): Promise<void> {
     const doc = this.get(path);
     if (doc == null) {
-      logger.debug(`SyncDocs: close ${path} -- no need, as it is not opened`);
+      logger?.debug(`SyncDocs: close ${path} -- no need, as it is not opened`);
       return;
     }
     try {
-      logger.debug(`SyncDocs: close ${path} -- starting close`);
+      logger?.debug(`SyncDocs: close ${path} -- starting close`);
       this.closing.add(path);
       // As soon as this close starts, doc is in an undefined state.
       // Also, this can take an **unbounded** amount of time to finish,
@@ -63,14 +79,16 @@ export class SyncDocs extends EventEmitter {
       // track down heisenbug. See also
       // https://github.com/sagemathinc/cocalc/issues/5617
       await doc.close();
-      logger.debug(`SyncDocs: close ${path} -- successfully closed`);
+      logger?.debug(`SyncDocs: close ${path} -- successfully closed`);
     } finally {
       // No matter what happens above when it finishes, we clear it
       // and consider it closed.
       // There is perhaps a chance closing fails above (no idea how),
       // but we don't want it to be impossible to attempt to open
       // the path again I.e., we don't want to leave around a lock.
-      logger.debug(`SyncDocs: close ${path} -- recording that close succeeded`);
+      logger?.debug(
+        `SyncDocs: close ${path} -- recording that close succeeded`,
+      );
       delete this.syncdocs[path];
       this.closing.delete(path);
       // I think close-${path} is used only internally in this.create below
@@ -95,11 +113,11 @@ export class SyncDocs extends EventEmitter {
   async create(type, opts): Promise<SyncDoc> {
     const path = opts.path;
     if (this.closing.has(path)) {
-      logger.debug(
+      logger?.debug(
         `SyncDocs: create ${path} -- waiting for previous version to completely finish closing...`,
       );
       await once(this, `close-${path}`);
-      logger.debug(`SyncDocs: create ${path} -- successfully closed.`);
+      logger?.debug(`SyncDocs: create ${path} -- successfully closed.`);
     }
     let doc;
     switch (type) {
@@ -113,11 +131,14 @@ export class SyncDocs extends EventEmitter {
         throw Error(`unknown syncdoc type ${type}`);
     }
     this.syncdocs[path] = doc;
-    logger.debug(`SyncDocs: create ${path} -- successfully created`);
+    logger?.debug(`SyncDocs: create ${path} -- successfully created`);
     // This is used by computeServerOpenFileTracking:
     this.emit("open", path);
-    if (path == COMPUTER_SERVER_DB_NAME) {
-      logger.debug(
+    if (
+      computeServerOpenFileTracking != null &&
+      path == COMPUTER_SERVER_DB_NAME
+    ) {
+      logger?.debug(
         "SyncDocs: also initializing open file tracking for ",
         COMPUTER_SERVER_DB_NAME,
       );
@@ -127,7 +148,7 @@ export class SyncDocs extends EventEmitter {
   }
 
   async closeAll(filename: string): Promise<void> {
-    logger.debug(`SyncDocs: closeAll("${filename}")`);
+    logger?.debug(`SyncDocs: closeAll("${filename}")`);
     for (const path in this.syncdocs) {
       if (path == filename || path.startsWith(filename + "/")) {
         await this.close(path);
@@ -144,7 +165,7 @@ const syncDocs = new SyncDocs();
 // instead of syncdoc_metadata (say) because it was created when we only
 // used strings for sync.
 
-export function init_syncdoc(client: Client, synctable: SyncTable): void {
+export function initSyncDoc(client: Client, synctable: SyncTable): void {
   if (synctable.get_table() !== "syncstrings") {
     throw Error("table must be 'syncstrings'");
   }
@@ -153,33 +174,33 @@ export function init_syncdoc(client: Client, synctable: SyncTable): void {
   }
   // It's the right type of table and not closed.  Now do
   // the real setup work (without blocking).
-  init_syncdoc_async(client, synctable);
+  initSyncDocAsync(client, synctable);
 }
 
 // If there is an already existing syncdoc for this path,
 // return it; otherwise, return undefined.  This is useful
 // for getting a reference to a syncdoc, e.g., for prettier.
-export function get_syncdoc(path: string): SyncDoc | undefined {
+export function getSyncDoc(path: string): SyncDoc | undefined {
   return syncDocs.get(path);
 }
 
 export function getSyncDocFromSyncTable(synctable: SyncTable) {
-  const { opts } = get_type_and_opts(synctable);
-  return get_syncdoc(opts.path);
+  const { opts } = getTypeAndOpts(synctable);
+  return getSyncDoc(opts.path);
 }
 
-async function init_syncdoc_async(
+async function initSyncDocAsync(
   client: Client,
   synctable: SyncTable,
 ): Promise<void> {
   function log(...args): void {
-    logger.debug("init_syncdoc_async: ", ...args);
+    logger?.debug("initSyncDocAsync: ", ...args);
   }
 
   log("waiting until synctable is ready");
-  await wait_until_synctable_ready(synctable);
+  await waitUntilSyncTableReady(synctable);
   log("synctable ready.  Now getting type and opts");
-  const { type, opts } = get_type_and_opts(synctable);
+  const { type, opts } = getTypeAndOpts(synctable);
   const project_id = (opts.project_id = client.client_id());
   //   log("type = ", type);
   //   log("opts = ", JSON.stringify(opts));
@@ -208,47 +229,50 @@ async function init_syncdoc_async(
   log("ext = ", ext);
   switch (ext) {
     case "sage-jupyter2":
-      log("initializing Jupyter backend");
-      await initJupyterRedux(syncdoc, client);
-      const path = original_path(syncdoc.get_path());
-      synctable.on("closed", async () => {
-        log("removing Jupyter backend");
-        await removeJupyterRedux(path, project_id);
-      });
+      if (jupyter != null) {
+        const { initJupyterRedux, removeJupyterRedux } = jupyter;
+        log("initializing Jupyter backend");
+        await initJupyterRedux(syncdoc, client);
+        const path = original_path(syncdoc.get_path());
+        synctable.on("closed", async () => {
+          log("removing Jupyter backend");
+          await removeJupyterRedux(path, project_id);
+        });
+      }
       break;
   }
 }
 
-async function wait_until_synctable_ready(synctable: SyncTable): Promise<void> {
+async function waitUntilSyncTableReady(synctable: SyncTable): Promise<void> {
   if (synctable.get_state() == "disconnected") {
-    logger.debug("wait_until_synctable_ready: wait for synctable be connected");
+    logger?.debug("waitUntilSyncTableReady: wait for synctable be connected");
     await once(synctable, "connected");
   }
 
   const t = synctable.get_one();
   if (t != null) {
-    logger.debug("wait_until_synctable_ready: currently", t.toJS());
+    logger?.debug("waitUntilSyncTableReady: currently", t.toJS());
   }
-  logger.debug(
-    "wait_until_synctable_ready: wait for document info to get loaded into synctable...",
+  logger?.debug(
+    "waitUntilSyncTableReady: wait for document info to get loaded into synctable...",
   );
   // Next wait until there's a document in the synctable, since that will
   // have the path, patch type, etc. in it.  That is set by the frontend.
   function is_ready(): boolean {
     const t = synctable.get_one();
     if (t == null) {
-      logger.debug("wait_until_synctable_ready: is_ready: table is null still");
+      logger?.debug("waitUntilSyncTableReady: is_ready: table is null still");
       return false;
     } else {
-      logger.debug("wait_until_synctable_ready: is_ready", JSON.stringify(t));
+      logger?.debug("waitUntilSyncTableReady: is_ready", JSON.stringify(t));
       return t.has("path");
     }
   }
   await synctable.wait(is_ready, 0);
-  logger.debug("wait_until_synctable_ready: document info is now in synctable");
+  logger?.debug("waitUntilSyncTableReady: document info is now in synctable");
 }
 
-function get_type_and_opts(synctable: SyncTable): { type: string; opts: any } {
+function getTypeAndOpts(synctable: SyncTable): { type: string; opts: any } {
   const s = synctable.get_one();
   if (s == null) {
     throw Error("synctable must not be empty");
@@ -281,18 +305,18 @@ function get_type_and_opts(synctable: SyncTable): { type: string; opts: any } {
   return { type, opts };
 }
 
-export async function syncdoc_call(path: string, mesg: any): Promise<string> {
-  logger.debug("syncdoc_call", path, mesg);
+export async function callSyncDoc(path: string, mesg: any): Promise<string> {
+  logger?.debug("callSyncDoc", path, mesg);
   const doc = syncDocs.get(path);
   if (doc == null) {
-    logger.debug("syncdoc_call -- not open: ", path);
+    logger?.debug("callSyncDoc -- not open: ", path);
     return "not open";
   }
   switch (mesg.cmd) {
     case "close":
-      logger.debug("syncdoc_call -- now closing: ", path);
+      logger?.debug("callSyncDoc -- now closing: ", path);
       await syncDocs.close(path);
-      logger.debug("syncdoc_call -- closed: ", path);
+      logger?.debug("callSyncDoc -- closed: ", path);
       return "successfully closed";
     default:
       throw Error(`unknown command ${mesg.cmd}`);
@@ -301,9 +325,7 @@ export async function syncdoc_call(path: string, mesg: any): Promise<string> {
 
 // This is used when deleting a file/directory
 // filename may be a directory or actual filename
-export async function close_all_syncdocs_in_tree(
-  filename: string,
-): Promise<void> {
-  logger.debug("close_all_syncdocs_in_tree", filename);
+export async function closeAllSyncDocsInTree(filename: string): Promise<void> {
+  logger?.debug("closeAllSyncDocsInTree", filename);
   return await syncDocs.closeAll(filename);
 }
diff --git a/src/packages/sync/table/synctable-no-changefeed.ts b/src/packages/sync/table/synctable-no-changefeed.ts
index 93b57c1194..209375db26 100644
--- a/src/packages/sync/table/synctable-no-changefeed.ts
+++ b/src/packages/sync/table/synctable-no-changefeed.ts
@@ -125,4 +125,6 @@ class ClientNoChangefeed extends EventEmitter {
     // not implemented yet in general
     return undefined;
   };
+
+  client_id = () => this.client.client_id();
 }
diff --git a/src/packages/sync/table/synctable-no-database.ts b/src/packages/sync/table/synctable-no-database.ts
index d86e5f0989..454e20dcda 100644
--- a/src/packages/sync/table/synctable-no-database.ts
+++ b/src/packages/sync/table/synctable-no-database.ts
@@ -124,4 +124,6 @@ class ClientNoDatabase extends EventEmitter {
     // not implemented yet in general
     return undefined;
   };
+
+  client_id = () => this.client.client_id();
 }
diff --git a/src/packages/sync/table/test/client-test.ts b/src/packages/sync/table/test/client-test.ts
index 08268e260c..fec8a7a9bf 100644
--- a/src/packages/sync/table/test/client-test.ts
+++ b/src/packages/sync/table/test/client-test.ts
@@ -71,4 +71,7 @@ export class ClientTest extends EventEmitter {
     // not implemented yet in general
     return undefined;
   };
+
+  // a random uuid...
+  client_id = () => "3fa218e5-7196-4020-8b30-e2127847cc4f";
 }

From 162e030fec1dc99cec972c26919974f34af4eb36 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 15:07:54 +0000
Subject: [PATCH 22/35] add a new package "@cocalc/sync-server"

- obviously way better approach than trying to put server in
  @cocalc/sync where it can only go with a bunch of brittle
  runtime insertions
- very natural module to define
---
 src/compute/pnpm-workspace.yaml               |   1 +
 src/compute/sync-server                       |   1 +
 src/packages/pnpm-lock.yaml                   | 207 ++++++++++++++----
 src/packages/project/browser-websocket/api.ts |   2 +-
 src/packages/project/client.ts                |   4 +-
 src/packages/project/package.json             |   5 +-
 src/packages/project/project-info/server.ts   |   1 -
 .../sync/compute-server-open-file-tracking.ts |   2 +-
 src/packages/project/sync/listings.ts         |   2 +-
 src/packages/project/sync/server.ts           |   4 +-
 src/packages/project/tsconfig.json            |   1 +
 .../{sync/server => sync-server}/README.md    |   0
 .../server => sync-server}/open-synctables.ts |   0
 src/packages/sync-server/package.json         |  35 +++
 .../syncdocs-manager.ts                       |   0
 src/packages/sync-server/tsconfig.json        |   9 +
 src/workspaces.py                             |   1 +
 17 files changed, 222 insertions(+), 53 deletions(-)
 create mode 120000 src/compute/sync-server
 rename src/packages/{sync/server => sync-server}/README.md (100%)
 rename src/packages/{sync/server => sync-server}/open-synctables.ts (100%)
 create mode 100644 src/packages/sync-server/package.json
 rename src/packages/{sync/server => sync-server}/syncdocs-manager.ts (100%)
 create mode 100644 src/packages/sync-server/tsconfig.json

diff --git a/src/compute/pnpm-workspace.yaml b/src/compute/pnpm-workspace.yaml
index 45a7342538..f8d2ae35e2 100644
--- a/src/compute/pnpm-workspace.yaml
+++ b/src/compute/pnpm-workspace.yaml
@@ -6,6 +6,7 @@ packages:
   - "sync"
   - "sync-fs"
   - "sync-client"
+  - "sync-server"
   - "util"
   - "terminal"
   - "compute"
diff --git a/src/compute/sync-server b/src/compute/sync-server
new file mode 120000
index 0000000000..50f34b5b37
--- /dev/null
+++ b/src/compute/sync-server
@@ -0,0 +1 @@
+../packages/sync-server
\ No newline at end of file
diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml
index 2e83faaa1b..d58b2c50eb 100644
--- a/src/packages/pnpm-lock.yaml
+++ b/src/packages/pnpm-lock.yaml
@@ -26,10 +26,10 @@ importers:
         version: 29.5.13
       jest:
         specifier: ^29.7.0
-        version: 29.7.0(@types/node@18.19.50)
+        version: 29.7.0(@types/node@18.19.55)
       ts-jest:
         specifier: ^29.2.3
-        version: 29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.50))(typescript@5.6.3)
+        version: 29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.55))(typescript@5.6.3)
       typescript:
         specifier: ^5.6.3
         version: 5.6.3
@@ -1188,6 +1188,9 @@ importers:
       '@cocalc/sync-fs':
         specifier: workspace:*
         version: link:../sync-fs
+      '@cocalc/sync-server':
+        specifier: workspace:*
+        version: link:../sync-server
       '@cocalc/terminal':
         specifier: workspace:*
         version: link:../terminal
@@ -1906,6 +1909,31 @@ importers:
         specifier: ^18.16.14
         version: 18.19.50
 
+  sync-server:
+    dependencies:
+      '@cocalc/backend':
+        specifier: workspace:*
+        version: link:../backend
+      '@cocalc/sync':
+        specifier: workspace:*
+        version: link:../sync
+      '@cocalc/sync-server':
+        specifier: workspace:*
+        version: 'link:'
+      '@cocalc/util':
+        specifier: workspace:*
+        version: link:../util
+      awaiting:
+        specifier: ^3.0.0
+        version: 3.0.0
+      events:
+        specifier: 3.3.0
+        version: 3.3.0
+    devDependencies:
+      '@types/node':
+        specifier: ^18.16.14
+        version: 18.19.55
+
   terminal:
     dependencies:
       '@cocalc/api-client':
@@ -12874,7 +12902,7 @@ snapshots:
   '@jest/console@29.7.0':
     dependencies:
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       chalk: 4.1.2
       jest-message-util: 29.7.0
       jest-util: 29.7.0
@@ -12887,14 +12915,14 @@ snapshots:
       '@jest/test-result': 29.7.0
       '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       ansi-escapes: 4.3.2
       chalk: 4.1.2
       ci-info: 3.9.0
       exit: 0.1.2
       graceful-fs: 4.2.11
       jest-changed-files: 29.7.0
-      jest-config: 29.7.0(@types/node@18.19.50)
+      jest-config: 29.7.0(@types/node@18.19.55)
       jest-haste-map: 29.7.0
       jest-message-util: 29.7.0
       jest-regex-util: 29.6.3
@@ -12919,7 +12947,7 @@ snapshots:
     dependencies:
       '@jest/fake-timers': 29.7.0
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       jest-mock: 29.7.0
 
   '@jest/expect-utils@29.7.0':
@@ -12937,7 +12965,7 @@ snapshots:
     dependencies:
       '@jest/types': 29.6.3
       '@sinonjs/fake-timers': 10.3.0
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       jest-message-util: 29.7.0
       jest-mock: 29.7.0
       jest-util: 29.7.0
@@ -12959,7 +12987,7 @@ snapshots:
       '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
       '@jridgewell/trace-mapping': 0.3.25
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       chalk: 4.1.2
       collect-v8-coverage: 1.0.2
       exit: 0.1.2
@@ -13028,7 +13056,7 @@ snapshots:
     dependencies:
       '@types/istanbul-lib-coverage': 2.0.6
       '@types/istanbul-reports': 3.0.1
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       '@types/yargs': 15.0.19
       chalk: 4.1.2
 
@@ -13037,7 +13065,7 @@ snapshots:
       '@jest/schemas': 29.6.3
       '@types/istanbul-lib-coverage': 2.0.6
       '@types/istanbul-reports': 3.0.1
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       '@types/yargs': 17.0.24
       chalk: 4.1.2
 
@@ -14285,7 +14313,7 @@ snapshots:
 
   '@types/bonjour@3.5.13':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/caseless@0.12.5': {}
 
@@ -14298,11 +14326,11 @@ snapshots:
   '@types/connect-history-api-fallback@1.5.4':
     dependencies:
       '@types/express-serve-static-core': 4.19.0
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/connect@3.4.35':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/cookie@0.3.3': {}
 
@@ -14339,7 +14367,7 @@ snapshots:
 
   '@types/express-serve-static-core@4.19.0':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       '@types/qs': 6.9.7
       '@types/range-parser': 1.2.4
       '@types/send': 0.17.4
@@ -14368,11 +14396,11 @@ snapshots:
   '@types/glob@7.2.0':
     dependencies:
       '@types/minimatch': 5.1.2
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/graceful-fs@4.1.9':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/hast@2.3.10':
     dependencies:
@@ -14420,11 +14448,11 @@ snapshots:
 
   '@types/keyv@3.1.4':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/ldapjs@2.2.5':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/linkify-it@5.0.0': {}
 
@@ -14434,7 +14462,7 @@ snapshots:
 
   '@types/lz4@0.6.4':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/mapbox__point-geometry@0.1.4': {}
 
@@ -14473,12 +14501,12 @@ snapshots:
 
   '@types/node-fetch@2.6.11':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       form-data: 4.0.0
 
   '@types/node-forge@1.3.11':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/node-zendesk@2.0.15':
     dependencies:
@@ -14502,7 +14530,7 @@ snapshots:
 
   '@types/oauth@0.9.1':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/parse5@6.0.3': {}
 
@@ -14568,13 +14596,13 @@ snapshots:
   '@types/request@2.48.12':
     dependencies:
       '@types/caseless': 0.12.5
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       '@types/tough-cookie': 4.0.5
       form-data: 2.5.1
 
   '@types/responselike@1.0.3':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/retry@0.12.0': {}
 
@@ -14591,7 +14619,7 @@ snapshots:
   '@types/send@0.17.4':
     dependencies:
       '@types/mime': 1.3.5
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/serve-index@1.9.4':
     dependencies:
@@ -14600,19 +14628,19 @@ snapshots:
   '@types/serve-static@1.15.0':
     dependencies:
       '@types/mime': 3.0.1
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/serve-static@1.15.7':
     dependencies:
       '@types/http-errors': 2.0.4
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       '@types/send': 0.17.4
 
   '@types/sizzle@2.3.3': {}
 
   '@types/sockjs@0.3.36':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/stack-utils@2.0.3': {}
 
@@ -14643,20 +14671,20 @@ snapshots:
 
   '@types/ws@8.5.12':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/xml-crypto@1.4.6':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       xpath: 0.0.27
 
   '@types/xml-encryption@1.2.4':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/xml2js@0.4.14':
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
 
   '@types/yargs-parser@21.0.0': {}
 
@@ -16039,6 +16067,21 @@ snapshots:
       - supports-color
       - ts-node
 
+  create-jest@29.7.0(@types/node@18.19.55):
+    dependencies:
+      '@jest/types': 29.6.3
+      chalk: 4.1.2
+      exit: 0.1.2
+      graceful-fs: 4.2.11
+      jest-config: 29.7.0(@types/node@18.19.55)
+      jest-util: 29.7.0
+      prompts: 2.4.2
+    transitivePeerDependencies:
+      - '@types/node'
+      - babel-plugin-macros
+      - supports-color
+      - ts-node
+
   create-react-class@15.7.0:
     dependencies:
       loose-envify: 1.4.0
@@ -18697,7 +18740,7 @@ snapshots:
       '@jest/expect': 29.7.0
       '@jest/test-result': 29.7.0
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       chalk: 4.1.2
       co: 4.6.0
       dedent: 1.5.3
@@ -18736,6 +18779,25 @@ snapshots:
       - supports-color
       - ts-node
 
+  jest-cli@29.7.0(@types/node@18.19.55):
+    dependencies:
+      '@jest/core': 29.7.0
+      '@jest/test-result': 29.7.0
+      '@jest/types': 29.6.3
+      chalk: 4.1.2
+      create-jest: 29.7.0(@types/node@18.19.55)
+      exit: 0.1.2
+      import-local: 3.2.0
+      jest-config: 29.7.0(@types/node@18.19.55)
+      jest-util: 29.7.0
+      jest-validate: 29.7.0
+      yargs: 17.7.2
+    transitivePeerDependencies:
+      - '@types/node'
+      - babel-plugin-macros
+      - supports-color
+      - ts-node
+
   jest-config@29.7.0(@types/node@18.19.50):
     dependencies:
       '@babel/core': 7.25.8
@@ -18766,6 +18828,36 @@ snapshots:
       - babel-plugin-macros
       - supports-color
 
+  jest-config@29.7.0(@types/node@18.19.55):
+    dependencies:
+      '@babel/core': 7.25.8
+      '@jest/test-sequencer': 29.7.0
+      '@jest/types': 29.6.3
+      babel-jest: 29.7.0(@babel/core@7.25.8)
+      chalk: 4.1.2
+      ci-info: 3.9.0
+      deepmerge: 4.3.1
+      glob: 7.2.3
+      graceful-fs: 4.2.11
+      jest-circus: 29.7.0
+      jest-environment-node: 29.7.0
+      jest-get-type: 29.6.3
+      jest-regex-util: 29.6.3
+      jest-resolve: 29.7.0
+      jest-runner: 29.7.0
+      jest-util: 29.7.0
+      jest-validate: 29.7.0
+      micromatch: 4.0.8
+      parse-json: 5.2.0
+      pretty-format: 29.7.0
+      slash: 3.0.0
+      strip-json-comments: 3.1.1
+    optionalDependencies:
+      '@types/node': 18.19.55
+    transitivePeerDependencies:
+      - babel-plugin-macros
+      - supports-color
+
   jest-diff@26.6.2:
     dependencies:
       chalk: 4.1.2
@@ -18797,7 +18889,7 @@ snapshots:
       '@jest/environment': 29.7.0
       '@jest/fake-timers': 29.7.0
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       jest-mock: 29.7.0
       jest-util: 29.7.0
 
@@ -18809,7 +18901,7 @@ snapshots:
     dependencies:
       '@jest/types': 29.6.3
       '@types/graceful-fs': 4.1.9
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       anymatch: 3.1.3
       fb-watchman: 2.0.2
       graceful-fs: 4.2.11
@@ -18867,7 +18959,7 @@ snapshots:
   jest-mock@29.7.0:
     dependencies:
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       jest-util: 29.7.0
 
   jest-pnp-resolver@1.2.3(jest-resolve@29.7.0):
@@ -18904,7 +18996,7 @@ snapshots:
       '@jest/test-result': 29.7.0
       '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       chalk: 4.1.2
       emittery: 0.13.1
       graceful-fs: 4.2.11
@@ -18932,7 +19024,7 @@ snapshots:
       '@jest/test-result': 29.7.0
       '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       chalk: 4.1.2
       cjs-module-lexer: 1.2.3
       collect-v8-coverage: 1.0.2
@@ -18978,7 +19070,7 @@ snapshots:
   jest-util@29.7.0:
     dependencies:
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       chalk: 4.1.2
       ci-info: 3.9.0
       graceful-fs: 4.2.11
@@ -18997,7 +19089,7 @@ snapshots:
     dependencies:
       '@jest/test-result': 29.7.0
       '@jest/types': 29.6.3
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       ansi-escapes: 4.3.2
       chalk: 4.1.2
       emittery: 0.13.1
@@ -19012,7 +19104,7 @@ snapshots:
 
   jest-worker@29.7.0:
     dependencies:
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       jest-util: 29.7.0
       merge-stream: 2.0.0
       supports-color: 8.1.1
@@ -19029,6 +19121,18 @@ snapshots:
       - supports-color
       - ts-node
 
+  jest@29.7.0(@types/node@18.19.55):
+    dependencies:
+      '@jest/core': 29.7.0
+      '@jest/types': 29.6.3
+      import-local: 3.2.0
+      jest-cli: 29.7.0(@types/node@18.19.55)
+    transitivePeerDependencies:
+      - '@types/node'
+      - babel-plugin-macros
+      - supports-color
+      - ts-node
+
   jmp@2.0.0:
     dependencies:
       uuid: 3.4.0
@@ -21155,7 +21259,7 @@ snapshots:
       '@protobufjs/path': 1.1.2
       '@protobufjs/pool': 1.1.0
       '@protobufjs/utf8': 1.1.0
-      '@types/node': 18.19.50
+      '@types/node': 18.19.55
       long: 5.2.3
 
   protocol-buffers-schema@3.6.0: {}
@@ -22992,6 +23096,25 @@ snapshots:
       '@jest/types': 29.6.3
       babel-jest: 29.7.0(@babel/core@7.25.8)
 
+  ts-jest@29.2.5(@babel/core@7.25.8)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.8))(jest@29.7.0(@types/node@18.19.55))(typescript@5.6.3):
+    dependencies:
+      bs-logger: 0.2.6
+      ejs: 3.1.10
+      fast-json-stable-stringify: 2.1.0
+      jest: 29.7.0(@types/node@18.19.55)
+      jest-util: 29.7.0
+      json5: 2.2.3
+      lodash.memoize: 4.1.2
+      make-error: 1.3.6
+      semver: 7.6.3
+      typescript: 5.6.3
+      yargs-parser: 21.1.1
+    optionalDependencies:
+      '@babel/core': 7.25.8
+      '@jest/transform': 29.7.0
+      '@jest/types': 29.6.3
+      babel-jest: 29.7.0(@babel/core@7.25.8)
+
   tsd@0.22.0:
     dependencies:
       '@tsd/typescript': 4.7.4
diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts
index 23e5a601b0..810bda05c1 100644
--- a/src/packages/project/browser-websocket/api.ts
+++ b/src/packages/project/browser-websocket/api.ts
@@ -22,7 +22,7 @@ import { lean, lean_channel } from "../lean/server";
 import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse";
 import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run";
 import { synctable_channel } from "../sync/server";
-import { callSyncDoc } from "@cocalc/sync/server/syncdocs-manager";
+import { callSyncDoc } from "@cocalc/sync-server/syncdocs-manager";
 import { terminal } from "@cocalc/terminal";
 import { x11_channel } from "../x11/server";
 import { canonical_paths } from "./canonical-path";
diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts
index 83af21f584..6b92afc266 100644
--- a/src/packages/project/client.ts
+++ b/src/packages/project/client.ts
@@ -46,8 +46,8 @@ import * as kucalc from "./kucalc";
 import { getLogger } from "./logger";
 import * as sage_session from "./sage_session";
 import { getListingsTable } from "@cocalc/project/sync/listings";
-import { get_synctable } from "@cocalc/sync/server/open-synctables";
-import { getSyncDoc } from "@cocalc/sync/server/syncdocs-manager";
+import { get_synctable } from "@cocalc/sync-server/open-synctables";
+import { getSyncDoc } from "@cocalc/sync-server/syncdocs-manager";
 
 const winston = getLogger("client");
 
diff --git a/src/packages/project/package.json b/src/packages/project/package.json
index 044e8c8f4e..0dcb5dcc02 100644
--- a/src/packages/project/package.json
+++ b/src/packages/project/package.json
@@ -27,6 +27,7 @@
     "@cocalc/project": "workspace:*",
     "@cocalc/sync": "workspace:*",
     "@cocalc/sync-client": "workspace:*",
+    "@cocalc/sync-server": "workspace:*",
     "@cocalc/sync-fs": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
@@ -82,9 +83,7 @@
     "clean": "rm -rf dist"
   },
   "author": "SageMath, Inc.",
-  "contributors": [
-    "William Stein <wstein@sagemath.com>"
-  ],
+  "contributors": ["William Stein <wstein@sagemath.com>"],
   "license": "SEE LICENSE.md",
   "bugs": {
     "url": "https://github.com/sagemathinc/cocalc/issues"
diff --git a/src/packages/project/project-info/server.ts b/src/packages/project/project-info/server.ts
index 2d7466be87..61e405fce5 100644
--- a/src/packages/project/project-info/server.ts
+++ b/src/packages/project/project-info/server.ts
@@ -13,7 +13,6 @@ import type { DiskUsage as DF_DiskUsage } from "diskusage";
 import { check as df } from "diskusage";
 import { EventEmitter } from "node:events";
 import { readFile } from "node:fs/promises";
-
 import { ProcessStats } from "@cocalc/backend/process-stats";
 import { get_kernel_by_pid } from "@cocalc/jupyter/kernel";
 import { pidToPath as terminalPidToPath } from "@cocalc/terminal";
diff --git a/src/packages/project/sync/compute-server-open-file-tracking.ts b/src/packages/project/sync/compute-server-open-file-tracking.ts
index 6d74a6849a..13aee7273b 100644
--- a/src/packages/project/sync/compute-server-open-file-tracking.ts
+++ b/src/packages/project/sync/compute-server-open-file-tracking.ts
@@ -4,7 +4,7 @@ Manage the state of open files in the compute servers syncdb sync'd file.
 NOTE: terminals aren't handled at all here, since they don't have a syncdoc.
 */
 
-import type { SyncDocs } from "@cocalc/sync/server/syncdocs-manager";
+import type { SyncDocs } from "@cocalc/sync-server/syncdocs-manager";
 import type { SyncDB } from "@cocalc/sync/editor/db/sync";
 import { once } from "@cocalc/util/async-utils";
 import { meta_file, auxFileToOriginal } from "@cocalc/util/misc";
diff --git a/src/packages/project/sync/listings.ts b/src/packages/project/sync/listings.ts
index 4a5ebf2c58..f8693b766a 100644
--- a/src/packages/project/sync/listings.ts
+++ b/src/packages/project/sync/listings.ts
@@ -9,7 +9,7 @@ import {
 } from "@cocalc/sync/listings";
 import getListing from "@cocalc/backend/get-listing";
 import { Watcher } from "@cocalc/backend/path-watcher";
-import { closeAllSyncDocsInTree } from "@cocalc/sync/server/syncdocs-manager";
+import { closeAllSyncDocsInTree } from "@cocalc/sync-server/syncdocs-manager";
 import { getLogger } from "@cocalc/backend/logger";
 import { existsSync } from "fs";
 
diff --git a/src/packages/project/sync/server.ts b/src/packages/project/sync/server.ts
index dce5ab6d72..045745fc84 100644
--- a/src/packages/project/sync/server.ts
+++ b/src/packages/project/sync/server.ts
@@ -51,7 +51,7 @@ import {
 // @ts-ignore -- typescript nonsense.
 const _ = set_debug;
 
-import { key, register_synctable } from "@cocalc/sync/server/open-synctables";
+import { key, register_synctable } from "@cocalc/sync-server/open-synctables";
 import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { once } from "@cocalc/util/async-utils";
 import { delay } from "awaiting";
@@ -71,7 +71,7 @@ import {
   initSyncDoc,
   getSyncDocFromSyncTable,
   initSyncDocsManager,
-} from "@cocalc/sync/server/syncdocs-manager";
+} from "@cocalc/sync-server/syncdocs-manager";
 
 import computeServerOpenFileTracking from "./compute-server-open-file-tracking";
 import { getLogger } from "@cocalc/backend/logger";
diff --git a/src/packages/project/tsconfig.json b/src/packages/project/tsconfig.json
index 4e6d052ed7..c2a3f0e822 100644
--- a/src/packages/project/tsconfig.json
+++ b/src/packages/project/tsconfig.json
@@ -16,6 +16,7 @@
     { "path": "../sync" },
     { "path": "../sync-client" },
     { "path": "../sync-fs" },
+    { "path": "../sync-server" },
     { "path": "../terminal" },
     { "path": "../util" }
   ]
diff --git a/src/packages/sync/server/README.md b/src/packages/sync-server/README.md
similarity index 100%
rename from src/packages/sync/server/README.md
rename to src/packages/sync-server/README.md
diff --git a/src/packages/sync/server/open-synctables.ts b/src/packages/sync-server/open-synctables.ts
similarity index 100%
rename from src/packages/sync/server/open-synctables.ts
rename to src/packages/sync-server/open-synctables.ts
diff --git a/src/packages/sync-server/package.json b/src/packages/sync-server/package.json
new file mode 100644
index 0000000000..c56931acf2
--- /dev/null
+++ b/src/packages/sync-server/package.json
@@ -0,0 +1,35 @@
+{
+  "name": "@cocalc/sync-server",
+  "version": "0.1.0",
+  "description": "CoCalc realtime synchronization framework -- server component",
+  "exports": {
+    "./*": "./dist/*.js"
+  },
+  "scripts": {
+    "clean": "rm -rf dist",
+    "preinstall": "npx only-allow pnpm",
+    "build": "../node_modules/.bin/tsc --build",
+    "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput",
+    "prepublishOnly": "pnpm test"
+  },
+  "files": ["dist/**", "bin/**", "README.md", "package.json"],
+  "author": "SageMath, Inc.",
+  "keywords": ["cocalc", "realtime synchronization"],
+  "license": "SEE LICENSE.md",
+  "dependencies": {
+    "@cocalc/backend": "workspace:*",
+    "@cocalc/sync-server": "workspace:*",
+    "@cocalc/sync": "workspace:*",
+    "@cocalc/util": "workspace:*",
+    "events": "3.3.0",
+    "awaiting": "^3.0.0"
+  },
+  "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/sync-server",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/sagemathinc/cocalc"
+  },
+  "devDependencies": {
+    "@types/node": "^18.16.14"
+  }
+}
diff --git a/src/packages/sync/server/syncdocs-manager.ts b/src/packages/sync-server/syncdocs-manager.ts
similarity index 100%
rename from src/packages/sync/server/syncdocs-manager.ts
rename to src/packages/sync-server/syncdocs-manager.ts
diff --git a/src/packages/sync-server/tsconfig.json b/src/packages/sync-server/tsconfig.json
new file mode 100644
index 0000000000..6424448b9a
--- /dev/null
+++ b/src/packages/sync-server/tsconfig.json
@@ -0,0 +1,9 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./",
+    "outDir": "dist"
+  },
+  "exclude": ["node_modules", "dist", "test"],
+  "references": [{ "path": "../util", "path": "../backend", "path": "../sync" }]
+}
diff --git a/src/workspaces.py b/src/workspaces.py
index a07aa4eca3..92cd25dcd6 100755
--- a/src/workspaces.py
+++ b/src/workspaces.py
@@ -113,6 +113,7 @@ def all_packages() -> List[str]:
         'packages/api-client',
         'packages/jupyter',
         'packages/comm',
+        'packages/sync-server',
         'packages/assets',
         'packages/frontend',  # static depends on frontend; frontend depends on assets
         'packages/project',  # project depends on frontend for nbconvert (but NEVER vice versa again), which also depends on assets

From 997b42dbe945758ba167bd42e00e69838b485a4d Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 15:26:07 +0000
Subject: [PATCH 23/35] jupyter: fallback to raw cell instead of crashing
 cocalc, in case of invalid cell type

---
 src/packages/jupyter/redux/store.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts
index 82525e3f03..073fe23d02 100644
--- a/src/packages/jupyter/redux/store.ts
+++ b/src/packages/jupyter/redux/store.ts
@@ -136,7 +136,10 @@ export class JupyterStore extends Store<JupyterStoreState> {
     // TODO: We use unsafe_getIn because maybe the cell type isn't spelled out yet, or our typescript isn't good enough.
     const type = this.unsafe_getIn(["cells", id, "cell_type"], "code");
     if (type != "markdown" && type != "code" && type != "raw") {
-      throw Error(`invalid cell type ${type} for cell ${id}`);
+      console.warn(
+        `Jupyter: invalid cell type ${type} for cell ${id} -- falling back to raw`,
+      );
+      return "raw";
     }
     return type;
   }

From ebca01048961530acbf8dcb6de83718af166351d Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 15:26:39 +0000
Subject: [PATCH 24/35] making sync-server package slightly better (more deps)

---
 src/packages/pnpm-lock.yaml                   |  6 ++
 src/packages/project/sync/server.ts           | 15 +---
 .../compute-server-open-file-tracking.ts      |  0
 src/packages/sync-server/package.json         |  2 +
 src/packages/sync-server/syncdocs-manager.ts  | 90 ++++++++-----------
 src/packages/sync-server/tsconfig.json        | 10 ++-
 6 files changed, 56 insertions(+), 67 deletions(-)
 rename src/packages/{project/sync => sync-server}/compute-server-open-file-tracking.ts (100%)

diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml
index d58b2c50eb..4760d177a4 100644
--- a/src/packages/pnpm-lock.yaml
+++ b/src/packages/pnpm-lock.yaml
@@ -1914,12 +1914,18 @@ importers:
       '@cocalc/backend':
         specifier: workspace:*
         version: link:../backend
+      '@cocalc/jupyter':
+        specifier: workspace:*
+        version: link:../jupyter
       '@cocalc/sync':
         specifier: workspace:*
         version: link:../sync
       '@cocalc/sync-server':
         specifier: workspace:*
         version: 'link:'
+      '@cocalc/terminal':
+        specifier: workspace:*
+        version: link:../terminal
       '@cocalc/util':
         specifier: workspace:*
         version: link:../util
diff --git a/src/packages/project/sync/server.ts b/src/packages/project/sync/server.ts
index 045745fc84..ade3cb64d1 100644
--- a/src/packages/project/sync/server.ts
+++ b/src/packages/project/sync/server.ts
@@ -62,25 +62,12 @@ import { register_project_status_table } from "./project-status";
 import { register_usage_info_table } from "./usage-info";
 import type { MergeType } from "@cocalc/sync/table/synctable";
 import Client from "@cocalc/sync-client";
-import {
-  getJupyterRedux,
-  initJupyterRedux,
-  removeJupyterRedux,
-} from "@cocalc/jupyter/kernel";
+import { getJupyterRedux } from "@cocalc/jupyter/kernel";
 import {
   initSyncDoc,
   getSyncDocFromSyncTable,
-  initSyncDocsManager,
 } from "@cocalc/sync-server/syncdocs-manager";
 
-import computeServerOpenFileTracking from "./compute-server-open-file-tracking";
-import { getLogger } from "@cocalc/backend/logger";
-initSyncDocsManager({
-  logger: getLogger("project:sync:sync-doc"),
-  computeServerOpenFileTracking,
-  jupyter: { initJupyterRedux, removeJupyterRedux },
-});
-
 type Query = { [key: string]: any };
 
 interface Spark {
diff --git a/src/packages/project/sync/compute-server-open-file-tracking.ts b/src/packages/sync-server/compute-server-open-file-tracking.ts
similarity index 100%
rename from src/packages/project/sync/compute-server-open-file-tracking.ts
rename to src/packages/sync-server/compute-server-open-file-tracking.ts
diff --git a/src/packages/sync-server/package.json b/src/packages/sync-server/package.json
index c56931acf2..34828b1a5e 100644
--- a/src/packages/sync-server/package.json
+++ b/src/packages/sync-server/package.json
@@ -18,8 +18,10 @@
   "license": "SEE LICENSE.md",
   "dependencies": {
     "@cocalc/backend": "workspace:*",
+    "@cocalc/jupyter": "workspace:*",
     "@cocalc/sync-server": "workspace:*",
     "@cocalc/sync": "workspace:*",
+    "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
     "events": "3.3.0",
     "awaiting": "^3.0.0"
diff --git a/src/packages/sync-server/syncdocs-manager.ts b/src/packages/sync-server/syncdocs-manager.ts
index 78f17b8568..eac66e49d6 100644
--- a/src/packages/sync-server/syncdocs-manager.ts
+++ b/src/packages/sync-server/syncdocs-manager.ts
@@ -27,21 +27,11 @@ import { once } from "@cocalc/util/async-utils";
 import { filename_extension, original_path } from "@cocalc/util/misc";
 import { EventEmitter } from "events";
 import { COMPUTER_SERVER_DB_NAME } from "@cocalc/util/compute/manager";
+import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel";
+import computeServerOpenFileTracking from "./compute-server-open-file-tracking";
+import { getLogger } from "@cocalc/backend/logger";
 
-// This must be called externally to initialize the logger and open file tracker,
-// if you need that functionality
-let logger: undefined | { debug: Function } = undefined;
-let computeServerOpenFileTracking: undefined | Function = undefined;
-let jupyter: undefined | { initJupyterRedux; removeJupyterRedux } = undefined;
-export function initSyncDocsManager(opts: {
-  logger?;
-  computeServerOpenFileTracking?;
-  jupyter?;
-}) {
-  logger = opts.logger;
-  computeServerOpenFileTracking = opts.computeServerOpenFileTracking;
-  jupyter = opts.jupyter;
-}
+const logger = getLogger("project:sync:sync-doc");
 
 type SyncDoc = SyncDB | SyncString;
 
@@ -55,11 +45,11 @@ export class SyncDocs extends EventEmitter {
   async close(path: string): Promise<void> {
     const doc = this.get(path);
     if (doc == null) {
-      logger?.debug(`SyncDocs: close ${path} -- no need, as it is not opened`);
+      logger.debug(`SyncDocs: close ${path} -- no need, as it is not opened`);
       return;
     }
     try {
-      logger?.debug(`SyncDocs: close ${path} -- starting close`);
+      logger.debug(`SyncDocs: close ${path} -- starting close`);
       this.closing.add(path);
       // As soon as this close starts, doc is in an undefined state.
       // Also, this can take an **unbounded** amount of time to finish,
@@ -79,16 +69,14 @@ export class SyncDocs extends EventEmitter {
       // track down heisenbug. See also
       // https://github.com/sagemathinc/cocalc/issues/5617
       await doc.close();
-      logger?.debug(`SyncDocs: close ${path} -- successfully closed`);
+      logger.debug(`SyncDocs: close ${path} -- successfully closed`);
     } finally {
       // No matter what happens above when it finishes, we clear it
       // and consider it closed.
       // There is perhaps a chance closing fails above (no idea how),
       // but we don't want it to be impossible to attempt to open
       // the path again I.e., we don't want to leave around a lock.
-      logger?.debug(
-        `SyncDocs: close ${path} -- recording that close succeeded`,
-      );
+      logger.debug(`SyncDocs: close ${path} -- recording that close succeeded`);
       delete this.syncdocs[path];
       this.closing.delete(path);
       // I think close-${path} is used only internally in this.create below
@@ -113,11 +101,11 @@ export class SyncDocs extends EventEmitter {
   async create(type, opts): Promise<SyncDoc> {
     const path = opts.path;
     if (this.closing.has(path)) {
-      logger?.debug(
+      logger.debug(
         `SyncDocs: create ${path} -- waiting for previous version to completely finish closing...`,
       );
       await once(this, `close-${path}`);
-      logger?.debug(`SyncDocs: create ${path} -- successfully closed.`);
+      logger.debug(`SyncDocs: create ${path} -- successfully closed.`);
     }
     let doc;
     switch (type) {
@@ -131,14 +119,11 @@ export class SyncDocs extends EventEmitter {
         throw Error(`unknown syncdoc type ${type}`);
     }
     this.syncdocs[path] = doc;
-    logger?.debug(`SyncDocs: create ${path} -- successfully created`);
+    logger.debug(`SyncDocs: create ${path} -- successfully created`);
     // This is used by computeServerOpenFileTracking:
     this.emit("open", path);
-    if (
-      computeServerOpenFileTracking != null &&
-      path == COMPUTER_SERVER_DB_NAME
-    ) {
-      logger?.debug(
+    if (path == COMPUTER_SERVER_DB_NAME) {
+      logger.debug(
         "SyncDocs: also initializing open file tracking for ",
         COMPUTER_SERVER_DB_NAME,
       );
@@ -148,7 +133,7 @@ export class SyncDocs extends EventEmitter {
   }
 
   async closeAll(filename: string): Promise<void> {
-    logger?.debug(`SyncDocs: closeAll("${filename}")`);
+    logger.debug(`SyncDocs: closeAll("${filename}")`);
     for (const path in this.syncdocs) {
       if (path == filename || path.startsWith(filename + "/")) {
         await this.close(path);
@@ -194,14 +179,18 @@ async function initSyncDocAsync(
   synctable: SyncTable,
 ): Promise<void> {
   function log(...args): void {
-    logger?.debug("initSyncDocAsync: ", ...args);
+    logger.debug("initSyncDocAsync: ", ...args);
   }
 
   log("waiting until synctable is ready");
   await waitUntilSyncTableReady(synctable);
   log("synctable ready.  Now getting type and opts");
   const { type, opts } = getTypeAndOpts(synctable);
-  const project_id = (opts.project_id = client.client_id());
+  const project_id = client.client_id();
+  if (project_id == null) {
+    throw Error("client_id must be defined");
+  }
+  opts.project_id = project_id;
   //   log("type = ", type);
   //   log("opts = ", JSON.stringify(opts));
   opts.client = client;
@@ -229,31 +218,28 @@ async function initSyncDocAsync(
   log("ext = ", ext);
   switch (ext) {
     case "sage-jupyter2":
-      if (jupyter != null) {
-        const { initJupyterRedux, removeJupyterRedux } = jupyter;
-        log("initializing Jupyter backend");
-        await initJupyterRedux(syncdoc, client);
-        const path = original_path(syncdoc.get_path());
-        synctable.on("closed", async () => {
-          log("removing Jupyter backend");
-          await removeJupyterRedux(path, project_id);
-        });
-      }
+      log("initializing Jupyter backend");
+      await initJupyterRedux(syncdoc, client);
+      const path = original_path(syncdoc.get_path());
+      synctable.on("closed", async () => {
+        log("removing Jupyter backend");
+        await removeJupyterRedux(path, project_id);
+      });
       break;
   }
 }
 
 async function waitUntilSyncTableReady(synctable: SyncTable): Promise<void> {
   if (synctable.get_state() == "disconnected") {
-    logger?.debug("waitUntilSyncTableReady: wait for synctable be connected");
+    logger.debug("waitUntilSyncTableReady: wait for synctable be connected");
     await once(synctable, "connected");
   }
 
   const t = synctable.get_one();
   if (t != null) {
-    logger?.debug("waitUntilSyncTableReady: currently", t.toJS());
+    logger.debug("waitUntilSyncTableReady: currently", t.toJS());
   }
-  logger?.debug(
+  logger.debug(
     "waitUntilSyncTableReady: wait for document info to get loaded into synctable...",
   );
   // Next wait until there's a document in the synctable, since that will
@@ -261,15 +247,15 @@ async function waitUntilSyncTableReady(synctable: SyncTable): Promise<void> {
   function is_ready(): boolean {
     const t = synctable.get_one();
     if (t == null) {
-      logger?.debug("waitUntilSyncTableReady: is_ready: table is null still");
+      logger.debug("waitUntilSyncTableReady: is_ready: table is null still");
       return false;
     } else {
-      logger?.debug("waitUntilSyncTableReady: is_ready", JSON.stringify(t));
+      logger.debug("waitUntilSyncTableReady: is_ready", JSON.stringify(t));
       return t.has("path");
     }
   }
   await synctable.wait(is_ready, 0);
-  logger?.debug("waitUntilSyncTableReady: document info is now in synctable");
+  logger.debug("waitUntilSyncTableReady: document info is now in synctable");
 }
 
 function getTypeAndOpts(synctable: SyncTable): { type: string; opts: any } {
@@ -306,17 +292,17 @@ function getTypeAndOpts(synctable: SyncTable): { type: string; opts: any } {
 }
 
 export async function callSyncDoc(path: string, mesg: any): Promise<string> {
-  logger?.debug("callSyncDoc", path, mesg);
+  logger.debug("callSyncDoc", path, mesg);
   const doc = syncDocs.get(path);
   if (doc == null) {
-    logger?.debug("callSyncDoc -- not open: ", path);
+    logger.debug("callSyncDoc -- not open: ", path);
     return "not open";
   }
   switch (mesg.cmd) {
     case "close":
-      logger?.debug("callSyncDoc -- now closing: ", path);
+      logger.debug("callSyncDoc -- now closing: ", path);
       await syncDocs.close(path);
-      logger?.debug("callSyncDoc -- closed: ", path);
+      logger.debug("callSyncDoc -- closed: ", path);
       return "successfully closed";
     default:
       throw Error(`unknown command ${mesg.cmd}`);
@@ -326,6 +312,6 @@ export async function callSyncDoc(path: string, mesg: any): Promise<string> {
 // This is used when deleting a file/directory
 // filename may be a directory or actual filename
 export async function closeAllSyncDocsInTree(filename: string): Promise<void> {
-  logger?.debug("closeAllSyncDocsInTree", filename);
+  logger.debug("closeAllSyncDocsInTree", filename);
   return await syncDocs.closeAll(filename);
 }
diff --git a/src/packages/sync-server/tsconfig.json b/src/packages/sync-server/tsconfig.json
index 6424448b9a..e2e9ff42d7 100644
--- a/src/packages/sync-server/tsconfig.json
+++ b/src/packages/sync-server/tsconfig.json
@@ -5,5 +5,13 @@
     "outDir": "dist"
   },
   "exclude": ["node_modules", "dist", "test"],
-  "references": [{ "path": "../util", "path": "../backend", "path": "../sync" }]
+  "references": [
+    {
+      "path": "../util",
+      "path": "../backend",
+      "path": "../jupyter",
+      "path": "../sync",
+      "path": "../terminal"
+    }
+  ]
 }

From 516952a2223e0890625b4a87b8fd0a419b323544 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 15:46:41 +0000
Subject: [PATCH 25/35] fix #7942 -- jupyter cell toolbar: make assistant
 button work same as run -- it's a button with an optional dropdown, and the
 button brings up the full assistant dialog with proper context, focused on
 this cell.

---
 .../frame-tree/commands/generic-commands.tsx  |  3 +
 .../frame-editors/frame-tree/title-bar.tsx    |  5 ++
 .../frontend/i18n/trans/extracted.json        |  2 +-
 .../frontend/jupyter/llm/cell-tool.tsx        | 84 +++++++++++--------
 src/packages/jupyter/redux/actions.ts         |  2 +-
 5 files changed, 60 insertions(+), 36 deletions(-)

diff --git a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx
index 4edf6e656a..85039a7633 100644
--- a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx
+++ b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx
@@ -1064,6 +1064,9 @@ addCommands({
     label: labels.save,
     keyboard: `${IS_MACOS ? "⌘" : "control"} + S`,
   },
+  // DO NOT just change the name from chatgpt to something else, e.g.,
+  // this is explicitly used in /frontend/jupyter/llm/cell-tool.tsx
+  // to search for and run the command.
   chatgpt: {
     pos: 1,
     group: "show-frames",
diff --git a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx
index 5823e29e41..4374a26357 100644
--- a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx
+++ b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx
@@ -84,6 +84,10 @@ export interface FrameActions extends Actions {
   // optional, set in frame-editors/jupyter-editor/editor.ts → initMenus
   jupyter_actions?: JupyterActions;
   frame_actions?: NotebookFrameActions;
+
+  // the menu bar and buttons - can be used to explicitly run any menu command
+  // programatically, etc.
+  manageCommands?;
 }
 
 interface EditorActions extends Actions {
@@ -251,6 +255,7 @@ export function FrameTitleBar(props: FrameTitleBarProps) {
       intl,
     ],
   );
+  props.actions.manageCommands = manageCommands;
 
   const has_unsaved_changes: boolean = useRedux([
     props.editor_actions.name,
diff --git a/src/packages/frontend/i18n/trans/extracted.json b/src/packages/frontend/i18n/trans/extracted.json
index 7c4479144d..816efe4991 100644
--- a/src/packages/frontend/i18n/trans/extracted.json
+++ b/src/packages/frontend/i18n/trans/extracted.json
@@ -1024,7 +1024,7 @@
     "defaultMessage": "Translate"
   },
   "jupyter.llm.cell-tool.assistant.title": {
-    "defaultMessage": "Use AI assitant on this cell"
+    "defaultMessage": "Use AI assistant on this cell"
   },
   "jupyter.llm.cell-tool.explanation.bugfix": {
     "defaultMessage": "Explain the problem of the code in the cell and the selected language model will attempt to fix it. Usually, it will tell you if it found a problem and explain it to you."
diff --git a/src/packages/frontend/jupyter/llm/cell-tool.tsx b/src/packages/frontend/jupyter/llm/cell-tool.tsx
index 6b004974f0..7ff1bd4894 100644
--- a/src/packages/frontend/jupyter/llm/cell-tool.tsx
+++ b/src/packages/frontend/jupyter/llm/cell-tool.tsx
@@ -431,49 +431,65 @@ export function LLMCellTool({ actions, id, style, llmTools }: Props) {
 
   function renderDropdown() {
     return (
-      <Dropdown
-        trigger={["click"]}
-        mouseLeaveDelay={1.5}
-        menu={{
-          items: (Object.entries(ACTIONS) as Entries<typeof ACTIONS>).map(
-            ([mode, action]) => {
-              return {
-                key: mode,
-                label: (
-                  <Tooltip
-                    title={intl.formatMessage(action.descr)}
-                    placement={"left"}
-                  >
-                    <Icon name={action.icon} style={{ marginRight: "5px" }} />{" "}
-                    {intl.formatMessage(action.label)}
-                  </Tooltip>
-                ),
-                onClick: () => setMode(mode as Mode),
-              };
-            },
-          ),
-        }}
+      <Tooltip
+        title={intl.formatMessage({
+          id: "jupyter.llm.cell-tool.assistant.title",
+          defaultMessage: "Use AI assistant on this cell",
+        })}
       >
-        <Tooltip
-          title={intl.formatMessage({
-            id: "jupyter.llm.cell-tool.assistant.title",
-            defaultMessage: "Use AI assitant on this cell",
-          })}
-        >
+        <Space.Compact>
           <Button
             disabled={isQuerying}
             type="text"
             size="small"
             style={CODE_BAR_BTN_STYLE}
             icon={<AIAvatar size={14} style={{ top: "1px" }} />}
+            onClick={() => {
+              actions
+                ?.getFrameActions()
+                ?.manageCommands?.searchCommands("chatgpt")[0]
+                ?.onClick();
+            }}
           >
-            <Space size="small">
-              {intl.formatMessage(labels.assistant)}
-              <Icon name="angle-down" />
-            </Space>
+            {intl.formatMessage(labels.assistant)}
           </Button>
-        </Tooltip>
-      </Dropdown>
+          <Dropdown
+            trigger={["click"]}
+            mouseLeaveDelay={1.5}
+            menu={{
+              items: (Object.entries(ACTIONS) as Entries<typeof ACTIONS>).map(
+                ([mode, action]) => {
+                  return {
+                    key: mode,
+                    label: (
+                      <Tooltip
+                        title={intl.formatMessage(action.descr)}
+                        placement={"left"}
+                      >
+                        <Icon
+                          name={action.icon}
+                          style={{ marginRight: "5px" }}
+                        />{" "}
+                        {intl.formatMessage(action.label)}
+                      </Tooltip>
+                    ),
+                    onClick: () => setMode(mode as Mode),
+                  };
+                },
+              ),
+            }}
+          >
+            <Button
+              disabled={isQuerying}
+              type="text"
+              size="small"
+              style={CODE_BAR_BTN_STYLE}
+            >
+              <Icon name="angle-down" />
+            </Button>
+          </Dropdown>
+        </Space.Compact>
+      </Tooltip>
     );
   }
 
diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts
index d5bccb0d07..f02d32608f 100644
--- a/src/packages/jupyter/redux/actions.ts
+++ b/src/packages/jupyter/redux/actions.ts
@@ -159,7 +159,7 @@ export abstract class JupyterActions extends Actions<JupyterStoreState> {
   }
 
   // Only use this on the frontend, of course.
-  protected getFrameActions() {
+  getFrameActions() {
     return this.redux.getEditorActions(this.project_id, this.path);
   }
 

From e9227fa2b63d2b2b9f9afd6db025b89e6e0c477b Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 16:02:09 +0000
Subject: [PATCH 26/35] sync-server refactor: move x11 channel management

---
 src/packages/project/browser-websocket/api.ts              | 2 +-
 src/packages/project/project-info/server.ts                | 3 +--
 src/packages/{project/x11/server.ts => sync-server/x11.ts} | 6 +++---
 3 files changed, 5 insertions(+), 6 deletions(-)
 rename src/packages/{project/x11/server.ts => sync-server/x11.ts} (95%)

diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts
index 810bda05c1..701c4bc638 100644
--- a/src/packages/project/browser-websocket/api.ts
+++ b/src/packages/project/browser-websocket/api.ts
@@ -24,7 +24,7 @@ import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run";
 import { synctable_channel } from "../sync/server";
 import { callSyncDoc } from "@cocalc/sync-server/syncdocs-manager";
 import { terminal } from "@cocalc/terminal";
-import { x11_channel } from "../x11/server";
+import { x11_channel } from "@cocalc/sync-server/x11";
 import { canonical_paths } from "./canonical-path";
 import { delete_files } from "@cocalc/backend/files/delete-files";
 import { eval_code } from "@cocalc/backend/eval-code";
diff --git a/src/packages/project/project-info/server.ts b/src/packages/project/project-info/server.ts
index 61e405fce5..c776e29e89 100644
--- a/src/packages/project/project-info/server.ts
+++ b/src/packages/project/project-info/server.ts
@@ -24,8 +24,7 @@ import {
   Processes,
   ProjectInfo,
 } from "@cocalc/util/types/project-info/types";
-import { get_path_for_pid as x11_pid2path } from "../x11/server";
-//import { get_sage_path } from "../sage_session"
+import { get_path_for_pid as x11_pid2path } from "@cocalc/sync-server/x11";
 import { getLogger } from "../logger";
 
 const L = getLogger("project-info:server").debug;
diff --git a/src/packages/project/x11/server.ts b/src/packages/sync-server/x11.ts
similarity index 95%
rename from src/packages/project/x11/server.ts
rename to src/packages/sync-server/x11.ts
index a7ac85f4a8..457cc7329d 100644
--- a/src/packages/project/x11/server.ts
+++ b/src/packages/sync-server/x11.ts
@@ -4,17 +4,17 @@
  */
 
 /*
-X11 server channel.
+X11 server channel.  This is implemented under the hood using the standard 
+xpra server. Frontends use a custom xpra client we wrote.
 
 TODO:
   - [ ] other user activity
-  - [ ] when stopping project, kill xpra's
+  - [ ] when stopping sync server, kill xpra sessions
 */
 
 import { spawn, SpawnOptions } from "node:child_process";
 import { callback } from "awaiting";
 import { clone } from "lodash";
-
 import abspath from "@cocalc/backend/misc/abspath";
 import { path_split } from "@cocalc/util/misc";
 

From 9451f48ae8751d515f65179496251009274f98c5 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 16:19:24 +0000
Subject: [PATCH 27/35] refactoring work: project-info -->
 sync-server/monitor/activity

- needed to support this functionality outside of the home base project
---
 src/packages/pnpm-lock.yaml                   | 30 +++++++++++--------
 src/packages/project/autorenice.ts            |  5 +++-
 src/packages/project/browser-websocket/api.ts |  2 +-
 src/packages/project/package.json             |  7 +++--
 src/packages/project/project-status/server.ts |  2 +-
 src/packages/project/sync/project-info.ts     | 14 +++++----
 src/packages/project/usage-info/server.ts     |  7 ++---
 src/packages/sync-server/monitor/README.md    |  9 ++++++
 .../monitor/activity}/index.ts                |  0
 .../monitor/activity}/project-info.ts         |  0
 .../monitor/activity}/server.ts               |  4 +--
 .../monitor/activity}/utils.ts                |  0
 src/packages/sync-server/package.json         | 10 +++++--
 13 files changed, 56 insertions(+), 34 deletions(-)
 create mode 100644 src/packages/sync-server/monitor/README.md
 rename src/packages/{project/project-info => sync-server/monitor/activity}/index.ts (100%)
 rename src/packages/{project/project-info => sync-server/monitor/activity}/project-info.ts (100%)
 rename src/packages/{project/project-info => sync-server/monitor/activity}/server.ts (98%)
 rename src/packages/{project/project-info => sync-server/monitor/activity}/utils.ts (100%)

diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml
index 4760d177a4..96c2e9c5f6 100644
--- a/src/packages/pnpm-lock.yaml
+++ b/src/packages/pnpm-lock.yaml
@@ -1226,10 +1226,7 @@ importers:
         version: 3.0.0
       debug:
         specifier: ^4.3.2
-        version: 4.3.7(supports-color@8.1.1)
-      diskusage:
-        specifier: ^1.1.3
-        version: 1.2.0
+        version: 4.3.7
       expect:
         specifier: ^26.6.2
         version: 26.6.2
@@ -1932,10 +1929,19 @@ importers:
       awaiting:
         specifier: ^3.0.0
         version: 3.0.0
+      diskusage:
+        specifier: ^1.1.3
+        version: 1.2.0
       events:
         specifier: 3.3.0
         version: 3.3.0
+      lodash:
+        specifier: ^4.17.21
+        version: 4.17.21
     devDependencies:
+      '@types/lodash':
+        specifier: ^4.14.202
+        version: 4.17.9
       '@types/node':
         specifier: ^18.16.14
         version: 18.19.55
@@ -8903,9 +8909,6 @@ packages:
   nan@2.17.0:
     resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==}
 
-  nan@2.19.0:
-    resolution: {integrity: sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==}
-
   nan@2.20.0:
     resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==}
 
@@ -12637,7 +12640,7 @@ snapshots:
 
   '@cocalc/primus-responder@1.0.5':
     dependencies:
-      debug: 4.3.7(supports-color@8.1.1)
+      debug: 4.3.7
       node-uuid: 1.4.8
     transitivePeerDependencies:
       - supports-color
@@ -15611,7 +15614,7 @@ snapshots:
   canvas@2.11.2(encoding@0.1.13):
     dependencies:
       '@mapbox/node-pre-gyp': 1.0.11(encoding@0.1.13)
-      nan: 2.19.0
+      nan: 2.20.0
       simple-get: 3.1.1
     transitivePeerDependencies:
       - encoding
@@ -16474,6 +16477,10 @@ snapshots:
     dependencies:
       ms: 2.1.2
 
+  debug@4.3.7:
+    dependencies:
+      ms: 2.1.3
+
   debug@4.3.7(supports-color@8.1.1):
     dependencies:
       ms: 2.1.3
@@ -20158,9 +20165,6 @@ snapshots:
 
   nan@2.17.0: {}
 
-  nan@2.19.0:
-    optional: true
-
   nan@2.20.0: {}
 
   nanoid@3.3.6: {}
@@ -23725,7 +23729,7 @@ snapshots:
     dependencies:
       '@wwa/statvfs': 1.1.18
       awaiting: 3.0.0
-      debug: 4.3.7(supports-color@8.1.1)
+      debug: 4.3.7
       port-get: 1.0.4
       ws: 8.18.0
     transitivePeerDependencies:
diff --git a/src/packages/project/autorenice.ts b/src/packages/project/autorenice.ts
index de83fc5e89..f01f80cf18 100644
--- a/src/packages/project/autorenice.ts
+++ b/src/packages/project/autorenice.ts
@@ -13,7 +13,10 @@ import { reverse, sortBy } from "lodash";
 import { setPriority } from "node:os";
 
 import { getLogger } from "./logger";
-import { ProjectInfoServer, get_ProjectInfoServer } from "./project-info";
+import {
+  ProjectInfoServer,
+  get_ProjectInfoServer,
+} from "@cocalc/sync-server/monitor/activity";
 import {
   Process,
   Processes,
diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts
index 701c4bc638..4d4d453c65 100644
--- a/src/packages/project/browser-websocket/api.ts
+++ b/src/packages/project/browser-websocket/api.ts
@@ -32,7 +32,7 @@ import computeFilesystemCache from "./compute-filesystem-cache";
 import { move_files } from "@cocalc/backend/files/move-files";
 import { rename_file } from "@cocalc/backend/files/rename-file";
 import realpath from "@cocalc/backend/realpath";
-import { project_info_ws } from "../project-info";
+import { project_info_ws } from "@cocalc/sync-server/monitor/activity";
 import query from "./query";
 import { browser_symmetric_channel } from "./symmetric_channel";
 import type { Mesg } from "@cocalc/comm/websocket/types";
diff --git a/src/packages/project/package.json b/src/packages/project/package.json
index 0dcb5dcc02..f960880c4f 100644
--- a/src/packages/project/package.json
+++ b/src/packages/project/package.json
@@ -27,8 +27,8 @@
     "@cocalc/project": "workspace:*",
     "@cocalc/sync": "workspace:*",
     "@cocalc/sync-client": "workspace:*",
-    "@cocalc/sync-server": "workspace:*",
     "@cocalc/sync-fs": "workspace:*",
+    "@cocalc/sync-server": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
     "@nteract/messaging": "^7.0.20",
@@ -41,7 +41,6 @@
     "compression": "^1.7.4",
     "daemonize-process": "^3.0.0",
     "debug": "^4.3.2",
-    "diskusage": "^1.1.3",
     "expect": "^26.6.2",
     "express": "^4.21.1",
     "express-rate-limit": "^7.4.0",
@@ -83,7 +82,9 @@
     "clean": "rm -rf dist"
   },
   "author": "SageMath, Inc.",
-  "contributors": ["William Stein <wstein@sagemath.com>"],
+  "contributors": [
+    "William Stein <wstein@sagemath.com>"
+  ],
   "license": "SEE LICENSE.md",
   "bugs": {
     "url": "https://github.com/sagemathinc/cocalc/issues"
diff --git a/src/packages/project/project-status/server.ts b/src/packages/project/project-status/server.ts
index e15b6104aa..9daad6d5f6 100644
--- a/src/packages/project/project-status/server.ts
+++ b/src/packages/project/project-status/server.ts
@@ -21,7 +21,7 @@ import { version as smcVersion } from "@cocalc/util/smc-version";
 import { delay } from "awaiting";
 import { EventEmitter } from "events";
 import { isEqual } from "lodash";
-import { get_ProjectInfoServer, ProjectInfoServer } from "../project-info";
+import { get_ProjectInfoServer, ProjectInfoServer } from "@cocalc/sync-server/monitor/activity";
 import { ProjectInfo } from "@cocalc/util/types/project-info/types";
 import {
   ALERT_DISK_FREE,
diff --git a/src/packages/project/sync/project-info.ts b/src/packages/project/sync/project-info.ts
index c31d0eb2ee..5ed1ec71c3 100644
--- a/src/packages/project/sync/project-info.ts
+++ b/src/packages/project/sync/project-info.ts
@@ -6,9 +6,11 @@
 import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { close } from "@cocalc/util/misc";
 import { SyncTable } from "@cocalc/sync/table";
-import { get_ProjectInfoServer } from "../project-info";
+import {
+  get_ProjectInfoServer,
+  type ProjectInfoServer,
+} from "@cocalc/sync-server/monitor/activity";
 import { ProjectInfo } from "@cocalc/util/types/project-info/types";
-import { ProjectInfoServer } from "../project-info";
 
 class ProjectInfoTable {
   private table: SyncTable;
@@ -21,7 +23,7 @@ class ProjectInfoTable {
   constructor(
     table: SyncTable,
     logger: { debug: Function },
-    project_id: string
+    project_id: string,
   ) {
     this.project_id = project_id;
     this.logger = logger;
@@ -48,7 +50,7 @@ class ProjectInfoTable {
       this.log(
         `ProjectInfoTable state = '${
           this.state
-        }' and table is '${this.table?.get_state()}'`
+        }' and table is '${this.table?.get_state()}'`,
       );
     }
   }
@@ -72,12 +74,12 @@ let project_info_table: ProjectInfoTable | undefined = undefined;
 export function register_project_info_table(
   table: SyncTable,
   logger: any,
-  project_id: string
+  project_id: string,
 ): void {
   logger.debug("register_project_info_table");
   if (project_info_table != null) {
     logger.debug(
-      "register_project_info_table: cleaning up an already existing one"
+      "register_project_info_table: cleaning up an already existing one",
     );
     project_info_table.close();
   }
diff --git a/src/packages/project/usage-info/server.ts b/src/packages/project/usage-info/server.ts
index 9f0c4ff9bb..f26ada4010 100644
--- a/src/packages/project/usage-info/server.ts
+++ b/src/packages/project/usage-info/server.ts
@@ -13,14 +13,13 @@ from the ProjectInfoServer (which collects data about everything)
 
 import { delay } from "awaiting";
 import { EventEmitter } from "node:events";
-
-import { getLogger } from "../logger";
-import { ProjectInfoServer, get_ProjectInfoServer } from "../project-info";
+import { getLogger } from "@cocalc/backend/logger";
+import { ProjectInfoServer, get_ProjectInfoServer } from "@cocalc/sync-server/monitor/activity";
 import { Process, ProjectInfo } from "@cocalc/util/types/project-info/types";
 import type { UsageInfo } from "@cocalc/util/types/project-usage-info";
 import { throttle } from "lodash";
 
-const L = getLogger("usage-info:server").debug;
+const L = getLogger("sync-server:usage:server").debug;
 
 const throttled_dbg = throttle((...args) => L(...args), 10000);
 
diff --git a/src/packages/sync-server/monitor/README.md b/src/packages/sync-server/monitor/README.md
new file mode 100644
index 0000000000..aa4827c52f
--- /dev/null
+++ b/src/packages/sync-server/monitor/README.md
@@ -0,0 +1,9 @@
+The modules here all collect and update tables that provide various
+info about the sync-server, which is relevant to the sync client.
+
+- process
+- disk usage
+- cpu usage
+- X11 ports
+
+etc
\ No newline at end of file
diff --git a/src/packages/project/project-info/index.ts b/src/packages/sync-server/monitor/activity/index.ts
similarity index 100%
rename from src/packages/project/project-info/index.ts
rename to src/packages/sync-server/monitor/activity/index.ts
diff --git a/src/packages/project/project-info/project-info.ts b/src/packages/sync-server/monitor/activity/project-info.ts
similarity index 100%
rename from src/packages/project/project-info/project-info.ts
rename to src/packages/sync-server/monitor/activity/project-info.ts
diff --git a/src/packages/project/project-info/server.ts b/src/packages/sync-server/monitor/activity/server.ts
similarity index 98%
rename from src/packages/project/project-info/server.ts
rename to src/packages/sync-server/monitor/activity/server.ts
index c776e29e89..de730f2dfa 100644
--- a/src/packages/project/project-info/server.ts
+++ b/src/packages/sync-server/monitor/activity/server.ts
@@ -25,9 +25,9 @@ import {
   ProjectInfo,
 } from "@cocalc/util/types/project-info/types";
 import { get_path_for_pid as x11_pid2path } from "@cocalc/sync-server/x11";
-import { getLogger } from "../logger";
+import { getLogger } from "@cocalc/backend/logger";
 
-const L = getLogger("project-info:server").debug;
+const L = getLogger("sync-server:project-info:server").debug;
 
 // function is_in_dev_project() {
 //   return process.env.SMC_LOCAL_HUB_HOME != null;
diff --git a/src/packages/project/project-info/utils.ts b/src/packages/sync-server/monitor/activity/utils.ts
similarity index 100%
rename from src/packages/project/project-info/utils.ts
rename to src/packages/sync-server/monitor/activity/utils.ts
diff --git a/src/packages/sync-server/package.json b/src/packages/sync-server/package.json
index 34828b1a5e..f2c0be984c 100644
--- a/src/packages/sync-server/package.json
+++ b/src/packages/sync-server/package.json
@@ -3,7 +3,8 @@
   "version": "0.1.0",
   "description": "CoCalc realtime synchronization framework -- server component",
   "exports": {
-    "./*": "./dist/*.js"
+    "./*": "./dist/*.js",
+    "./monitor/activity": "./dist/monitor/activity/index.js"
   },
   "scripts": {
     "clean": "rm -rf dist",
@@ -19,12 +20,14 @@
   "dependencies": {
     "@cocalc/backend": "workspace:*",
     "@cocalc/jupyter": "workspace:*",
-    "@cocalc/sync-server": "workspace:*",
     "@cocalc/sync": "workspace:*",
+    "@cocalc/sync-server": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
+    "awaiting": "^3.0.0",
+    "diskusage": "^1.1.3",
     "events": "3.3.0",
-    "awaiting": "^3.0.0"
+    "lodash": "^4.17.21"
   },
   "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/sync-server",
   "repository": {
@@ -32,6 +35,7 @@
     "url": "https://github.com/sagemathinc/cocalc"
   },
   "devDependencies": {
+    "@types/lodash": "^4.14.202",
     "@types/node": "^18.16.14"
   }
 }

From 3e5ab757bdd1a10c1f91e5a92d6a9ad32198845b Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 16:36:55 +0000
Subject: [PATCH 28/35] sync-server: move usage monitoring from project to new
 package

---
 src/packages/pnpm-lock.yaml                         | 13 ++++++-------
 src/packages/project/jupyter/http-server.ts         |  2 +-
 src/packages/project/project-status/index.ts        |  6 ------
 src/packages/project/sync/project-status.ts         |  2 +-
 src/packages/project/sync/usage-info.ts             | 11 +++++++----
 .../monitor/status-and-alerts.ts}                   |  0
 .../monitor/usage}/const.ts                         |  0
 .../monitor/usage}/index.ts                         |  0
 .../monitor/usage}/server.ts                        |  0
 src/packages/sync-server/package.json               |  5 ++++-
 src/packages/sync-server/tsconfig.json              |  5 +++--
 11 files changed, 22 insertions(+), 22 deletions(-)
 delete mode 100644 src/packages/project/project-status/index.ts
 rename src/packages/{project/project-status/server.ts => sync-server/monitor/status-and-alerts.ts} (100%)
 rename src/packages/{project/usage-info => sync-server/monitor/usage}/const.ts (100%)
 rename src/packages/{project/usage-info => sync-server/monitor/usage}/index.ts (100%)
 rename src/packages/{project/usage-info => sync-server/monitor/usage}/server.ts (100%)

diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml
index 96c2e9c5f6..866bbabd62 100644
--- a/src/packages/pnpm-lock.yaml
+++ b/src/packages/pnpm-lock.yaml
@@ -1226,7 +1226,7 @@ importers:
         version: 3.0.0
       debug:
         specifier: ^4.3.2
-        version: 4.3.7
+        version: 4.3.7(supports-color@8.1.1)
       expect:
         specifier: ^26.6.2
         version: 26.6.2
@@ -1911,6 +1911,9 @@ importers:
       '@cocalc/backend':
         specifier: workspace:*
         version: link:../backend
+      '@cocalc/comm':
+        specifier: workspace:*
+        version: link:../comm
       '@cocalc/jupyter':
         specifier: workspace:*
         version: link:../jupyter
@@ -12640,7 +12643,7 @@ snapshots:
 
   '@cocalc/primus-responder@1.0.5':
     dependencies:
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       node-uuid: 1.4.8
     transitivePeerDependencies:
       - supports-color
@@ -16477,10 +16480,6 @@ snapshots:
     dependencies:
       ms: 2.1.2
 
-  debug@4.3.7:
-    dependencies:
-      ms: 2.1.3
-
   debug@4.3.7(supports-color@8.1.1):
     dependencies:
       ms: 2.1.3
@@ -23729,7 +23728,7 @@ snapshots:
     dependencies:
       '@wwa/statvfs': 1.1.18
       awaiting: 3.0.0
-      debug: 4.3.7
+      debug: 4.3.7(supports-color@8.1.1)
       port-get: 1.0.4
       ws: 8.18.0
     transitivePeerDependencies:
diff --git a/src/packages/project/jupyter/http-server.ts b/src/packages/project/jupyter/http-server.ts
index 29cd44eb65..4e04b6fcef 100644
--- a/src/packages/project/jupyter/http-server.ts
+++ b/src/packages/project/jupyter/http-server.ts
@@ -23,7 +23,7 @@ import {
   BlobStoreSqlite,
 } from "@cocalc/jupyter/blobs";
 import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data";
-import { get_ProjectStatusServer } from "@cocalc/project/project-status/server";
+import { get_ProjectStatusServer } from "@cocalc/sync-server/monitor/status-and-alerts";
 import { delay } from "awaiting";
 
 const log = getLogger("jupyter-http-server");
diff --git a/src/packages/project/project-status/index.ts b/src/packages/project/project-status/index.ts
deleted file mode 100644
index b30058a77f..0000000000
--- a/src/packages/project/project-status/index.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-/*
- *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
- *  License: MS-RSL – see LICENSE.md for details
- */
-
-export { get_ProjectStatusServer, ProjectStatusServer } from "./server";
diff --git a/src/packages/project/sync/project-status.ts b/src/packages/project/sync/project-status.ts
index 2d4d205d29..7366638669 100644
--- a/src/packages/project/sync/project-status.ts
+++ b/src/packages/project/sync/project-status.ts
@@ -9,7 +9,7 @@ import { SyncTable } from "@cocalc/sync/table";
 import {
   get_ProjectStatusServer,
   ProjectStatusServer,
-} from "../project-status";
+} from "@cocalc/sync-server/monitor/status-and-alerts";
 import type { ProjectStatus } from "@cocalc/comm/project-status/types";
 import { getLogger } from "@cocalc/backend/logger";
 
diff --git a/src/packages/project/sync/usage-info.ts b/src/packages/project/sync/usage-info.ts
index ad3cac0412..223aaea08e 100644
--- a/src/packages/project/sync/usage-info.ts
+++ b/src/packages/project/sync/usage-info.ts
@@ -9,8 +9,11 @@
 import { SyncTable, SyncTableState } from "@cocalc/sync/table";
 import { once } from "@cocalc/util/async-utils";
 import { close, merge } from "@cocalc/util/misc";
-import { UsageInfoServer } from "../usage-info";
-import type { ImmutableUsageInfo, UsageInfo } from "@cocalc/util/types/project-usage-info";
+import { UsageInfoServer } from "@cocalc/sync-server/monitor/usage";
+import type {
+  ImmutableUsageInfo,
+  UsageInfo,
+} from "@cocalc/util/types/project-usage-info";
 import { getLogger } from "@cocalc/backend/logger";
 
 const L = getLogger("sync:usage-info");
@@ -85,7 +88,7 @@ class UsageInfoTable {
   async set(obj: { path: string; usage?: UsageInfo }): Promise<void> {
     this.get_table().set(
       merge({ project_id: this.project_id }, obj),
-      "shallow"
+      "shallow",
     );
     await this.get_table().save();
   }
@@ -167,7 +170,7 @@ class UsageInfoTable {
 let usage_info_table: UsageInfoTable | undefined = undefined;
 export function register_usage_info_table(
   table: SyncTable,
-  project_id: string
+  project_id: string,
 ): void {
   L.debug("register_usage_info_table");
   if (usage_info_table != null) {
diff --git a/src/packages/project/project-status/server.ts b/src/packages/sync-server/monitor/status-and-alerts.ts
similarity index 100%
rename from src/packages/project/project-status/server.ts
rename to src/packages/sync-server/monitor/status-and-alerts.ts
diff --git a/src/packages/project/usage-info/const.ts b/src/packages/sync-server/monitor/usage/const.ts
similarity index 100%
rename from src/packages/project/usage-info/const.ts
rename to src/packages/sync-server/monitor/usage/const.ts
diff --git a/src/packages/project/usage-info/index.ts b/src/packages/sync-server/monitor/usage/index.ts
similarity index 100%
rename from src/packages/project/usage-info/index.ts
rename to src/packages/sync-server/monitor/usage/index.ts
diff --git a/src/packages/project/usage-info/server.ts b/src/packages/sync-server/monitor/usage/server.ts
similarity index 100%
rename from src/packages/project/usage-info/server.ts
rename to src/packages/sync-server/monitor/usage/server.ts
diff --git a/src/packages/sync-server/package.json b/src/packages/sync-server/package.json
index f2c0be984c..b069b4cc11 100644
--- a/src/packages/sync-server/package.json
+++ b/src/packages/sync-server/package.json
@@ -4,7 +4,9 @@
   "description": "CoCalc realtime synchronization framework -- server component",
   "exports": {
     "./*": "./dist/*.js",
-    "./monitor/activity": "./dist/monitor/activity/index.js"
+    "./monitor/activity": "./dist/monitor/activity/index.js",
+    "./monitor/usage": "./dist/monitor/usage/index.js",
+    "./monitor/status-and-alerts": "./dist/monitor/status-and-alerts.js"
   },
   "scripts": {
     "clean": "rm -rf dist",
@@ -19,6 +21,7 @@
   "license": "SEE LICENSE.md",
   "dependencies": {
     "@cocalc/backend": "workspace:*",
+    "@cocalc/comm": "workspace:*",
     "@cocalc/jupyter": "workspace:*",
     "@cocalc/sync": "workspace:*",
     "@cocalc/sync-server": "workspace:*",
diff --git a/src/packages/sync-server/tsconfig.json b/src/packages/sync-server/tsconfig.json
index e2e9ff42d7..48a120589e 100644
--- a/src/packages/sync-server/tsconfig.json
+++ b/src/packages/sync-server/tsconfig.json
@@ -7,11 +7,12 @@
   "exclude": ["node_modules", "dist", "test"],
   "references": [
     {
-      "path": "../util",
       "path": "../backend",
+      "path": "../comm",
       "path": "../jupyter",
       "path": "../sync",
-      "path": "../terminal"
+      "path": "../terminal",
+      "path": "../util"
     }
   ]
 }

From 3d5ae7f0a9d0b529418210014fe1767e690ecd14 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 16:57:17 +0000
Subject: [PATCH 29/35] finish moving sync code out of @cocalc/project

- this breaks compute, will fix later
---
 src/packages/pnpm-lock.yaml                     |  6 ++++++
 src/packages/project/browser-websocket/api.ts   |  4 ++--
 src/packages/project/client.ts                  |  6 +++---
 src/packages/sync-server/package.json           | 17 ++++++++++++++---
 .../compute-server-open-file-tracking.ts        |  2 +-
 .../sync => sync-server/server}/listings.ts     |  2 +-
 .../sync-server/{ => server}/open-synctables.ts |  0
 .../sync => sync-server/server}/project-info.ts |  0
 .../server}/project-status.ts                   |  0
 .../sync => sync-server/server}/server.ts       |  9 ++++++---
 .../{ => server}/syncdocs-manager.ts            |  0
 .../sync => sync-server/server}/usage-info.ts   |  0
 src/packages/sync/editor/generic/sync-doc.ts    |  4 ++--
 13 files changed, 35 insertions(+), 15 deletions(-)
 rename src/packages/sync-server/{ => server}/compute-server-open-file-tracking.ts (98%)
 rename src/packages/{project/sync => sync-server/server}/listings.ts (94%)
 rename src/packages/sync-server/{ => server}/open-synctables.ts (100%)
 rename src/packages/{project/sync => sync-server/server}/project-info.ts (100%)
 rename src/packages/{project/sync => sync-server/server}/project-status.ts (100%)
 rename src/packages/{project/sync => sync-server/server}/server.ts (98%)
 rename src/packages/sync-server/{ => server}/syncdocs-manager.ts (100%)
 rename src/packages/{project/sync => sync-server/server}/usage-info.ts (100%)

diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml
index 866bbabd62..0c670d9c1b 100644
--- a/src/packages/pnpm-lock.yaml
+++ b/src/packages/pnpm-lock.yaml
@@ -1938,10 +1938,16 @@ importers:
       events:
         specifier: 3.3.0
         version: 3.3.0
+      json-stable-stringify:
+        specifier: ^1.0.1
+        version: 1.1.1
       lodash:
         specifier: ^4.17.21
         version: 4.17.21
     devDependencies:
+      '@types/json-stable-stringify':
+        specifier: ^1.0.32
+        version: 1.0.36
       '@types/lodash':
         specifier: ^4.14.202
         version: 4.17.9
diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts
index 4d4d453c65..fce0452d7f 100644
--- a/src/packages/project/browser-websocket/api.ts
+++ b/src/packages/project/browser-websocket/api.ts
@@ -21,8 +21,8 @@ import { nbconvert as jupyter_nbconvert } from "../jupyter/convert";
 import { lean, lean_channel } from "../lean/server";
 import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse";
 import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run";
-import { synctable_channel } from "../sync/server";
-import { callSyncDoc } from "@cocalc/sync-server/syncdocs-manager";
+import { synctable_channel } from "@cocalc/sync-server/server/server";
+import { callSyncDoc } from "@cocalc/sync-server/server/syncdocs-manager";
 import { terminal } from "@cocalc/terminal";
 import { x11_channel } from "@cocalc/sync-server/x11";
 import { canonical_paths } from "./canonical-path";
diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts
index 6b92afc266..df84003786 100644
--- a/src/packages/project/client.ts
+++ b/src/packages/project/client.ts
@@ -45,9 +45,9 @@ import initJupyter from "./jupyter/init";
 import * as kucalc from "./kucalc";
 import { getLogger } from "./logger";
 import * as sage_session from "./sage_session";
-import { getListingsTable } from "@cocalc/project/sync/listings";
-import { get_synctable } from "@cocalc/sync-server/open-synctables";
-import { getSyncDoc } from "@cocalc/sync-server/syncdocs-manager";
+import { getListingsTable } from "@cocalc/sync-server/server/listings";
+import { get_synctable } from "@cocalc/sync-server/server/open-synctables";
+import { getSyncDoc } from "@cocalc/sync-server/server/syncdocs-manager";
 
 const winston = getLogger("client");
 
diff --git a/src/packages/sync-server/package.json b/src/packages/sync-server/package.json
index b069b4cc11..e510f4d6df 100644
--- a/src/packages/sync-server/package.json
+++ b/src/packages/sync-server/package.json
@@ -3,7 +3,8 @@
   "version": "0.1.0",
   "description": "CoCalc realtime synchronization framework -- server component",
   "exports": {
-    "./*": "./dist/*.js",
+    "./server/*": "./dist/server/*.js",
+    "./x11": "./dist/x11.js",
     "./monitor/activity": "./dist/monitor/activity/index.js",
     "./monitor/usage": "./dist/monitor/usage/index.js",
     "./monitor/status-and-alerts": "./dist/monitor/status-and-alerts.js"
@@ -15,9 +16,17 @@
     "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput",
     "prepublishOnly": "pnpm test"
   },
-  "files": ["dist/**", "bin/**", "README.md", "package.json"],
+  "files": [
+    "dist/**",
+    "bin/**",
+    "README.md",
+    "package.json"
+  ],
   "author": "SageMath, Inc.",
-  "keywords": ["cocalc", "realtime synchronization"],
+  "keywords": [
+    "cocalc",
+    "realtime synchronization"
+  ],
   "license": "SEE LICENSE.md",
   "dependencies": {
     "@cocalc/backend": "workspace:*",
@@ -30,6 +39,7 @@
     "awaiting": "^3.0.0",
     "diskusage": "^1.1.3",
     "events": "3.3.0",
+    "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21"
   },
   "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/sync-server",
@@ -38,6 +48,7 @@
     "url": "https://github.com/sagemathinc/cocalc"
   },
   "devDependencies": {
+    "@types/json-stable-stringify": "^1.0.32",
     "@types/lodash": "^4.14.202",
     "@types/node": "^18.16.14"
   }
diff --git a/src/packages/sync-server/compute-server-open-file-tracking.ts b/src/packages/sync-server/server/compute-server-open-file-tracking.ts
similarity index 98%
rename from src/packages/sync-server/compute-server-open-file-tracking.ts
rename to src/packages/sync-server/server/compute-server-open-file-tracking.ts
index 13aee7273b..e9ce20cb6d 100644
--- a/src/packages/sync-server/compute-server-open-file-tracking.ts
+++ b/src/packages/sync-server/server/compute-server-open-file-tracking.ts
@@ -4,7 +4,7 @@ Manage the state of open files in the compute servers syncdb sync'd file.
 NOTE: terminals aren't handled at all here, since they don't have a syncdoc.
 */
 
-import type { SyncDocs } from "@cocalc/sync-server/syncdocs-manager";
+import type { SyncDocs } from "@cocalc/sync-server/server/syncdocs-manager";
 import type { SyncDB } from "@cocalc/sync/editor/db/sync";
 import { once } from "@cocalc/util/async-utils";
 import { meta_file, auxFileToOriginal } from "@cocalc/util/misc";
diff --git a/src/packages/project/sync/listings.ts b/src/packages/sync-server/server/listings.ts
similarity index 94%
rename from src/packages/project/sync/listings.ts
rename to src/packages/sync-server/server/listings.ts
index f8693b766a..a477c76fa4 100644
--- a/src/packages/project/sync/listings.ts
+++ b/src/packages/sync-server/server/listings.ts
@@ -9,7 +9,7 @@ import {
 } from "@cocalc/sync/listings";
 import getListing from "@cocalc/backend/get-listing";
 import { Watcher } from "@cocalc/backend/path-watcher";
-import { closeAllSyncDocsInTree } from "@cocalc/sync-server/syncdocs-manager";
+import { closeAllSyncDocsInTree } from "@cocalc/sync-server/server/syncdocs-manager";
 import { getLogger } from "@cocalc/backend/logger";
 import { existsSync } from "fs";
 
diff --git a/src/packages/sync-server/open-synctables.ts b/src/packages/sync-server/server/open-synctables.ts
similarity index 100%
rename from src/packages/sync-server/open-synctables.ts
rename to src/packages/sync-server/server/open-synctables.ts
diff --git a/src/packages/project/sync/project-info.ts b/src/packages/sync-server/server/project-info.ts
similarity index 100%
rename from src/packages/project/sync/project-info.ts
rename to src/packages/sync-server/server/project-info.ts
diff --git a/src/packages/project/sync/project-status.ts b/src/packages/sync-server/server/project-status.ts
similarity index 100%
rename from src/packages/project/sync/project-status.ts
rename to src/packages/sync-server/server/project-status.ts
diff --git a/src/packages/project/sync/server.ts b/src/packages/sync-server/server/server.ts
similarity index 98%
rename from src/packages/project/sync/server.ts
rename to src/packages/sync-server/server/server.ts
index ade3cb64d1..23cb8ba90b 100644
--- a/src/packages/project/sync/server.ts
+++ b/src/packages/sync-server/server/server.ts
@@ -51,7 +51,10 @@ import {
 // @ts-ignore -- typescript nonsense.
 const _ = set_debug;
 
-import { key, register_synctable } from "@cocalc/sync-server/open-synctables";
+import {
+  key,
+  register_synctable,
+} from "@cocalc/sync-server/server/open-synctables";
 import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { once } from "@cocalc/util/async-utils";
 import { delay } from "awaiting";
@@ -61,12 +64,12 @@ import { register_project_info_table } from "./project-info";
 import { register_project_status_table } from "./project-status";
 import { register_usage_info_table } from "./usage-info";
 import type { MergeType } from "@cocalc/sync/table/synctable";
-import Client from "@cocalc/sync-client";
+import type { Client } from "@cocalc/sync/client/types";
 import { getJupyterRedux } from "@cocalc/jupyter/kernel";
 import {
   initSyncDoc,
   getSyncDocFromSyncTable,
-} from "@cocalc/sync-server/syncdocs-manager";
+} from "@cocalc/sync-server/server/syncdocs-manager";
 
 type Query = { [key: string]: any };
 
diff --git a/src/packages/sync-server/syncdocs-manager.ts b/src/packages/sync-server/server/syncdocs-manager.ts
similarity index 100%
rename from src/packages/sync-server/syncdocs-manager.ts
rename to src/packages/sync-server/server/syncdocs-manager.ts
diff --git a/src/packages/project/sync/usage-info.ts b/src/packages/sync-server/server/usage-info.ts
similarity index 100%
rename from src/packages/project/sync/usage-info.ts
rename to src/packages/sync-server/server/usage-info.ts
diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts
index ed5f6e7c56..35ea7725fb 100644
--- a/src/packages/sync/editor/generic/sync-doc.ts
+++ b/src/packages/sync/editor/generic/sync-doc.ts
@@ -1072,7 +1072,7 @@ export class SyncDoc extends EventEmitter {
 
     // WARNING: that 'closed' is emitted at the beginning of the
     // close function (before anything async) for the project is
-    // assumed in src/packages/project/sync/sync-doc.ts, because
+    // assumed in @cocalc/sync-server/server/sync-doc.ts, because
     // that ensures that the moment close is called we lock trying
     // try create the syncdoc again until closing is finished.
     // (This set_state call emits "closed"):
@@ -1717,7 +1717,7 @@ export class SyncDoc extends EventEmitter {
 
     // this only potentially happens for tables in the project,
     // e.g., jupyter and compute servers:
-    // see packages/project/sync/server.ts
+    // see @cocalc/sync-server/server/server.ts
     this.patches_table.on("message", (...args) => {
       dbg("received message", args);
       this.emit("message", ...args);

From 21e4764977bdee229559c7c8d7c152bb1d2ce6c9 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 19:38:43 +0000
Subject: [PATCH 30/35] start wiring in the new sync-server to compute server

---
 .../compute/lib/http-server/synctable-channel.ts       | 10 ++--------
 src/compute/compute/package.json                       |  1 +
 src/compute/compute/tsconfig.json                      |  1 +
 src/compute/pnpm-lock.yaml                             |  3 +++
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/compute/compute/lib/http-server/synctable-channel.ts b/src/compute/compute/lib/http-server/synctable-channel.ts
index 56f920f8d9..7ac8782aea 100644
--- a/src/compute/compute/lib/http-server/synctable-channel.ts
+++ b/src/compute/compute/lib/http-server/synctable-channel.ts
@@ -1,4 +1,5 @@
 import { getLogger } from "../logger";
+import { synctable_channel } from "@cocalc/sync-server/server/server";
 
 const log = getLogger("synctable-channel");
 
@@ -14,12 +15,5 @@ export default async function synctableChannel({
   primus;
 }) {
   log.debug("synctableChannel ", query, options);
-  console.log("synctableChannel", primus != null);
-  const table = await manager.client.synctable_project(
-    manager.project_id,
-    query,
-    options ?? [],
-  );
-  console.log("have our syncTable!", table.get()?.toJS());
-  throw Error("not implemented");
+  return await synctable_channel(manager.client, primus, log, query, options);
 }
diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index 758ea3e271..4c1608ad13 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -30,6 +30,7 @@
     "@cocalc/static": "workspace:*",
     "@cocalc/sync": "workspace:*",
     "@cocalc/sync-client": "workspace:*",
+    "@cocalc/sync-server": "workspace:*",
     "@cocalc/sync-fs": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
diff --git a/src/compute/compute/tsconfig.json b/src/compute/compute/tsconfig.json
index 9b192c7b00..31e04b938c 100644
--- a/src/compute/compute/tsconfig.json
+++ b/src/compute/compute/tsconfig.json
@@ -12,6 +12,7 @@
     { "path": "../sync" },
     { "path": "../sync-client" },
     { "path": "../sync-fs" },
+    { "path": "../sync-server" },
     { "path": "../terminal" },
     { "path": "../util" }
   ]
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index 2e8b589b93..f9a9c96b6a 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -41,6 +41,9 @@ importers:
       '@cocalc/sync-fs':
         specifier: workspace:*
         version: link:../sync-fs
+      '@cocalc/sync-server':
+        specifier: workspace:*
+        version: link:../sync-server
       '@cocalc/terminal':
         specifier: workspace:*
         version: link:../terminal

From 30f1c98449c5ecc0471f1b3a75e9d33a9d975716 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 23:15:08 +0000
Subject: [PATCH 31/35] implement compute server's raw server

---
 src/compute/compute/lib/http-server/index.ts  |  7 +-
 .../compute/lib/http-server/raw-server.ts     | 30 +++++++
 src/compute/compute/package.json              | 17 +++-
 src/compute/pnpm-lock.yaml                    | 79 +++++++++++++++++++
 .../project/servers/browser/http-server.ts    |  1 -
 .../project/servers/browser/static.ts         | 17 +++-
 6 files changed, 141 insertions(+), 10 deletions(-)
 create mode 100644 src/compute/compute/lib/http-server/raw-server.ts

diff --git a/src/compute/compute/lib/http-server/index.ts b/src/compute/compute/lib/http-server/index.ts
index e0223a6c1f..0d2719669c 100644
--- a/src/compute/compute/lib/http-server/index.ts
+++ b/src/compute/compute/lib/http-server/index.ts
@@ -13,6 +13,7 @@ import { join } from "path";
 import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
 import initWebsocket from "./websocket";
 import initHttpNextApi from "./http-next-api";
+import initRaw from "./raw-server";
 
 const logger = getLogger("http-server");
 
@@ -48,7 +49,7 @@ export function initHttpServer({
     );
   });
 
-  app.get("/settings", (_req, res) => {
+  app.get(`${manager.project_id}/settings`, (_req, res) => {
     res.redirect(join("/static", ENTRY_POINT));
   });
 
@@ -74,6 +75,10 @@ export function initHttpServer({
 
   app.use("/api/v2", initHttpNextApi({ manager }));
 
+  const rawUrl = `/${manager.project_id}/raw`;
+  logger.debug("raw server at ", { rawUrl });
+  app.use(rawUrl, initRaw({ home: manager.home }));
+
   server.listen(port, host, () => {
     logger.info(`Server listening http://${host}:${port}`);
   });
diff --git a/src/compute/compute/lib/http-server/raw-server.ts b/src/compute/compute/lib/http-server/raw-server.ts
new file mode 100644
index 0000000000..c4e0b810c6
--- /dev/null
+++ b/src/compute/compute/lib/http-server/raw-server.ts
@@ -0,0 +1,30 @@
+/*
+Serve static files from the home directory.  
+
+NOTE: There is a very similar server in /src/packages/project/servers/browser/static.ts
+See comments there.
+*/
+
+import { static as staticServer } from "express";
+import index from "serve-index";
+import { getLogger } from "../logger";
+import { Router } from "express";
+
+const log = getLogger("http-server:static");
+
+export default function initStatic({ home }: { home: string }): Router {
+  const router = Router();
+  router.use("/", (req, res, next) => {
+    if (req.query.download != null) {
+      res.setHeader("Content-Type", "application/octet-stream");
+    }
+    res.setHeader("Cache-Control", "private, must-revalidate");
+    next();
+  });
+
+  log.info(`serving up HOME="${home}"`);
+
+  router.use("/", index(home, { hidden: true, icons: true }));
+  router.use("/", staticServer(home, { dotfiles: "allow" }));
+  return router;
+}
diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json
index 4c1608ad13..0823f0a2ff 100644
--- a/src/compute/compute/package.json
+++ b/src/compute/compute/package.json
@@ -15,23 +15,31 @@
   "bin": {
     "cocalc-compute-start": "./bin/start.js"
   },
-  "files": ["dist/**", "bin/**", "README.md", "package.json"],
+  "files": [
+    "dist/**",
+    "bin/**",
+    "README.md",
+    "package.json"
+  ],
   "author": "SageMath, Inc.",
-  "keywords": ["cocalc", "jupyter"],
+  "keywords": [
+    "cocalc",
+    "jupyter"
+  ],
   "license": "SEE LICENSE.md",
   "dependencies": {
     "@cocalc/api-client": "workspace:*",
     "@cocalc/backend": "workspace:*",
-    "@cocalc/compute": "link:",
     "@cocalc/comm": "workspace:*",
+    "@cocalc/compute": "link:",
     "@cocalc/jupyter": "workspace:*",
     "@cocalc/primus-multiplex": "^1.1.0",
     "@cocalc/primus-responder": "^1.0.5",
     "@cocalc/static": "workspace:*",
     "@cocalc/sync": "workspace:*",
     "@cocalc/sync-client": "workspace:*",
-    "@cocalc/sync-server": "workspace:*",
     "@cocalc/sync-fs": "workspace:*",
+    "@cocalc/sync-server": "workspace:*",
     "@cocalc/terminal": "workspace:*",
     "@cocalc/util": "workspace:*",
     "awaiting": "^3.0.0",
@@ -40,6 +48,7 @@
     "debug": "^4.3.2",
     "express": "^4.21.1",
     "primus": "^8.0.7",
+    "serve-index": "^1.9.1",
     "websocketfs": "^0.17.4",
     "ws": "^8.18.0"
   },
diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml
index f9a9c96b6a..9c3108c33c 100644
--- a/src/compute/pnpm-lock.yaml
+++ b/src/compute/pnpm-lock.yaml
@@ -6,6 +6,12 @@ settings:
 
 importers:
 
+  api-client: {}
+
+  backend: {}
+
+  comm: {}
+
   compute:
     dependencies:
       '@cocalc/api-client':
@@ -68,6 +74,9 @@ importers:
       primus:
         specifier: ^8.0.7
         version: 8.0.9
+      serve-index:
+        specifier: ^1.9.1
+        version: 1.9.1
       websocketfs:
         specifier: ^0.17.4
         version: 0.17.4
@@ -97,6 +106,22 @@ importers:
         specifier: ^5.6.3
         version: 5.6.3
 
+  jupyter: {}
+
+  static: {}
+
+  sync: {}
+
+  sync-client: {}
+
+  sync-fs: {}
+
+  sync-server: {}
+
+  terminal: {}
+
+  util: {}
+
 packages:
 
   '@cocalc/fuse-native@2.4.1':
@@ -184,6 +209,9 @@ packages:
   base64-js@1.5.1:
     resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
 
+  batch@0.6.1:
+    resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==}
+
   binarysearch@1.0.1:
     resolution: {integrity: sha512-FqhwdeXh1ZSAS/YpJ6lD9+SMf8JodCibe7c51Z9L1zAjHKUDTBisQgdmpfaL+m1qHvwAHnSLR8d9UHc79Hr34g==}
 
@@ -305,6 +333,10 @@ packages:
     resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==}
     engines: {node: '>= 0.4'}
 
+  depd@1.1.2:
+    resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==}
+    engines: {node: '>= 0.6'}
+
   depd@2.0.0:
     resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
     engines: {node: '>= 0.8'}
@@ -437,6 +469,10 @@ packages:
     resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
     engines: {node: '>= 0.4'}
 
+  http-errors@1.6.3:
+    resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==}
+    engines: {node: '>= 0.6'}
+
   http-errors@2.0.0:
     resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
     engines: {node: '>= 0.8'}
@@ -448,6 +484,9 @@ packages:
   ieee754@1.2.1:
     resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
 
+  inherits@2.0.3:
+    resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==}
+
   inherits@2.0.4:
     resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
 
@@ -636,6 +675,10 @@ packages:
     resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==}
     engines: {node: '>= 0.8.0'}
 
+  serve-index@1.9.1:
+    resolution: {integrity: sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==}
+    engines: {node: '>= 0.8.0'}
+
   serve-static@1.16.2:
     resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==}
     engines: {node: '>= 0.8.0'}
@@ -647,6 +690,9 @@ packages:
   setheader@1.0.2:
     resolution: {integrity: sha512-A704nIwzqGed0CnJZIqDE+0udMPS839ocgf1R9OJ8aq8vw4U980HWeNaD9ec8VnmBni9lyGEWDedOWXT/C5kxA==}
 
+  setprototypeof@1.1.0:
+    resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==}
+
   setprototypeof@1.2.0:
     resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
 
@@ -663,6 +709,10 @@ packages:
   simple-swizzle@0.2.2:
     resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
 
+  statuses@1.5.0:
+    resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==}
+    engines: {node: '>= 0.6'}
+
   statuses@2.0.1:
     resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
     engines: {node: '>= 0.8'}
@@ -854,6 +904,8 @@ snapshots:
 
   base64-js@1.5.1: {}
 
+  batch@0.6.1: {}
+
   binarysearch@1.0.1: {}
 
   bindings@1.5.0:
@@ -985,6 +1037,8 @@ snapshots:
       es-errors: 1.3.0
       gopd: 1.0.1
 
+  depd@1.1.2: {}
+
   depd@2.0.0: {}
 
   destroy@1.2.0: {}
@@ -1135,6 +1189,13 @@ snapshots:
     dependencies:
       function-bind: 1.1.2
 
+  http-errors@1.6.3:
+    dependencies:
+      depd: 1.1.2
+      inherits: 2.0.3
+      setprototypeof: 1.1.0
+      statuses: 1.5.0
+
   http-errors@2.0.0:
     dependencies:
       depd: 2.0.0
@@ -1149,6 +1210,8 @@ snapshots:
 
   ieee754@1.2.1: {}
 
+  inherits@2.0.3: {}
+
   inherits@2.0.4: {}
 
   ini@1.3.8: {}
@@ -1334,6 +1397,18 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  serve-index@1.9.1:
+    dependencies:
+      accepts: 1.3.8
+      batch: 0.6.1
+      debug: 2.6.9
+      escape-html: 1.0.3
+      http-errors: 1.6.3
+      mime-types: 2.1.35
+      parseurl: 1.3.3
+    transitivePeerDependencies:
+      - supports-color
+
   serve-static@1.16.2:
     dependencies:
       encodeurl: 2.0.0
@@ -1356,6 +1431,8 @@ snapshots:
     dependencies:
       diagnostics: 1.1.1
 
+  setprototypeof@1.1.0: {}
+
   setprototypeof@1.2.0: {}
 
   side-channel@1.0.6:
@@ -1377,6 +1454,8 @@ snapshots:
     dependencies:
       is-arrayish: 0.3.2
 
+  statuses@1.5.0: {}
+
   statuses@2.0.1: {}
 
   storage-engine@3.0.7:
diff --git a/src/packages/project/servers/browser/http-server.ts b/src/packages/project/servers/browser/http-server.ts
index 20a6db64a1..5a70774ccb 100644
--- a/src/packages/project/servers/browser/http-server.ts
+++ b/src/packages/project/servers/browser/http-server.ts
@@ -16,7 +16,6 @@ import express from "express";
 import { createServer } from "http";
 import { writeFile } from "node:fs/promises";
 import { join } from "node:path";
-
 import basePath from "@cocalc/backend/base-path";
 import initWebsocket from "@cocalc/project/browser-websocket/server";
 import initWebsocketFs from "../websocketfs";
diff --git a/src/packages/project/servers/browser/static.ts b/src/packages/project/servers/browser/static.ts
index 1fe3c2331d..ef73d64cba 100644
--- a/src/packages/project/servers/browser/static.ts
+++ b/src/packages/project/servers/browser/static.ts
@@ -1,10 +1,19 @@
-import { Application, static as staticServer } from "express";
+/*
+Serve static files from the HOME directory.  
+
+Security: Authentication is not handled at this level.
+
+NOTE: There is a very similar server in src/compute/compute/lib/http-server/static.ts
+*/
+
+import { type Application, static as staticServer } from "express";
 import index from "serve-index";
 import { getLogger } from "@cocalc/project/logger";
 
+const log = getLogger("serve-static-files-to-browser");
+
 export default function init(app: Application, base: string) {
-  const winston = getLogger("serve-static-files-to-browser");
-  winston.info(`initialize with base="${base}"`);
+  log.info(`initialize with base="${base}"`);
   // Setup the static raw HTTP server.  This must happen after anything above,
   // since it serves all URL's (so it has to be the fallback).
   app.use(base, (req, res, next) => {
@@ -24,7 +33,7 @@ export default function init(app: Application, base: string) {
   if (HOME == null) {
     throw Error("HOME env variable must be defined");
   }
-  winston.info(`serving up HOME="${HOME}"`);
+  log.info(`serving up HOME="${HOME}"`);
 
   app.use(base, index(HOME, { hidden: true, icons: true }));
   app.use(base, staticServer(HOME, { dotfiles: "allow" }));

From a8f208227bc66211ea3512fe59f4556b4fe34c83 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 23:15:49 +0000
Subject: [PATCH 32/35] clean up -- there were *SEVEN* places in the frontend
 that all compute the raw url to a file in a project, in many different
 ways...; replace all of these by one

---
 .../frontend/account/account-page.tsx         |  2 +-
 .../frontend/app-framework/entry-point.ts     | 14 +++++++++++
 src/packages/frontend/app-framework/index.ts  |  6 -----
 src/packages/frontend/app/active-content.tsx  |  2 +-
 src/packages/frontend/app/page.tsx            |  2 +-
 src/packages/frontend/client/project.ts       |  7 ++----
 src/packages/frontend/compute/entry-point.ts  |  3 ++-
 src/packages/frontend/embed/index.ts          |  4 ++--
 src/packages/frontend/entry-point.ts          |  3 ++-
 .../frontend/frame-editors/frame-tree/util.ts | 11 +++------
 src/packages/frontend/jupyter/nbconvert.tsx   |  4 ++--
 src/packages/frontend/lib/raw-url.ts          | 19 +++++++--------
 .../frontend/misc/process-links/generic.ts    |  9 ++++---
 .../frontend/project/explorer/download.tsx    | 10 ++++----
 .../explorer/file-listing/file-row.tsx        |  7 ++++--
 .../project/page/flyouts/file-list-item.tsx   | 24 +++++++++----------
 .../project/page/flyouts/files-bottom.tsx     |  6 ++---
 src/packages/frontend/project/utils.ts        |  9 ++-----
 .../frontend/project/websocket/connect.ts     |  3 ++-
 src/packages/frontend/project_actions.ts      |  4 ++--
 src/packages/frontend/project_store.ts        |  7 ------
 src/packages/frontend/projects/store.ts       |  2 +-
 src/packages/frontend/user-tracking.ts        |  3 ++-
 23 files changed, 77 insertions(+), 84 deletions(-)
 create mode 100644 src/packages/frontend/app-framework/entry-point.ts

diff --git a/src/packages/frontend/account/account-page.tsx b/src/packages/frontend/account/account-page.tsx
index c9dda55960..3023a5b73d 100644
--- a/src/packages/frontend/account/account-page.tsx
+++ b/src/packages/frontend/account/account-page.tsx
@@ -15,8 +15,8 @@ import { Space } from "antd";
 import { useIntl } from "react-intl";
 import { SignOut } from "@cocalc/frontend/account/sign-out";
 import { AntdTabItem, Col, Row, Tabs } from "@cocalc/frontend/antd-bootstrap";
+import { entryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import {
-  entryPoint,
   React,
   redux,
   useIsMountedRef,
diff --git a/src/packages/frontend/app-framework/entry-point.ts b/src/packages/frontend/app-framework/entry-point.ts
new file mode 100644
index 0000000000..437d3c7193
--- /dev/null
+++ b/src/packages/frontend/app-framework/entry-point.ts
@@ -0,0 +1,14 @@
+type EntryPoint =
+  | "next" // the next frontend app from @cocalc/next
+  | "app" // the full normal frontend app, loaded from the main website
+  | "embed" // an embedded version of the app, e.g., kiosk mode
+  | "compute"; // cocalc loaded from the compute server
+
+let entryPoint: EntryPoint = "next";
+export { entryPoint };
+
+// called only from the entry points themselves.  I wish I could all this
+// stuff using rspack, but I couldn't figure out how.
+export function setEntryPoint(x): void {
+  entryPoint = x;
+}
diff --git a/src/packages/frontend/app-framework/index.ts b/src/packages/frontend/app-framework/index.ts
index 3fdf2a2aff..132b936755 100644
--- a/src/packages/frontend/app-framework/index.ts
+++ b/src/packages/frontend/app-framework/index.ts
@@ -3,12 +3,6 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
-let entryPoint = "app"; // default
-export { entryPoint };
-export function setEntryPoint(x) {
-  entryPoint = x;
-}
-
 // Not sure where this should go...
 declare global {
   interface Window {
diff --git a/src/packages/frontend/app/active-content.tsx b/src/packages/frontend/app/active-content.tsx
index 34c2c77845..e52f6236a5 100644
--- a/src/packages/frontend/app/active-content.tsx
+++ b/src/packages/frontend/app/active-content.tsx
@@ -6,9 +6,9 @@
 import { AccountPage } from "@cocalc/frontend/account/account-page";
 import { AdminPage } from "@cocalc/frontend/admin";
 import { Alert } from "@cocalc/frontend/antd-bootstrap";
+import { entryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import {
   CSS,
-  entryPoint,
   React,
   redux,
   useActions,
diff --git a/src/packages/frontend/app/page.tsx b/src/packages/frontend/app/page.tsx
index 626633ce2b..a8f89f848e 100644
--- a/src/packages/frontend/app/page.tsx
+++ b/src/packages/frontend/app/page.tsx
@@ -14,9 +14,9 @@ import { useIntl } from "react-intl";
 import { Avatar } from "@cocalc/frontend/account/avatar/avatar";
 import { alert_message } from "@cocalc/frontend/alerts";
 import { Button } from "@cocalc/frontend/antd-bootstrap";
+import { entryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import {
   CSS,
-  entryPoint,
   React,
   useActions,
   useEffect,
diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts
index 732564e96b..a85a155a50 100644
--- a/src/packages/frontend/client/project.ts
+++ b/src/packages/frontend/client/project.ts
@@ -7,10 +7,8 @@
 Functionality that mainly involves working with a specific project.
 */
 
-import { join } from "path";
 import { redux } from "@cocalc/frontend/app-framework";
 import computeServers from "@cocalc/frontend/compute/manager";
-import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
 import { dialogs, getIntl } from "@cocalc/frontend/i18n";
 import { ipywidgetsGetBufferUrl } from "@cocalc/frontend/jupyter/server-urls";
 import { allow_project_to_run } from "@cocalc/frontend/project/client-side-throttle";
@@ -45,7 +43,6 @@ import {
   coerce_codomain_to_numbers,
   copy_without,
   defaults,
-  encode_path,
   is_valid_uuid_string,
   required,
 } from "@cocalc/util/misc";
@@ -53,6 +50,7 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { DirectoryListingEntry } from "@cocalc/util/types";
 import httpApi from "./api";
 import { WebappClient } from "./client";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 
 export class ProjectClient {
   private client: WebappClient;
@@ -88,12 +86,11 @@ export class ProjectClient {
     project_id: string; // string or array of strings
     path: string; // string or array of strings
   }): string {
-    const base_path = appBasePath;
     if (opts.path[0] === "/") {
       // absolute path to the root
       opts.path = HOME_ROOT + opts.path; // use root symlink, which is created by start_smc
     }
-    return encode_path(join(base_path, `${opts.project_id}/raw/${opts.path}`));
+    return rawUrl(opts);
   }
 
   public async copy_path_between_projects(opts: {
diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts
index 588e5535f1..8d09c6b21b 100644
--- a/src/packages/frontend/compute/entry-point.ts
+++ b/src/packages/frontend/compute/entry-point.ts
@@ -9,7 +9,8 @@ Entry point for compute server version of CoCalc...
 
 // Load/initialize Redux-based react functionality
 import "@cocalc/frontend/client/client";
-import { redux, setEntryPoint } from "@cocalc/frontend/app-framework";
+import { redux } from "@cocalc/frontend/app-framework";
+import { setEntryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import "@cocalc/frontend/jquery-plugins";
 import { init as initAccount } from "@cocalc/frontend/account";
 import { init as initApp } from "@cocalc/frontend/app/init";
diff --git a/src/packages/frontend/embed/index.ts b/src/packages/frontend/embed/index.ts
index 8dc77f4a48..12c7d501e1 100644
--- a/src/packages/frontend/embed/index.ts
+++ b/src/packages/frontend/embed/index.ts
@@ -12,7 +12,8 @@ console.log("Embed mode");
 
 // Load/initialize Redux-based react functionality
 import "@cocalc/frontend/client/client";
-import { redux, setEntryPoint } from "../app-framework";
+import { redux } from "../app-framework";
+import { setEntryPoint } from "@cocalc/frontend/app-framework/entry-point";
 
 import "../jquery-plugins";
 
@@ -23,7 +24,6 @@ import { init as initProjects } from "../projects";
 import { init as initMarkdown } from "../markdown/markdown-input/main";
 import { init as initCrashBanner } from "../crash-banner";
 
-
 // Do not delete this without first looking at https://github.com/sagemathinc/cocalc/issues/5390
 // This import of codemirror forces the initial full load of codemirror
 // as part of the main webpack entry point.
diff --git a/src/packages/frontend/entry-point.ts b/src/packages/frontend/entry-point.ts
index bf786f3392..f970bf1b8f 100644
--- a/src/packages/frontend/entry-point.ts
+++ b/src/packages/frontend/entry-point.ts
@@ -13,7 +13,8 @@ debug.log = console.log.bind(console); // see https://github.com/debug-js/debug#
 import { COCALC_MINIMAL } from "./fullscreen";
 
 // Load/initialize Redux-based react functionality
-import { redux, setEntryPoint } from "./app-framework";
+import { redux } from "./app-framework";
+import { setEntryPoint } from "@cocalc/frontend/app-framework/entry-point";
 
 // Systemwide notifications that are broadcast to all users (and set by admins)
 import "./system-notifications";
diff --git a/src/packages/frontend/frame-editors/frame-tree/util.ts b/src/packages/frontend/frame-editors/frame-tree/util.ts
index 0164bf7b86..0f8a25a427 100644
--- a/src/packages/frontend/frame-editors/frame-tree/util.ts
+++ b/src/packages/frontend/frame-editors/frame-tree/util.ts
@@ -8,9 +8,7 @@ Utility functions useful for frame-tree editors.
 */
 
 import { path_split, separate_file_extension } from "@cocalc/util/misc";
-import { encode_path } from "@cocalc/util/misc";
-import { join } from "path";
-import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 
 export function parse_path(path: string): {
   directory: string;
@@ -22,10 +20,7 @@ export function parse_path(path: string): {
   return { directory: x.head, base: y.name, filename: x.tail };
 }
 
+// todo: rewrite everything that calls this...
 export function raw_url(project_id: string, path: string): string {
-  // we have to encode the path, since we query this raw server. see
-  // https://github.com/sagemathinc/cocalc/issues/5542
-  // but actually, this is a problem for types of files, not just PDF
-  const path_enc = encode_path(path);
-  return join(appBasePath, project_id, "raw", path_enc);
+  return rawUrl({ project_id, path });
 }
diff --git a/src/packages/frontend/jupyter/nbconvert.tsx b/src/packages/frontend/jupyter/nbconvert.tsx
index 7b3aa82516..711b3041e1 100644
--- a/src/packages/frontend/jupyter/nbconvert.tsx
+++ b/src/packages/frontend/jupyter/nbconvert.tsx
@@ -9,7 +9,7 @@ NBConvert dialog -- for running nbconvert
 import { Button, Modal } from "antd";
 import * as immutable from "immutable";
 import React, { useEffect, useRef } from "react";
-
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 import { redux } from "@cocalc/frontend/app-framework";
 import { A, Icon, Loading, TimeAgo } from "@cocalc/frontend/components";
 import * as misc from "@cocalc/util/misc";
@@ -196,7 +196,7 @@ export const NBConvert: React.FC<NBConvertProps> = React.memo(
         ext = info.ext;
       }
       const targetPath = misc.change_filename_extension(path, ext);
-      const url = actions.store.get_raw_link(targetPath);
+      const url = rawUrl({ path: targetPath, project_id });
       return { targetPath, url, info };
     }
 
diff --git a/src/packages/frontend/lib/raw-url.ts b/src/packages/frontend/lib/raw-url.ts
index 8287c85cb0..d070fc6eff 100644
--- a/src/packages/frontend/lib/raw-url.ts
+++ b/src/packages/frontend/lib/raw-url.ts
@@ -2,10 +2,14 @@
 The raw URL is the following, of course encoded as a URL:
 
 .../{project_id}/raw/{full relative path in the project to file}
+
+On a compute server though the project_id is not redundant (there is only one project),
+so not in the URL.
 */
 
 import { join } from "path";
 import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
+import { encode_path } from "@cocalc/util/misc";
 
 interface Options {
   project_id: string;
@@ -13,14 +17,9 @@ interface Options {
 }
 
 export default function rawURL({ project_id, path }: Options): string {
-  return join(appBasePath, project_id, "raw", encodePath(path));
-}
-
-export function encodePath(path: string) {
-  const segments = path.split("/");
-  const encoded: string[] = [];
-  for (const segment of segments) {
-    encoded.push(encodeURIComponent(segment));
-  }
-  return encoded.join("/");
+  // we have to encode the path, since we query this raw server. see
+  // https://github.com/sagemathinc/cocalc/issues/5542
+  // but actually, this is a problem for types of files, not just PDF
+  const path_enc = encode_path(path);
+  return join(appBasePath, project_id, "raw", path_enc);
 }
diff --git a/src/packages/frontend/misc/process-links/generic.ts b/src/packages/frontend/misc/process-links/generic.ts
index c1fc5327d2..13a38eee12 100644
--- a/src/packages/frontend/misc/process-links/generic.ts
+++ b/src/packages/frontend/misc/process-links/generic.ts
@@ -13,9 +13,9 @@ Define a jQuery plugin that processes links.
 
 import { join } from "path";
 import { is_valid_uuid_string as isUUID } from "@cocalc/util/misc";
-import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
 import { isCoCalcURL } from "@cocalc/frontend/lib/cocalc-urls";
 import Fragment, { FragmentId } from "@cocalc/frontend/misc/fragment-id";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 
 type jQueryAPI = Function;
 
@@ -191,15 +191,14 @@ function processMediaTag(
     // absolute path or data: url
     newSrc = src;
   } else if (opts.projectId != null && opts.filePath != null) {
-    let projectId: string;
     const i = src.indexOf("/projects/");
     const j = src.indexOf("/files/");
     if (isCoCalcURL(src) && i !== -1 && j !== -1 && j > i) {
       // the href is inside the app, points to the current project or another one
       // j-i should be 36, unless we ever start to have different (vanity) project_ids
       const path = src.slice(j + "/files/".length);
-      projectId = src.slice(i + "/projects/".length, j);
-      newSrc = join(appBasePath, projectId, "raw", path);
+      const project_id = src.slice(i + "/projects/".length, j);
+      newSrc = rawUrl({ project_id, path });
       y.attr(attr, newSrc);
       return;
     }
@@ -209,7 +208,7 @@ function processMediaTag(
     }
     // we do not have an absolute url, hence we assume it is a
     // relative URL to a file in a project
-    newSrc = join(appBasePath, opts.projectId, "raw", opts.filePath, src);
+    newSrc = rawUrl({ project_id: opts.projectId, path: opts.filePath });
   }
   if (newSrc != null) {
     y.attr(attr, newSrc);
diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx
index e41544f538..b64b53c910 100644
--- a/src/packages/frontend/project/explorer/download.tsx
+++ b/src/packages/frontend/project/explorer/download.tsx
@@ -8,6 +8,7 @@ import CheckedFiles from "./checked-files";
 import ShowError from "@cocalc/frontend/components/error";
 import { PRE_STYLE } from "./action-box";
 import { Icon } from "@cocalc/frontend/components/icon";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 
 export default function Download({}) {
   const inputRef = useRef<any>(null);
@@ -36,14 +37,13 @@ export default function Download({}) {
       setArchiveMode(true);
       return;
     }
-    const file = checked_files.first();
+    const path = checked_files.first();
     const isdir = redux.getProjectStore(project_id).get("displayed_listing")
-      ?.file_map?.[path_split(file).tail]?.isdir;
-    console.log({ isdir });
+      ?.file_map?.[path_split(path).tail]?.isdir;
     setArchiveMode(!!isdir);
     if (!isdir) {
-      const store = actions?.get_store();
-      setUrl(store?.get_raw_link(file) ?? "");
+      const url = rawUrl({ project_id, path });
+      setUrl(`${document.location.origin}${url}`);
     }
   }, [checked_files, current_path]);
 
diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx
index 25b4b58066..8962a9a784 100644
--- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx
+++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx
@@ -21,7 +21,7 @@ import { ProjectActions } from "@cocalc/frontend/project_actions";
 import track from "@cocalc/frontend/user-tracking";
 import * as misc from "@cocalc/util/misc";
 import { COLORS } from "@cocalc/util/theme";
-import { url_href } from "../../utils";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 import { FileCheckbox } from "./file-checkbox";
 import { PublicButton } from "./public-button";
 import { generate_click_for } from "./utils";
@@ -335,7 +335,10 @@ export const FileRow: React.FC<Props> = React.memo((props) => {
 
   // See https://github.com/sagemathinc/cocalc/issues/1020
   // support right-click → copy url for the download button
-  const url = url_href(props.actions.project_id, full_path());
+  const url = rawUrl({
+    project_id: props.actions.project_id,
+    path: full_path(),
+  });
 
   return (
     <Row
diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx
index 187267f940..20478a802c 100644
--- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx
+++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx
@@ -29,7 +29,7 @@ import {
   isDisabledSnapshots,
 } from "@cocalc/frontend/project/explorer/action-bar";
 import { VIEWABLE_FILE_EXT } from "@cocalc/frontend/project/explorer/file-listing/file-row";
-import { url_href } from "@cocalc/frontend/project/utils";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 import { FILE_ACTIONS } from "@cocalc/frontend/project_actions";
 import {
   filename_extension,
@@ -238,8 +238,8 @@ export const FileListItem = React.memo((props: Readonly<FileListItemProps>) => {
       ? item.isopen
         ? { fontWeight: "bold" }
         : item.isdir
-        ? undefined
-        : { color: COLORS.FILE_EXT }
+          ? undefined
+          : { color: COLORS.FILE_EXT }
       : undefined;
 
     return (
@@ -287,8 +287,8 @@ export const FileListItem = React.memo((props: Readonly<FileListItemProps>) => {
           ? "check-square"
           : "square"
         : item.isdir
-        ? "folder-open"
-        : file_options(item.name)?.icon ?? "file");
+          ? "folder-open"
+          : (file_options(item.name)?.icon ?? "file"));
 
     return (
       <Icon
@@ -410,8 +410,8 @@ export const FileListItem = React.memo((props: Readonly<FileListItemProps>) => {
     const actionNames = multiple
       ? ACTION_BUTTONS_MULTI
       : isdir
-      ? ACTION_BUTTONS_DIR
-      : ACTION_BUTTONS_FILE;
+        ? ACTION_BUTTONS_DIR
+        : ACTION_BUTTONS_FILE;
     for (const name of actionNames) {
       if (name === "download" && !item.isdir) continue;
       const disabled =
@@ -500,7 +500,7 @@ export const FileListItem = React.memo((props: Readonly<FileListItemProps>) => {
       const full_path = path_to_file(current_path, name);
       const ext = (filename_extension(name) ?? "").toLowerCase();
       const showView = VIEWABLE_FILE_EXT.includes(ext);
-      const url = url_href(project_id, full_path);
+      const url = rawUrl({ project_id, path: full_path });
 
       ctx.push({ key: "divide-download", type: "divider" });
 
@@ -533,10 +533,10 @@ export const FileListItem = React.memo((props: Readonly<FileListItemProps>) => {
         ? FILE_ITEM_ACTIVE_STYLE_2
         : {}
       : item.isopen
-      ? item.isactive
-        ? FILE_ITEM_ACTIVE_STYLE
-        : FILE_ITEM_OPENED_STYLE
-      : {};
+        ? item.isactive
+          ? FILE_ITEM_ACTIVE_STYLE
+          : FILE_ITEM_OPENED_STYLE
+        : {};
 
   return (
     <Dropdown menu={{ items: getContextMenu() }} trigger={["contextMenu"]}>
diff --git a/src/packages/frontend/project/page/flyouts/files-bottom.tsx b/src/packages/frontend/project/page/flyouts/files-bottom.tsx
index 35cd72f74a..a2f49b9306 100644
--- a/src/packages/frontend/project/page/flyouts/files-bottom.tsx
+++ b/src/packages/frontend/project/page/flyouts/files-bottom.tsx
@@ -29,7 +29,7 @@ import {
   DirectoryListing,
   DirectoryListingEntry,
 } from "@cocalc/frontend/project/explorer/types";
-import { url_href } from "@cocalc/frontend/project/utils";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 import {
   filename_extension,
   human_readable_size,
@@ -191,7 +191,7 @@ export function FilesBottom({
     const ext = (filename_extension(name) ?? "").toLowerCase();
     const showView = VIEWABLE_FILE_EXT.includes(ext);
     // the "href" part makes the link right-click copyable
-    const url = url_href(project_id, full_path);
+    const url = rawUrl({ project_id, path: full_path });
     const showDownload = !student_project_functionality.disableActions;
     const sizeStr = human_readable_size(size);
 
@@ -309,7 +309,7 @@ export function FilesBottom({
       const name = singleFile.name;
       const iconName = singleFile.isdir
         ? "folder"
-        : file_options(name)?.icon ?? "file";
+        : (file_options(name)?.icon ?? "file");
       return (
         <div style={{ whiteSpace: "nowrap" }} title={name}>
           <Icon name={iconName} /> {trunc_middle(name, 20)}
diff --git a/src/packages/frontend/project/utils.ts b/src/packages/frontend/project/utils.ts
index 9ed7ac5038..eade28101b 100644
--- a/src/packages/frontend/project/utils.ts
+++ b/src/packages/frontend/project/utils.ts
@@ -8,7 +8,6 @@ import * as dogNames from "dog-names";
 import * as os_path from "path";
 import { generate as heroku } from "project-name-generator";
 import * as superb from "superb";
-import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
 import { file_options } from "@cocalc/frontend/editor-tmp";
 import { BASE_URL } from "@cocalc/frontend/misc";
 import { webapp_client } from "@cocalc/frontend/webapp-client";
@@ -25,6 +24,7 @@ import {
   unreachable,
   uuid,
 } from "@cocalc/util/misc";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 
 export function randomPetName() {
   return Math.random() > 0.5 ? catNames.random() : dogNames.allRandom();
@@ -289,14 +289,9 @@ export function url_fullpath(project_id: string, path: string): string {
   );
 }
 
-// returns the URL for the file at the given path
-export function url_href(project_id: string, path: string): string {
-  return os_path.join(appBasePath, project_id, "raw", encode_path(path));
-}
-
 // returns the download URL for a file at a given path
 export function download_href(project_id: string, path: string): string {
-  return `${url_href(project_id, path)}?download`;
+  return `${rawUrl({ project_id, path })}?download`;
 }
 
 export function in_snapshot_path(path: string): boolean {
diff --git a/src/packages/frontend/project/websocket/connect.ts b/src/packages/frontend/project/websocket/connect.ts
index d009a9fe33..5e28edd24c 100644
--- a/src/packages/frontend/project/websocket/connect.ts
+++ b/src/packages/frontend/project/websocket/connect.ts
@@ -16,7 +16,8 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import { callback, delay } from "awaiting";
 import { ajax, globalEval } from "jquery";
 import { join } from "path";
-import { entryPoint, redux } from "@cocalc/frontend/app-framework";
+import { redux } from "@cocalc/frontend/app-framework";
+import { entryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
 import { webapp_client } from "@cocalc/frontend/webapp-client";
 import { allow_project_to_run } from "../client-side-throttle";
diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts
index 4588efbcdc..8c31010dac 100644
--- a/src/packages/frontend/project_actions.ts
+++ b/src/packages/frontend/project_actions.ts
@@ -84,7 +84,6 @@ import {
   download_href,
   in_snapshot_path,
   normalize,
-  url_href,
 } from "@cocalc/frontend/project/utils";
 import { API } from "@cocalc/frontend/project/websocket/api";
 import {
@@ -101,6 +100,7 @@ import {
   ProjectStoreState,
 } from "@cocalc/frontend/project_store";
 import { webapp_client } from "@cocalc/frontend/webapp-client";
+import rawUrl from "@cocalc/frontend/lib/raw-url";
 
 const { defaults, required } = misc;
 
@@ -2571,7 +2571,7 @@ export class ProjectActions extends Actions<ProjectStoreState> {
       url = download_href(this.project_id, opts.path);
       download_file(url);
     } else {
-      url = url_href(this.project_id, opts.path);
+      url = rawUrl({ project_id: this.project_id, path: opts.path });
       const tab = open_new_tab(url);
       if (tab != null && opts.print) {
         // "?" since there might be no print method -- could depend on browser API
diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts
index aa1f814246..b1dfe0fa5c 100644
--- a/src/packages/frontend/project_store.ts
+++ b/src/packages/frontend/project_store.ts
@@ -16,7 +16,6 @@ if (typeof window !== "undefined" && window !== null) {
 }
 
 import * as immutable from "immutable";
-
 import { alert_message } from "@cocalc/frontend/alerts";
 import {
   AppRedux,
@@ -559,12 +558,6 @@ export class ProjectStore extends Store<ProjectStoreState> {
     };
   };
 
-  get_raw_link = (path) => {
-    let url = document.URL;
-    url = url.slice(0, url.indexOf("/projects/"));
-    return `${url}/${this.project_id}/raw/${misc.encode_path(path)}`;
-  };
-
   // returns false, if this project isn't capable of opening a file with the given extension
   async can_open_file_ext(
     ext: string,
diff --git a/src/packages/frontend/projects/store.ts b/src/packages/frontend/projects/store.ts
index 3ac21bf757..06645c383f 100644
--- a/src/packages/frontend/projects/store.ts
+++ b/src/packages/frontend/projects/store.ts
@@ -5,8 +5,8 @@
 import { List, Map, Set } from "immutable";
 import { fromPairs, isEmpty } from "lodash";
 import LRU from "lru-cache";
+import { entryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import {
-  entryPoint,
   redux,
   Store,
   TypedMap,
diff --git a/src/packages/frontend/user-tracking.ts b/src/packages/frontend/user-tracking.ts
index 2ecb0e7a4c..bf7fd50a9e 100644
--- a/src/packages/frontend/user-tracking.ts
+++ b/src/packages/frontend/user-tracking.ts
@@ -9,7 +9,8 @@
 
 import { query, server_time } from "./frame-editors/generic/client";
 import { analytics_cookie_name as analytics, uuid } from "@cocalc/util/misc";
-import { entryPoint, redux } from "./app-framework";
+import { redux } from "./app-framework";
+import { entryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import { version } from "@cocalc/util/smc-version";
 import { get_cookie } from "./misc";
 import { webapp_client } from "./webapp-client";

From 54e0dd118f110d81fb994945e360bdb314621f75 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Sun, 20 Oct 2024 23:27:12 +0000
Subject: [PATCH 33/35] compute server: starting to work on project info

---
 .../compute/lib/http-server/websocket-api.ts   |  9 +++++----
 .../monitor/activity/project-info.ts           | 18 +++++++++++++-----
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index b855784aee..91dfc8ffb8 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -16,6 +16,7 @@ import { terminal } from "@cocalc/terminal";
 import realpath from "@cocalc/backend/realpath";
 import { eval_code } from "@cocalc/backend/eval-code";
 import synctableChannel from "./synctable-channel";
+import { project_info_ws } from "@cocalc/sync-server/monitor/activity";
 
 const log = getLogger("websocket-api");
 
@@ -76,9 +77,7 @@ async function handleApiCall(
 
     case "query":
       if (data.opts?.changes) {
-        throw Error(
-          `changefeeds are not supported for api queries`,
-        );
+        throw Error(`changefeeds are not supported for api queries`);
       }
       return await callback2(
         manager.client.query.bind(manager.client),
@@ -104,6 +103,9 @@ async function handleApiCall(
         primus,
       });
 
+    case "project_info":
+      return await project_info_ws(primus, log);
+
     // TODO
     case "delete_files":
     case "move_files":
@@ -122,7 +124,6 @@ async function handleApiCall(
     case "x11_channel":
     case "syncdoc_call":
     case "symmetric_channel":
-    case "project_info":
     case "compute_filesystem_cache":
     case "sync_fs":
     case "compute_server_sync_register":
diff --git a/src/packages/sync-server/monitor/activity/project-info.ts b/src/packages/sync-server/monitor/activity/project-info.ts
index cb897d01f5..398f33bb68 100644
--- a/src/packages/sync-server/monitor/activity/project-info.ts
+++ b/src/packages/sync-server/monitor/activity/project-info.ts
@@ -5,6 +5,14 @@
 
 /*
 Project information
+
+
+NOTE:
+It seems like project_info_ws sets up up an entire websocket channel, but literally the only 
+thing it does is implement one command to send a kill signal to a process. There's an exec
+api that could do the same thing already.  Maybe there was a big plan to add a much more sophisticated
+API, but it hasn't happened yet?
+
 */
 
 import { ProjectInfoCmds } from "@cocalc/util/types/project-info/types";
@@ -21,8 +29,8 @@ export function get_ProjectInfoServer(): ProjectInfoServer {
 }
 
 export async function project_info_ws(
-  primus: any,
-  logger: { debug: Function }
+  primus,
+  logger: { debug: Function },
 ): Promise<string> {
   const L = (...msg) => logger.debug("project_info:", ...msg);
   const name = `project_info`;
@@ -32,7 +40,7 @@ export async function project_info_ws(
     L(`deregistering ${spark.id}`);
   }
 
-  channel.on("connection", function (spark: any): void {
+  channel.on("connection", (spark): void => {
     // Now handle the connection
     L(`channel: new connection from ${spark.address.ip} -- ${spark.id}`);
 
@@ -43,7 +51,7 @@ export async function project_info_ws(
 
     spark.on("close", () => close("close"));
     spark.on("end", () => close("end"));
-    spark.on("data", function (data: ProjectInfoCmds) {
+    spark.on("data", (data: ProjectInfoCmds) => {
       // we assume only ProjectInfoCmds should come in, but better check what this is
       if (typeof data === "object") {
         switch (data.cmd) {
@@ -58,7 +66,7 @@ export async function project_info_ws(
     });
   });
 
-  channel.on("disconnection", function (spark: any): void {
+  channel.on("disconnection", (spark): void => {
     L(`channel: disconnection from ${spark.address.ip} -- ${spark.id}`);
     deregister(spark);
   });

From aad35c59ff1855e22b6512effa29e1cc518eb503 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Mon, 21 Oct 2024 02:04:25 +0000
Subject: [PATCH 34/35] make sync-server table more robust

---
 src/packages/sync-server/server/server.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/packages/sync-server/server/server.ts b/src/packages/sync-server/server/server.ts
index 23cb8ba90b..93269e1560 100644
--- a/src/packages/sync-server/server/server.ts
+++ b/src/packages/sync-server/server/server.ts
@@ -466,6 +466,10 @@ class SyncTableChannel {
     if (this.closed || this.closing) {
       return; // closing or already closed
     }
+    if (this.synctable == null) {
+      this.log("save_if_possible: not initialized yet");
+      return;
+    }
     this.log("save_if_possible: saves changes to database");
     await this.synctable.save();
     if (this.synctable.table === "syncstrings") {
@@ -523,8 +527,8 @@ class SyncTableChannel {
     this.log("close: closing");
     this.closing = true;
     delete synctable_channels[this.name];
-    this.channel.destroy();
-    this.synctable.close_no_async();
+    this.channel?.destroy();
+    this.synctable?.close_no_async();
     this.log("close: closed");
     close(this); // don't call this.log after this!
     this.closed = true;

From 60d39fd198afd5449c04c2f6b6419dbe93604f75 Mon Sep 17 00:00:00 2001
From: William Stein <wstein@sagemath.com>
Date: Mon, 21 Oct 2024 03:51:04 +0000
Subject: [PATCH 35/35] compute http server: implement basic template for the
 hub's websocket server

- we have to implement a little of this to make the frontend happy
- implemented ping for starters
---
 .../compute/lib/http-server/hub-websocket.ts  | 98 +++++++++++++++++++
 src/compute/compute/lib/http-server/index.ts  | 11 ++-
 .../compute/lib/http-server/websocket-api.ts  |  2 +-
 src/packages/frontend/app/connection-info.tsx |  4 +-
 src/packages/frontend/compute/entry-point.ts  |  4 +-
 src/packages/frontend/i18n/common.ts          |  2 +-
 .../frontend/projects/projects-nav.tsx        |  6 +-
 7 files changed, 117 insertions(+), 10 deletions(-)
 create mode 100644 src/compute/compute/lib/http-server/hub-websocket.ts

diff --git a/src/compute/compute/lib/http-server/hub-websocket.ts b/src/compute/compute/lib/http-server/hub-websocket.ts
new file mode 100644
index 0000000000..0f9ef4f28b
--- /dev/null
+++ b/src/compute/compute/lib/http-server/hub-websocket.ts
@@ -0,0 +1,98 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+/*
+Create websocket similar to connection to normal hub.
+Handle stuff that doesn't directly involve the project or compute server,
+e.g., user identity.
+*/
+
+import { Router } from "express";
+import { Server } from "http";
+import Primus from "primus";
+import { getLogger } from "../logger";
+import { from_json_socket, to_json_socket } from "@cocalc/util/misc";
+import * as message from "@cocalc/util/message";
+
+const logger = getLogger("hub-websocket");
+
+export default function initHubWebsocket({
+  server,
+  manager,
+}: {
+  server: Server;
+  manager;
+}): Router {
+  const opts = {
+    pathname: "/hub",
+    transformer: "websockets",
+  } as const;
+  logger.debug(`Initializing primus websocket server at "${opts.pathname}"...`);
+  const primus = new Primus(server, opts);
+  initApi({ primus, manager });
+
+  const router = Router();
+  const library: string = primus.library();
+
+  // it isn't actually minified, but this is what the static code expects.
+  router.get("/primus.min.js", (_, res) => {
+    logger.debug("serving up /primus.min.js to a specific client");
+    res.send(library);
+  });
+  logger.debug(
+    `waiting for browser client to request primus.min.js (length=${library.length})...`,
+  );
+
+  return router;
+}
+
+function initApi({ primus, manager }): void {
+  primus.on("connection", (spark) => {
+    logger.debug(`HUB: new connection from ${spark.address.ip} -- ${spark.id}`);
+
+    const sendResponse = (mesg) => {
+      const data = to_json_socket(mesg);
+      spark.write(data);
+    };
+
+    spark.on("data", async (data) => {
+      const mesg = from_json_socket(data);
+      logger.debug("HUB:", "request", mesg, "REQUEST");
+      const t0 = Date.now();
+      try {
+        const resp0 = await handleApiCall(mesg, spark, manager, primus);
+        const resp = {
+          ...resp0,
+          id: mesg.id,
+        };
+        sendResponse(resp);
+        logger.debug(
+          "HUB",
+          "response",
+          resp,
+          `FINISHED: time=${Date.now() - t0}ms`,
+        );
+      } catch (err) {
+        // console.trace(); logger.debug("primus-api error stacktrack", err.stack, err);
+        logger.debug("HUB:", "failed response to", mesg, "FAILED", `${err}`);
+        sendResponse({ id: mesg.id, error: err.toString() });
+      }
+    });
+  });
+
+  primus.on("disconnection", (spark) => {
+    logger.debug(`HUB: end connection from ${spark.address.ip} -- ${spark.id}`);
+  });
+}
+
+async function handleApiCall(mesg, spark, manager, primus): Promise<object> {
+  // @ts-ignore
+  const _foo = { mesg, spark, manager, primus };
+  switch (mesg.event) {
+    case "ping":
+      return message.pong({ now: new Date() });
+  }
+  throw Error("not implemented");
+}
diff --git a/src/compute/compute/lib/http-server/index.ts b/src/compute/compute/lib/http-server/index.ts
index 0d2719669c..058ec94aa6 100644
--- a/src/compute/compute/lib/http-server/index.ts
+++ b/src/compute/compute/lib/http-server/index.ts
@@ -12,6 +12,7 @@ import { path as STATIC_PATH } from "@cocalc/static";
 import { join } from "path";
 import { cacheShortTerm, cacheLongTerm } from "@cocalc/util/http-caching";
 import initWebsocket from "./websocket";
+import initHubWebsocket from "./hub-websocket";
 import initHttpNextApi from "./http-next-api";
 import initRaw from "./raw-server";
 
@@ -38,6 +39,8 @@ export function initHttpServer({
   logger.info({ projectBase });
 
   app.use(projectBase, initWebsocket({ server, projectBase, manager }));
+  
+  app.use("/", initHubWebsocket({ server, manager }));
 
   // CRITICAL: compression must be after websocket above!
   app.use(compression());
@@ -49,10 +52,6 @@ export function initHttpServer({
     );
   });
 
-  app.get(`${manager.project_id}/settings`, (_req, res) => {
-    res.redirect(join("/static", ENTRY_POINT));
-  });
-
   app.use(
     `/static/${ENTRY_POINT}`,
     express.static(`/${STATIC_PATH}/${ENTRY_POINT}`, {
@@ -79,6 +78,10 @@ export function initHttpServer({
   logger.debug("raw server at ", { rawUrl });
   app.use(rawUrl, initRaw({ home: manager.home }));
 
+  app.get("*", (_req, res) => {
+    res.redirect(join("/static", ENTRY_POINT));
+  });
+
   server.listen(port, host, () => {
     logger.info(`Server listening http://${host}:${port}`);
   });
diff --git a/src/compute/compute/lib/http-server/websocket-api.ts b/src/compute/compute/lib/http-server/websocket-api.ts
index 91dfc8ffb8..81871025d6 100644
--- a/src/compute/compute/lib/http-server/websocket-api.ts
+++ b/src/compute/compute/lib/http-server/websocket-api.ts
@@ -32,7 +32,7 @@ export function initWebsocketApi({ primus, manager }): void {
         done(resp);
       } catch (err) {
         // console.trace(); log.debug("primus-api error stacktrack", err.stack, err);
-        console.log("primus-api", "request", data, "FAILED", err);
+        log.debug("primus-api", "request", data, "FAILED", err);
         done({ error: err.toString(), status: "error" });
       }
       log.debug(
diff --git a/src/packages/frontend/app/connection-info.tsx b/src/packages/frontend/app/connection-info.tsx
index 46c0e0fa3d..5f4099e3d9 100644
--- a/src/packages/frontend/app/connection-info.tsx
+++ b/src/packages/frontend/app/connection-info.tsx
@@ -50,7 +50,7 @@ export const ConnectionInfo: React.FC = React.memo(() => {
               <h4>
                 <FormattedMessage
                   id="connection-info.ping"
-                  defaultMessage="Ping time"
+                  defaultMessage="Ping Time"
                   description={"Ping how long a server takes to respond"}
                 />
               </h4>
@@ -74,7 +74,7 @@ export const ConnectionInfo: React.FC = React.memo(() => {
             <h4>
               <FormattedMessage
                 id="connection-info.hub_server"
-                defaultMessage="Hub server"
+                defaultMessage="Hub Server"
                 description={"Ping how long a server takes to respond"}
               />
             </h4>
diff --git a/src/packages/frontend/compute/entry-point.ts b/src/packages/frontend/compute/entry-point.ts
index 8d09c6b21b..7af4ebde9f 100644
--- a/src/packages/frontend/compute/entry-point.ts
+++ b/src/packages/frontend/compute/entry-point.ts
@@ -58,5 +58,7 @@ async function initEntryPointState() {
       state: { time: new Date(), state: "running" },
     },
   }) as any;
-  redux.getActions("projects").setState({ project_map });
+  const actions = redux.getActions("projects");
+  actions.setState({ project_map });
+  actions.open_project({ project_id: "81e0c408-ac65-4114-bad5-5f4b6539bd0e" });
 }
diff --git a/src/packages/frontend/i18n/common.ts b/src/packages/frontend/i18n/common.ts
index ee3d5a838f..265d892a6f 100644
--- a/src/packages/frontend/i18n/common.ts
+++ b/src/packages/frontend/i18n/common.ts
@@ -568,7 +568,7 @@ export const labels = defineMessages({
   },
   message_plural: {
     id: "labels.messsage.plural",
-    defaultMessage: "{num, plural, one {message} other {messages}}",
+    defaultMessage: "{num, plural, one {Message} other {Messages}}",
   },
   reconnect: {
     id: "labels.reconnect",
diff --git a/src/packages/frontend/projects/projects-nav.tsx b/src/packages/frontend/projects/projects-nav.tsx
index 774ba952e1..5c673df16d 100644
--- a/src/packages/frontend/projects/projects-nav.tsx
+++ b/src/packages/frontend/projects/projects-nav.tsx
@@ -5,7 +5,7 @@
 
 import type { TabsProps } from "antd";
 import { Avatar, Popover, Tabs, Tooltip } from "antd";
-
+import { entryPoint } from "@cocalc/frontend/app-framework/entry-point";
 import {
   redux,
   useActions,
@@ -207,6 +207,9 @@ function ProjectTab({ project_id }: ProjectTabProps) {
   }
 
   function onMouseUp(e: React.MouseEvent) {
+    if (entryPoint == "compute") {
+      return;
+    }
     // if middle mouse button has been clicked, close the project
     if (e.button === 1) {
       e.stopPropagation();
@@ -263,6 +266,7 @@ export function ProjectsNav(props: ProjectsNavProps) {
       return {
         label: <ProjectTab project_id={project_id} />,
         key: project_id,
+        closable: entryPoint != "compute",
       };
     });
   }, [openProjects]);