fix: pyodide test suite

Instead of trying to run the entire pytest testsuite in the pyodide
runtime, in selenium we now only run the end-to-end tests "outside" of
pyodide.

By passing the ``lsp-runtime pyodide`` argument to pytest, rather than
launching the server under test via CPython, the test suite will use
NodeJS and a small wrapper script to execute the given server in the
Pyodide runtime.

.github/workflows/ci.yml

@@ -44,7 +44,6 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: '1.5.1'
           virtualenvs-in-project: true
       - name: Load cached venv
         id: cached-poetry-dependencies
@@ -54,7 +53,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --all-extras
+        run: poetry install --with test
       - name: Run tests
         run: |
           source $VENV # Only needed because of Github Action caching
@@ -65,27 +64,37 @@ jobs:
     if: needs.pre_job.outputs.should_skip != 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Use Python "3.10"
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+
+      - uses: 'actions/setup-node@v4'
         with:
-          python-version: "3.10"
+          node-version: 18.x
+          cache: 'npm'
+          cache-dependency-path: 'tests/pyodide/package-lock.json'
+
+      - name: Use Python "3.12"
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
           virtualenvs-in-project: true
+
       - name: Install Dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
-          poetry install --with pyodide
-      - name: Run Testsuite
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 6
-          command: |
-            source $VENV
-            poe test-pyodide || true
+          poetry install --with test
+          poetry build
+
+          cd tests/pyodide
+          npm ci
+
+      - name: Run tests
+        run: |
+          source $VENV
+          poe test-pyodide
 
   lint:
     needs: pre_job

pyproject.toml

@@ -29,10 +29,8 @@ ws = ["websockets"]
 [tool.poetry.group.dev.dependencies]
 # Replaces (amongst many other things) flake8 and bandit
 ruff = ">=0.1.6"
-# TODO `poethepoet>=0.20` needs python 3.8
 poethepoet = ">=0.24.4"
 mypy = ">=1.7.1"
-# TODO `black>=23` needs python 3.8
 black = ">=23.11.0"
 
 [tool.poetry.group.test.dependencies]
@@ -46,14 +44,11 @@ pytest-asyncio = ">=0.21.0"
 sphinx = ">=7.1.2"
 sphinx-rtd-theme = ">=1.3.0"
 
-[tool.poetry.group.pyodide.dependencies]
-selenium = "^4.15.2"
-
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 
 [tool.poe.tasks]
-test-pyodide = "python tests/pyodide_testrunner/run.py"
+test-pyodide = "pytest tests/e2e --lsp-runtime pyodide"
 ruff = "ruff check ."
 mypy = "mypy -p pygls"
 check_generated_client = "python scripts/check_client_is_uptodate.py"

tests/conftest.py

@@ -73,8 +73,18 @@ def fn(*args):
         fpath = pathlib.Path(base_dir, *args)
         assert fpath.exists()
 
-        if runtime != "cpython":
-            raise NotImplementedError(f"uri_for: {runtime=}")
+        if runtime == "pyodide":
+            # Pyodide cannot see the whole file system, so this needs to be made relative to
+            # the workspace's parent folder
+            path = str(fpath).replace(str(WORKSPACE_DIR.parent), "")
+            return uris.from_fs_path(path)
+
+        elif runtime == "wasi":
+            # WASI cannot see the whole filesystem, so this needs to be made relative to
+            # the repo root
+            path = str(fpath).replace(str(REPO_DIR), "")
+            return uris.from_fs_path(path)
+
         return uris.from_fs_path(str(fpath))
 
     return fn
@@ -106,6 +116,60 @@ async def fn(server_name: str):
     return fn
 
 
+def get_client_for_pyodide_server(uri_fixture):
+    """Return a client configured to communicate with a server running under Pyodide.
+
+    This assumes that the pyodide environment has already been bootstrapped.
+    """
+
+    async def fn(server_name: str):
+        client = LanguageClient("pygls-test-suite", "v1")
+
+        PYODIDE_DIR = REPO_DIR / "tests" / "pyodide"
+        server_py = str(SERVER_DIR / server_name)
+
+        await client.start_io("node", str(PYODIDE_DIR / "run_server.js"), server_py)
+
+        response = await client.initialize(
+            types.InitializeParams(
+                capabilities=types.ClientCapabilities(),
+                root_uri=uri_fixture(""),
+            )
+        )
+        assert response is not None
+        return client, response
+
+    return fn
+
+
+def get_client_for_wasi_server(uri_fixture):
+    """Return a client configured to communicate with a server running under WASI.
+
+    This assumes the ``wasmtime`` executable is available to be used as the WASI host.
+    """
+
+    async def fn(server_name: str):
+        client = LanguageClient("pygls-test-suite", "v1")
+
+        # WASI cannot see the whole filesystem, so this needs to be made relative to the
+        # repo root
+        server_py = str(SERVER_DIR / server_name).replace(str(REPO_DIR), "")
+
+        # TODO: Un-nixfiy this
+        await client.start_io("python-wasi", server_py)
+
+        response = await client.initialize(
+            types.InitializeParams(
+                capabilities=types.ClientCapabilities(),
+                root_uri=uri_fixture(""),
+            )
+        )
+        assert response is not None
+        return client, response
+
+    return fn
+
+
 @pytest.fixture(scope="session")
 def get_client_for(runtime, uri_for):
     """Return a client configured to communicate with the specified server.
@@ -115,9 +179,15 @@ def get_client_for(runtime, uri_for):
     It's the consuming fixture's responsibility to stop the client.
     """
     # TODO: Add TCP/WS support.
-    if runtime != "cpython":
+    if runtime not in {"cpython", "pyodide", "wasi"}:
         raise NotImplementedError(f"get_client_for: {runtime=}")
 
+    elif runtime == "pyodide":
+        return get_client_for_pyodide_server(uri_for)
+
+    elif runtime == "wasi":
+        return get_client_for_wasi_server(uri_for)
+
     return get_client_for_cpython_server(uri_for)
 
 

tests/pyodide/.gitignore

@@ -0,0 +1,2 @@
+*.log
+node_modules/

tests/pyodide/package.json

@@ -0,0 +1,10 @@
+{
+    "name": "pyodide_tests",
+    "version": "0.0.0",
+    "description": "Simple wrapper that executes pygls servers in Pyodide",
+    "main": "run_server.js",
+    "author": "openlawlibrary",
+    "dependencies": {
+        "pyodide": "^0.24.1"
+    }
+}

tests/pyodide/run_server.js

@@ -0,0 +1,98 @@
+const fs = require('fs');
+const path = require('path')
+const { loadPyodide } = require('pyodide');
+
+const consoleLog = console.log
+
+const WORKSPACE = path.join(__dirname, "..", "..", "examples", "servers", "workspace")
+const DIST = path.join(__dirname, "..", "..", "dist")
+
+// Create a file to log pyodide output to.
+const logFile = fs.createWriteStream("pyodide.log")
+
+function writeToFile(...args) {
+    logFile.write(args[0] + `\n`);
+}
+
+// Load the workspace into the pyodide runtime.
+//
+// Unlike WASI, there is no "just works" solution for exposing the workspace/ folder
+// to the runtime - it's up to us to manually copy it into pyodide's in-memory filesystem.
+function loadWorkspace(pyodide) {
+    const FS = pyodide.FS
+
+    // Create a folder for the workspace to be copied into.
+    FS.mkdir('/workspace')
+
+    const workspace = fs.readdirSync(WORKSPACE)
+    workspace.forEach((file) => {
+      try {
+        const filename = "/" + path.join("workspace", file)
+        // consoleLog(`${file} -> ${filename}`)
+
+        const stream = FS.open(filename, 'w+')
+        const data = fs.readFileSync(path.join(WORKSPACE, file))
+
+        FS.write(stream, data, 0, data.length, 0)
+        FS.close(stream)
+      } catch (err) {
+        consoleLog(err)
+      }
+    })
+}
+
+// Find the *.whl file containing the build of pygls to test.
+function findWhl() {
+    const files = fs.readdirSync(DIST);
+    const whlFile = files.find(file => /pygls-.*\.whl/.test(file));
+
+    if (whlFile) {
+        return path.join(DIST, whlFile);
+    } else {
+        throw new Error("Unable to find whl file.");
+    }
+}
+
+async function runServer(serverCode) {
+    // Annoyingly, while we can redirect stderr/stdout to a file during this setup stage
+    // it doesn't prevent `micropip.install` from indirectly writing to console.log.
+    //
+    // Internally, `micropip.install` calls `pyodide.loadPackage` and doesn't expose loadPacakge's
+    // options for redirecting output i.e. messageCallback.
+    //
+    // So instead, we override console.log globally.
+    console.log = writeToFile
+    const pyodide = await loadPyodide({
+        // stdin:
+        stderr: writeToFile,
+    })
+
+    loadWorkspace(pyodide)
+
+    await pyodide.loadPackage("micropip")
+    const micropip = pyodide.pyimport("micropip")
+    await micropip.install(`file://${findWhl()}`)
+
+    // Restore the original console.log
+    console.log = consoleLog
+    await pyodide.runPythonAsync(serverCode)
+}
+
+if (process.argv.length < 3) {
+    console.error("Missing server.py file")
+    process.exit(1)
+}
+
+
+const serverCode = fs.readFileSync(process.argv[2], 'utf8')
+
+logFile.once('open', (fd) => {
+    runServer(serverCode).then(() => {
+        logFile.end();
+        process.exit(0)
+    }).catch(err => {
+        logFile.write(`Error in server process\n${err}`)
+        logFile.end();
+        process.exit(1);
+    })
+})