From cb88e8d7b16eeba6494b0f55f13c0def98080f41 Mon Sep 17 00:00:00 2001
From: patwie <mail@patwie.com>
Date: Sun, 11 Aug 2024 11:25:33 +0000
Subject: [PATCH] Add end-to-end test with nvim

This commit adds an end-to-end test that uses headless Neovim to apply
code actions with a mocked LLM response and diff-compares the output
against a given expected output.

The goal of this test is to ensure the code actions behave as expected,
particularly when interacting with a large language model (LLM) for code
generation and modification. By running the test in a headless Neovim
environment, we can verify the end-to-end functionality of the code
action integration without relying on manual testing.
---
 .github/workflows/build_test.yml              | 38 +++++++++++++++
 .github/workflows/rust.yml                    | 20 --------
 README.md                                     | 15 ++++++
 editor_integrations/nvim/nvim-config/init.lua | 12 +++++
 .../nvim/nvim-config/lazy-lock.json           |  4 ++
 .../nvim/nvim-config/lua/patwie/init.lua      |  2 +
 .../nvim/nvim-config/lua/patwie/lazy/lsp.lua  | 46 +++++++++++++++++++
 .../nvim/nvim-config/lua/patwie/lazy_init.lua | 27 +++++++++++
 tests/.gitignore                              |  1 +
 .../simple.2.2.py                             |  2 +
 .../simple.2.2.py.want                        |  2 +
 .../simple.2.2.py                             |  2 +
 .../simple.2.2.py.want                        |  3 ++
 tests/run_all.sh                              | 28 +++++++++++
 14 files changed, 182 insertions(+), 20 deletions(-)
 create mode 100644 .github/workflows/build_test.yml
 delete mode 100644 .github/workflows/rust.yml
 create mode 100644 editor_integrations/nvim/nvim-config/init.lua
 create mode 100644 editor_integrations/nvim/nvim-config/lazy-lock.json
 create mode 100644 editor_integrations/nvim/nvim-config/lua/patwie/init.lua
 create mode 100644 editor_integrations/nvim/nvim-config/lua/patwie/lazy/lsp.lua
 create mode 100644 editor_integrations/nvim/nvim-config/lua/patwie/lazy_init.lua
 create mode 100644 tests/.gitignore
 create mode 100644 tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py
 create mode 100644 tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py.want
 create mode 100644 tests/cases/Polyglot: Update Function Docstring/simple.2.2.py
 create mode 100644 tests/cases/Polyglot: Update Function Docstring/simple.2.2.py.want
 create mode 100755 tests/run_all.sh

diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
new file mode 100644
index 0000000..6ee6017
--- /dev/null
+++ b/.github/workflows/build_test.yml
@@ -0,0 +1,38 @@
+name: Rust Build and Nvim Integration
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build
+      run: cargo build --verbose
+
+    - uses: rhysd/action-setup-vim@v1
+      with:
+        neovim: true
+
+    # TODO(patwie): For some reason they fail in github workflow and but locally
+    # everything is fine.
+    - name: Run Tests
+      run: |
+        export NVIM_APPNAME=nvim-test
+        ln -s $(realpath editor_integrations/nvim/nvim-config) ${HOME}/.config/${NVIM_APPNAME}
+        nvim --version
+        cp ./target/debug/polyglot_ls /tmp/polyglot_ls
+        /tmp/polyglot_ls --version
+
+        nvim --headless "+Lazy! update" +qa
+        #./tests/run_all.sh
+        #cat ~/.local/state/nvim/log
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
deleted file mode 100644
index 7f05c5f..0000000
--- a/.github/workflows/rust.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-name: Rust
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Build
-      run: cargo build --verbose
diff --git a/README.md b/README.md
index c46b057..c2571ee 100644
--- a/README.md
+++ b/README.md
@@ -140,3 +140,18 @@ end
 ## Configuration Tutorial
 
 See the [Tutorial.md](./TUTORIAL.md).
+
+# Test Integration
+
+Prepare NVIM integration tests via
+
+```sh
+ln -s $(realpath editor_integrations/nvim/nvim-config) ${HOME}/.config/nvim-test
+NVIM_APPNAME=nvim-test nvim --headless "+Lazy! update" +qa
+```
+
+Then running
+
+```sh
+NVIM_APPNAME=nvim-test ./tests/run_all.sh
+```
diff --git a/editor_integrations/nvim/nvim-config/init.lua b/editor_integrations/nvim/nvim-config/init.lua
new file mode 100644
index 0000000..20c843a
--- /dev/null
+++ b/editor_integrations/nvim/nvim-config/init.lua
@@ -0,0 +1,12 @@
+require("patwie")
+function test_lsp_code_action(action_title)
+  vim.defer_fn(function()
+    vim.lsp.buf.code_action({
+      filter = function(action)
+        return action.title == action_title
+      end,
+      apply = true
+    })
+  end, 1000)   -- 1000 milliseconds = 1 second
+end
+
diff --git a/editor_integrations/nvim/nvim-config/lazy-lock.json b/editor_integrations/nvim/nvim-config/lazy-lock.json
new file mode 100644
index 0000000..d2b4a55
--- /dev/null
+++ b/editor_integrations/nvim/nvim-config/lazy-lock.json
@@ -0,0 +1,4 @@
+{
+  "lazy.nvim": { "branch": "main", "commit": "077102c5bfc578693f12377846d427f49bc50076" },
+  "nvim-lspconfig": { "branch": "master", "commit": "ff97d376b1d22b2eaf9274605531babf0cd0cf21" }
+}
diff --git a/editor_integrations/nvim/nvim-config/lua/patwie/init.lua b/editor_integrations/nvim/nvim-config/lua/patwie/init.lua
new file mode 100644
index 0000000..c12bdca
--- /dev/null
+++ b/editor_integrations/nvim/nvim-config/lua/patwie/init.lua
@@ -0,0 +1,2 @@
+require("patwie.lazy_init")
+
diff --git a/editor_integrations/nvim/nvim-config/lua/patwie/lazy/lsp.lua b/editor_integrations/nvim/nvim-config/lua/patwie/lazy/lsp.lua
new file mode 100644
index 0000000..d6c8c04
--- /dev/null
+++ b/editor_integrations/nvim/nvim-config/lua/patwie/lazy/lsp.lua
@@ -0,0 +1,46 @@
+return {
+
+  {
+    "neovim/nvim-lspconfig",
+
+    config = function()
+      local util = require 'lspconfig.util'
+      local configs = require 'lspconfig.configs'
+
+      if not configs.polyglot_ls then
+        configs.polyglot_ls = {
+          default_config = {
+            cmd = { "/tmp/polyglot_ls" },
+            filetypes = { 'python', 'rust', 'text', 'go', 'gitcommit', 'markdown' },
+            root_dir = util.root_pattern(unpack({
+              'pyproject.toml',
+              'ruff.toml',
+              'Cargo.toml',
+            })) or util.find_git_ancestor(),
+            single_file_support = true,
+          },
+        }
+      end
+
+      local servers = {
+        polyglot_ls = {
+          -- cmd = vim.lsp.rpc.connect('127.0.0.1', 9257),
+          cmd = { "/tmp/polyglot_ls", "--stdio" , "--use-mock" },
+          filetypes = { 'python', 'rust', 'text', 'go', 'gitcommit', 'markdown' },
+        },
+      }
+
+      local capabilities = vim.lsp.protocol.make_client_capabilities()
+
+      for server_name, server_config in pairs(servers) do
+        server_config.capabilities = capabilities
+        server_config.flags = {
+          debounce_text_changes = 200,
+          allow_incremental_sync = true,
+        }
+      local lspconfig = require("lspconfig")
+        lspconfig[server_name].setup(server_config)
+      end
+    end,
+  },
+}
diff --git a/editor_integrations/nvim/nvim-config/lua/patwie/lazy_init.lua b/editor_integrations/nvim/nvim-config/lua/patwie/lazy_init.lua
new file mode 100644
index 0000000..9fdc55c
--- /dev/null
+++ b/editor_integrations/nvim/nvim-config/lua/patwie/lazy_init.lua
@@ -0,0 +1,27 @@
+local lazypath = vim.fn.stdpath("data") .. "/lazy/lazy.nvim"
+if not vim.loop.fs_stat(lazypath) then
+  vim.fn.system({
+    "git",
+    "clone",
+    "--filter=blob:none",
+    "https://github.com/folke/lazy.nvim.git",
+    "--branch=stable", -- latest stable release
+    lazypath,
+  })
+end
+vim.opt.rtp:prepend(lazypath)
+
+require("lazy").setup({
+  spec = "patwie.lazy",
+  pkg = {
+    enabled = true,
+    cache = vim.fn.stdpath("state") .. "/lazy/pkg-cache.lua",
+    -- the first package source that is found for a plugin will be used.
+    sources = {
+      "lazy",
+      "rockspec", -- will only be used when rocks.enabled is true
+      "packspec",
+    },
+  },
+  change_detection = { notify = false }
+})
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..f47cb20
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1 @@
+*.out
diff --git a/tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py b/tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py
new file mode 100644
index 0000000..942cd81
--- /dev/null
+++ b/tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py	
@@ -0,0 +1,2 @@
+def add(a, b):
+    return a+b
diff --git a/tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py.want b/tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py.want
new file mode 100644
index 0000000..05e17e7
--- /dev/null
+++ b/tests/cases/Polyglot: Update Function Args Type Annotations/simple.2.2.py.want	
@@ -0,0 +1,2 @@
+def add(MOCK):
+    return a+b
diff --git a/tests/cases/Polyglot: Update Function Docstring/simple.2.2.py b/tests/cases/Polyglot: Update Function Docstring/simple.2.2.py
new file mode 100644
index 0000000..942cd81
--- /dev/null
+++ b/tests/cases/Polyglot: Update Function Docstring/simple.2.2.py	
@@ -0,0 +1,2 @@
+def add(a, b):
+    return a+b
diff --git a/tests/cases/Polyglot: Update Function Docstring/simple.2.2.py.want b/tests/cases/Polyglot: Update Function Docstring/simple.2.2.py.want
new file mode 100644
index 0000000..a4819f5
--- /dev/null
+++ b/tests/cases/Polyglot: Update Function Docstring/simple.2.2.py.want	
@@ -0,0 +1,3 @@
+def add(a, b):
+    MOCK
+    return a+b
diff --git a/tests/run_all.sh b/tests/run_all.sh
new file mode 100755
index 0000000..d34ab5b
--- /dev/null
+++ b/tests/run_all.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+TEST_DIR="tests/cases"
+
+find "$TEST_DIR" -type f -name "*.py" ! -name "*.want.py" | while IFS= read -r file; do
+  echo "---"
+  echo $file
+    if [ -d "$file" ] || [[ "$file" == "." ]] || [[ "$file" == ".." ]]; then
+        continue
+    fi
+    base_filename=$(basename "$file" .py)
+
+    IFS='.' read -r case_name cursor_line cursor_pos ext <<< "$base_filename"
+    want_file="${file}.want"
+    got_file="${file}.out"
+    action_name=$(basename "$(dirname "$file")")
+
+    nvim  -c "edit $file | call cursor($cursor_line, $cursor_pos)" -c "lua test_lsp_code_action(\"$action_name\")" -c 'sleep 4' -c "wq! $got_file" --headless
+    # NVIM_APPNAME=nvim-test nvim  -c "edit $file | call cursor($cursor_line, $cursor_pos)" -c "lua test_lsp_code_action(\"$action_name\")" -c 'sleep 4' -c "wq! $got_file" --headless
+
+    if diff -u "$want_file" "$got_file"; then
+        echo "Test passed: $file"
+    else
+        echo "Test failed: $file"
+        exit 1
+    fi
+done
+