Skip to content

Commit

Permalink
Add end-to-end test with nvim
Browse files Browse the repository at this point in the history
This commit adds an end-to-end test that uses headless Neovim to apply
code actions with a mocked LLM response and diff-compares the output
against a given expected output.

The goal of this test is to ensure the code actions behave as expected,
particularly when interacting with a large language model (LLM) for code
generation and modification. By running the test in a headless Neovim
environment, we can verify the end-to-end functionality of the code
action integration without relying on manual testing.
  • Loading branch information
PatWie committed Aug 11, 2024
1 parent ca7416f commit 6306a53
Show file tree
Hide file tree
Showing 13 changed files with 178 additions and 20 deletions.
37 changes: 37 additions & 0 deletions .github/workflows/build_test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: Rust Build and Nvim Integration

on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]

env:
CARGO_TERM_COLOR: always

jobs:
build:

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Build
run: cargo build --verbose

- uses: rhysd/action-setup-vim@v1
with:
neovim: true

- name: Create symbolic link for Neovim configuration
run: |
cp ./target/debug/polyglot_ls /tmp/polyglot_ls
ln -s $(realpath editor_integrations/nvim/nvim-config) ${HOME}/.config/nvim-test
- name: Run integration tests
run: |
nvim --version
# install all plugins
NVIM_APPNAME=nvim-test nvim -c 'sleep 20' -c "q!" --headless
cd tests && ./run_all.sh
20 changes: 0 additions & 20 deletions .github/workflows/rust.yml

This file was deleted.

26 changes: 26 additions & 0 deletions editor_integrations/nvim/nvim-config/init.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
require("patwie")
function test_lsp_code_action(action_title)
-- vim.api.nvim_win_set_cursor(0, { 2, 2 }) -- Line 3, Column 5 (0-indexed column)

vim.defer_fn(function()
vim.lsp.buf.code_action({
filter = function(action)
return action.title == action_title
end,
apply = true
})

-- -- After applying the code action, wait for a brief moment, then write the file and quit
-- vim.defer_fn(function()
-- -- Get the original filename
-- -- Construct the new filename
-- -- Write the buffer to the new file
-- -- vim.cmd("write! " .. new_filename)
-- -- vim.api.nvim_command("write! " .. new_filename)
-- -- Quit Neovim
-- -- vim.cmd("quit!")
-- -- vim.api.nvim_command("quit!")
-- end, 4000) -- Adjust delay as needed to ensure code action has completed
end, 1000) -- 1000 milliseconds = 1 second
end

3 changes: 3 additions & 0 deletions editor_integrations/nvim/nvim-config/lazy-lock.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"nvim-lspconfig": { "branch": "master", "commit": "652386deae739e38fa1bcf2f06e3e7de9b3436ba" }
}
2 changes: 2 additions & 0 deletions editor_integrations/nvim/nvim-config/lua/patwie/init.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
require("patwie.lazy_init")

46 changes: 46 additions & 0 deletions editor_integrations/nvim/nvim-config/lua/patwie/lazy/lsp.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
return {

{
"neovim/nvim-lspconfig",

config = function()
local util = require 'lspconfig.util'
local configs = require 'lspconfig.configs'

if not configs.polyglot_ls then
configs.polyglot_ls = {
default_config = {
cmd = { "/tmp/polyglot_ls" },
filetypes = { 'python', 'rust', 'text', 'go', 'gitcommit', 'markdown' },
root_dir = util.root_pattern(unpack({
'pyproject.toml',
'ruff.toml',
'Cargo.toml',
})) or util.find_git_ancestor(),
single_file_support = true,
},
}
end

local servers = {
polyglot_ls = {
-- cmd = vim.lsp.rpc.connect('127.0.0.1', 9257),
cmd = { "/tmp/polyglot_ls", "--stdio" , "--use-mock" },
filetypes = { 'python', 'rust', 'text', 'go', 'gitcommit', 'markdown' },
},
}

local capabilities = vim.lsp.protocol.make_client_capabilities()

for server_name, server_config in pairs(servers) do
server_config.capabilities = capabilities
server_config.flags = {
debounce_text_changes = 200,
allow_incremental_sync = true,
}
local lspconfig = require("lspconfig")
lspconfig[server_name].setup(server_config)
end
end,
},
}
27 changes: 27 additions & 0 deletions editor_integrations/nvim/nvim-config/lua/patwie/lazy_init.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
local lazypath = vim.fn.stdpath("data") .. "/lazy/lazy.nvim"
if not vim.loop.fs_stat(lazypath) then
vim.fn.system({
"git",
"clone",
"--filter=blob:none",
"https://github.com/folke/lazy.nvim.git",
"--branch=stable", -- latest stable release
lazypath,
})
end
vim.opt.rtp:prepend(lazypath)

require("lazy").setup({
spec = "patwie.lazy",
pkg = {
enabled = true,
cache = vim.fn.stdpath("state") .. "/lazy/pkg-cache.lua",
-- the first package source that is found for a plugin will be used.
sources = {
"lazy",
"rockspec", -- will only be used when rocks.enabled is true
"packspec",
},
},
change_detection = { notify = false }
})
1 change: 1 addition & 0 deletions tests/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def add(a, b):
return a+b
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def add(MOCK):
return a+b
2 changes: 2 additions & 0 deletions tests/cases/Polyglot: Update Function Docstring/simple.2.2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def add(a, b):
return a+b
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
def add(a, b):
MOCK
return a+b
27 changes: 27 additions & 0 deletions tests/run_all.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash

TEST_DIR="cases"

find "$TEST_DIR" -type f -name "*.py" ! -name "*.want.py" | while IFS= read -r file; do
echo "---"
echo $file
if [ -d "$file" ] || [[ "$file" == "." ]] || [[ "$file" == ".." ]]; then
continue
fi
base_filename=$(basename "$file" .py)

IFS='.' read -r case_name cursor_line cursor_pos ext <<< "$base_filename"
want_file="${file}.want"
got_file="${file}.out"
action_name=$(basename "$(dirname "$file")")

NVIM_APPNAME=nvim-test nvim -c "edit $file | call cursor($cursor_line, $cursor_pos)" -c "lua test_lsp_code_action(\"$action_name\")" -c 'sleep 4' -c "wq! $got_file" --headless

if diff -u "$want_file" "$got_file"; then
echo "Test passed: $file"
else
echo "Test failed: $file"
exit 1
fi
done

0 comments on commit 6306a53

Please sign in to comment.