Skip to content

Commit

Permalink
Add end-to-end test with nvim
Browse files Browse the repository at this point in the history
This commit adds an end-to-end test that uses headless Neovim to apply
code actions with a mocked LLM response and diff-compares the output
against a given expected output.

The goal of this test is to ensure the code actions behave as expected,
particularly when interacting with a large language model (LLM) for code
generation and modification. By running the test in a headless Neovim
environment, we can verify the end-to-end functionality of the code
action integration without relying on manual testing.
  • Loading branch information
PatWie committed Aug 11, 2024
1 parent ca7416f commit a457757
Show file tree
Hide file tree
Showing 13 changed files with 162 additions and 20 deletions.
34 changes: 34 additions & 0 deletions .github/workflows/build_test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: Rust Build and Nvim Integration

on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]

env:
CARGO_TERM_COLOR: always

jobs:
build:

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Build
run: cargo build --verbose

- uses: rhysd/action-setup-vim@v1
with:
neovim: true

- name: Run Tests
run: |
cp ./target/debug/polyglot_ls /tmp/polyglot_ls
ln -s $(realpath editor_integrations/nvim/nvim-config) ${HOME}/.config/nvim-test
/tmp/polyglot_ls --version
NVIM_APPNAME=nvim-test nvim --version
# install all plugins
NVIM_APPNAME=nvim-test nvim -c 'sleep 20' -c "q!" --headless
cd tests && ./run_all.sh
20 changes: 0 additions & 20 deletions .github/workflows/rust.yml

This file was deleted.

12 changes: 12 additions & 0 deletions editor_integrations/nvim/nvim-config/init.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
require("patwie")
function test_lsp_code_action(action_title)
vim.defer_fn(function()
vim.lsp.buf.code_action({
filter = function(action)
return action.title == action_title
end,
apply = true
})
end, 1000) -- 1000 milliseconds = 1 second
end

4 changes: 4 additions & 0 deletions editor_integrations/nvim/nvim-config/lazy-lock.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"lazy.nvim": { "branch": "main", "commit": "077102c5bfc578693f12377846d427f49bc50076" },
"nvim-lspconfig": { "branch": "master", "commit": "652386deae739e38fa1bcf2f06e3e7de9b3436ba" }
}
2 changes: 2 additions & 0 deletions editor_integrations/nvim/nvim-config/lua/patwie/init.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
require("patwie.lazy_init")

46 changes: 46 additions & 0 deletions editor_integrations/nvim/nvim-config/lua/patwie/lazy/lsp.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
return {

{
"neovim/nvim-lspconfig",

config = function()
local util = require 'lspconfig.util'
local configs = require 'lspconfig.configs'

if not configs.polyglot_ls then
configs.polyglot_ls = {
default_config = {
cmd = { "/tmp/polyglot_ls" },
filetypes = { 'python', 'rust', 'text', 'go', 'gitcommit', 'markdown' },
root_dir = util.root_pattern(unpack({
'pyproject.toml',
'ruff.toml',
'Cargo.toml',
})) or util.find_git_ancestor(),
single_file_support = true,
},
}
end

local servers = {
polyglot_ls = {
-- cmd = vim.lsp.rpc.connect('127.0.0.1', 9257),
cmd = { "/tmp/polyglot_ls", "--stdio" , "--use-mock" },
filetypes = { 'python', 'rust', 'text', 'go', 'gitcommit', 'markdown' },
},
}

local capabilities = vim.lsp.protocol.make_client_capabilities()

for server_name, server_config in pairs(servers) do
server_config.capabilities = capabilities
server_config.flags = {
debounce_text_changes = 200,
allow_incremental_sync = true,
}
local lspconfig = require("lspconfig")
lspconfig[server_name].setup(server_config)
end
end,
},
}
27 changes: 27 additions & 0 deletions editor_integrations/nvim/nvim-config/lua/patwie/lazy_init.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
local lazypath = vim.fn.stdpath("data") .. "/lazy/lazy.nvim"
if not vim.loop.fs_stat(lazypath) then
vim.fn.system({
"git",
"clone",
"--filter=blob:none",
"https://github.com/folke/lazy.nvim.git",
"--branch=stable", -- latest stable release
lazypath,
})
end
vim.opt.rtp:prepend(lazypath)

require("lazy").setup({
spec = "patwie.lazy",
pkg = {
enabled = true,
cache = vim.fn.stdpath("state") .. "/lazy/pkg-cache.lua",
-- the first package source that is found for a plugin will be used.
sources = {
"lazy",
"rockspec", -- will only be used when rocks.enabled is true
"packspec",
},
},
change_detection = { notify = false }
})
1 change: 1 addition & 0 deletions tests/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def add(a, b):
return a+b
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def add(MOCK):
return a+b
2 changes: 2 additions & 0 deletions tests/cases/Polyglot: Update Function Docstring/simple.2.2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def add(a, b):
return a+b
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
def add(a, b):
MOCK
return a+b
27 changes: 27 additions & 0 deletions tests/run_all.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash

TEST_DIR="cases"

find "$TEST_DIR" -type f -name "*.py" ! -name "*.want.py" | while IFS= read -r file; do
echo "---"
echo $file
if [ -d "$file" ] || [[ "$file" == "." ]] || [[ "$file" == ".." ]]; then
continue
fi
base_filename=$(basename "$file" .py)

IFS='.' read -r case_name cursor_line cursor_pos ext <<< "$base_filename"
want_file="${file}.want"
got_file="${file}.out"
action_name=$(basename "$(dirname "$file")")

NVIM_APPNAME=nvim-test nvim -c "edit $file | call cursor($cursor_line, $cursor_pos)" -c "lua test_lsp_code_action(\"$action_name\")" -c 'sleep 4' -c "wq! $got_file" --headless

if diff -u "$want_file" "$got_file"; then
echo "Test passed: $file"
else
echo "Test failed: $file"
exit 1
fi
done

0 comments on commit a457757

Please sign in to comment.