Given foobar (Some imaginary filetype): var y = 3+3; var y = 3 Before: sign unplace * function! TestCallback(buffer, output) return [ \ {'lnum': 1, 'text': 'foo', 'type': 'W'}, \ {'lnum': 2, 'text': 'foo', 'type': 'E'}, \] endfunction function! CollectSigns() redir => l:output silent exec 'sign place' redir END let l:actual_sign_list = [] for l:line in split(l:output, "\n") let l:match = matchlist(l:line, '\v^.*\=(\d+).*\=\d+.*\=(ALE[a-zA-Z]+Sign)') if len(l:match) > 0 call add(l:actual_sign_list, [l:match[1], l:match[2]]) endif endfor return l:actual_sign_list endfunction call ale#linter#Define('foobar', { \ 'name': 'testlinter', \ 'callback': 'TestCallback', \ 'executable': has('win32') ? 'cmd' : 'echo', \ 'command': has('win32') ? 'echo foo bar' : '/bin/sh -c ''echo foo bar''', \}) After: delfunction TestCallback delfunction CollectSigns sign unplace * let g:ale_buffer_info = {} call ale#engine#Cleanup(bufnr('')) call ale#linter#Reset() Execute(The signs should be updated after linting is done): call ale#Lint() call ale#engine#WaitForJobs(2000) AssertEqual [['1', 'ALEWarningSign'], ['2', 'ALEErrorSign']], CollectSigns()