diff --git a/autoload/ale/highlight.vim b/autoload/ale/highlight.vim
index 15b77d3d..8f6b54de 100644
--- a/autoload/ale/highlight.vim
+++ b/autoload/ale/highlight.vim
@@ -97,7 +97,10 @@ endfunction
 
 function! ale#highlight#BufferHidden(buffer) abort
     " Remember all matches, so they can be restored later.
-    let s:buffer_restore_map[a:buffer] = getmatches()
+    let s:buffer_restore_map[a:buffer] = filter(
+    \   getmatches(),
+    \   'get(v:val, ''group'', '''')[:2] ==# ''ALE'''
+    \)
     call clearmatches()
 endfunction
 
diff --git a/test/test_highlight_placement.vader b/test/test_highlight_placement.vader
index 14c16328..e67bca01 100644
--- a/test/test_highlight_placement.vader
+++ b/test/test_highlight_placement.vader
@@ -28,12 +28,14 @@ Before:
   \ 'command': 'echo',
   \ 'callback': 'GenerateResults',
   \})
+  highlight link SomeOtherGroup SpellBad
 
 After:
   delfunction GenerateResults
   call ale#linter#Reset()
   let g:ale_buffer_info = {}
   call clearmatches()
+  highlight clear SomeOtherGroup
 
 Given testft(A Javscript file with warnings/errors):
   foo
@@ -107,3 +109,23 @@ Execute(Highlights should be cleared when buffers are hidden):
   call ale#highlight#UpdateHighlights()
 
   AssertEqual 1, len(getmatches()), 'The highlights weren''t set again!'
+
+Execute(Only ALE highlights should be restored when buffers are restored):
+  call ale#engine#InitBufferInfo(bufnr('%'))
+  call ale#highlight#SetHighlights(bufnr('%'), [
+  \ {'bufnr': bufnr('%'), 'type': 'E', 'lnum': 3, 'col': 2},
+  \])
+
+  call matchaddpos('SomeOtherGroup', [[1, 1, 1]])
+
+  " We should have one more match here.
+  AssertEqual 2, len(getmatches()), 'The highlights weren''t initially set!'
+
+  call ale#highlight#BufferHidden(bufnr('%'))
+
+  AssertEqual 0, len(getmatches()), 'The highlights weren''t cleared!'
+
+  call ale#highlight#UpdateHighlights()
+
+  " Only our matches should appear again.
+  AssertEqual 1, len(getmatches()), 'The highlights weren''t set again!'