diff --git a/autoload/airline.vim b/autoload/airline.vim
index eb0f716..f191476 100644
--- a/autoload/airline.vim
+++ b/autoload/airline.vim
@@ -122,19 +122,6 @@ function! airline#check_mode(winnr)
   if g:airline_detect_modified
     if &modified
       call add(l:mode, 'modified')
-      let colors = exists('g:airline#themes#{g:airline_theme}#palette.inactive_modified.airline_c')
-            \ ? g:airline#themes#{g:airline_theme}#palette.inactive_modified.airline_c : []
-    else
-      let colors = exists('g:airline#themes#{g:airline_theme}#palette.inactive.airline_c')
-            \ ? g:airline#themes#{g:airline_theme}#palette.inactive.airline_c : []
-    endif
-
-    if !empty(colors)
-      for winnr in range(1, winnr('$'))
-        if winnr != a:winnr && has_key(s:contexts, winnr) && s:contexts[winnr].bufnr == context.bufnr
-          call airline#highlighter#exec('airline_c'.(context.bufnr).'_inactive', colors)
-        endif
-      endfor
     endif
   endif
 
@@ -144,6 +131,7 @@ function! airline#check_mode(winnr)
 
   let mode_string = join(l:mode)
   if get(w:, 'airline_lastmode', '') != mode_string
+    call airline#highlighter#highlight_modified_inactive(context.bufnr)
     call airline#highlighter#highlight(l:mode)
     let w:airline_lastmode = mode_string
   endif
diff --git a/autoload/airline/highlighter.vim b/autoload/airline/highlighter.vim
index b811dfa..70d47ee 100644
--- a/autoload/airline/highlighter.vim
+++ b/autoload/airline/highlighter.vim
@@ -91,6 +91,20 @@ function! airline#highlighter#add_separator(from, to, inverse)
   call <sid>exec_separator({}, a:from, a:to, a:inverse, '')
 endfunction
 
+function! airline#highlighter#highlight_modified_inactive(bufnr)
+  if getbufvar(a:bufnr, '&modified')
+    let colors = exists('g:airline#themes#{g:airline_theme}#palette.inactive_modified.airline_c')
+          \ ? g:airline#themes#{g:airline_theme}#palette.inactive_modified.airline_c : []
+  else
+    let colors = exists('g:airline#themes#{g:airline_theme}#palette.inactive.airline_c')
+          \ ? g:airline#themes#{g:airline_theme}#palette.inactive.airline_c : []
+  endif
+
+  if !empty(colors)
+    call airline#highlighter#exec('airline_c'.(a:bufnr).'_inactive', colors)
+  endif
+endfunction
+
 function! airline#highlighter#highlight(modes)
   " draw the base mode, followed by any overrides
   let mapped = map(a:modes, 'v:val == a:modes[0] ? v:val : a:modes[0]."_".v:val')