diff --git a/autoload/tagbar.vim b/autoload/tagbar.vim
index 12827e7..09b6888 100644
--- a/autoload/tagbar.vim
+++ b/autoload/tagbar.vim
@@ -3463,13 +3463,13 @@ function! s:HandleBufDelete(bufname, bufnr) abort
         return
     endif
 
+    call s:known_files.rm(fnamemodify(a:bufname, ':p'))
+
     let tagbarwinnr = bufwinnr(s:TagbarBufName())
     if tagbarwinnr == -1 || a:bufname =~# '__Tagbar__.*'
         return
     endif
 
-    call s:known_files.rm(fnamemodify(a:bufname, ':p'))
-
     if !s:HasOpenFileWindows()
         if tabpagenr('$') == 1 && exists('t:tagbar_buf_name')
             " The last normal window closed due to a :bdelete/:bwipeout.