diff --git a/lisp/subr.el b/lisp/subr.el
index 5d40aaae41c..535fa2d3d0e 100644
--- a/lisp/subr.el
+++ b/lisp/subr.el
@@ -4066,9 +4066,10 @@ that can be added."
 
 (defun remove-from-invisibility-spec (element)
   "Remove ELEMENT from `buffer-invisibility-spec'."
-  (if (consp buffer-invisibility-spec)
-      (setq buffer-invisibility-spec
-	    (delete element buffer-invisibility-spec))))
+  (setq buffer-invisibility-spec
+        (if (consp buffer-invisibility-spec)
+	    (delete element buffer-invisibility-spec)
+          (list t))))
 
 ;;;; Syntax tables.