Restrict symbol prettification to displayable glyphs

* lisp/international/mule.el (char-displayable-on-frame-p): New
function used to determine whether a character can be
meaningfully displayed on a given frame.
* doc/lispref/display.texi (Fontsets): Document it.
* lisp/progmodes/prog-mode.el
(prettify-symbols--composition-displayable-p): New function used
to restrict to displayable prettification symbols.  This
prevents issues with missing characters appearing as boxes.
(prettify-symbols--make-keywords): Use it.  (Bug#77381)

doc/lispref/display.texi

@@ -4122,6 +4122,14 @@ available, since it also checks whether the coding system for the text
 terminal can encode the character (@pxref{Terminal I/O Encoding}).
 @end defun
 
+@defun char-displayable-on-frame-p char frame
+This function behaves like @code{char-displayable-p} does (relative to
+@var{frame}), but in the graphical case, it does not perform the final
+check of whether the underlying text terminal can encode the character.
+It thus provides a displayability check for @var{char} more specific to
+@var{frame}.
+@end defun
+
 @node Low-Level Font
 @subsection Low-Level Font Representation
 @cindex font property

etc/NEWS

@@ -64,6 +64,9 @@ init file.
 
 * Changes in Emacs 31.1
 
+** `prettify-symbols-mode' attempts to ignore undisplayable characters.
+Previously, such characters would be rendered as, e.g., white boxes.
+
 +++
 ** 'standard-display-table' now has more extra slots.
 'standard-display-table' has been extended to allow specifying glyphs
@@ -1792,6 +1795,13 @@ It has been promoted from 'subr-x' to the C code.
 You can now directly pass it a string or a buffer rather than a function.
 Actually passing it a function is now deprecated.
 
++++
+** New function 'char-displayable-on-frame-p'.
+'char-displayable-on-frame-p' returns non-nil if Emacs ought to be able
+to display its char argument on a given frame.  This new function,
+unlike 'char-displayable-p', does not check whether the character can be
+encoded by the underlying terminal.
+
 +++
 ** New macros 'static-when' and 'static-unless'.
 Like 'static-if', these macros evaluate their condition at

lisp/international/mule.el

@@ -528,6 +528,32 @@ per-character basis, this may not be accurate."
                                    (throw 'tag3 charset)))
 			     charset-list)
 		       nil)))))))))))
+
+(defun char-displayable-on-frame-p (char &optional frame)
+  "Return non-nil if CHAR can be displayed in FRAME.
+FRAME nil means the selected frame.
+
+This function provides a stricter test than `char-displayable-p' does
+for determining if a character will display properly: in the graphical
+case, it does not check whether the underlying terminal can encode the
+character.
+
+Specifically, this function returns non-nil:
+
+- for a text terminal, if `char-displayable-p' returns non-nil.
+
+- for a graphical terminal, if `char-displayable-p' returns either t or
+  a font object.
+
+The two functions differ in behavior (i.e., `char-displayable-strict-p'
+returns nil but `char-displayable-p' does not) if the underlying
+terminal is graphical and can encode the character, but FRAME cannot."
+  (let ((display-capability (with-selected-frame (or frame (selected-frame))
+                              (char-displayable-p char))))
+    (if (display-graphic-p frame)
+        (or (eq display-capability t)
+            (fontp display-capability))
+      display-capability)))
 
 ;; Save the ASCII case table in case we need it later.  Some locales
 ;; (such as Turkish) modify the case behavior of ASCII characters,

lisp/progmodes/prog-mode.el

@@ -230,10 +230,37 @@ Regexp match data 0 specifies the characters to be composed."
   ;; Return nil because we're not adding any face property.
   nil)
 
+(defun prettify-symbols--composition-displayable-p (composition)
+  "Return non-nil if COMPOSITION can be displayed with the current fonts.
+COMPOSITION can be a single character, a string, or a sequence (vector or
+list) of characters and composition rules as described in the documentation
+of `prettify-symbols-alist' and `compose-region'."
+  (cond
+   ((characterp composition)
+    (char-displayable-on-frame-p composition))
+   ((stringp composition)
+    (seq-every-p #'char-displayable-on-frame-p composition))
+   ((seqp composition)
+    ;; check that every even-indexed element is displayable
+    (seq-every-p
+     (lambda (idx-elt)
+       (if (evenp (car idx-elt))
+           (char-displayable-on-frame-p (cdr idx-elt))
+         t))
+     (seq-map-indexed #'cons composition)))
+   (t
+    ;; silently ignore invalid compositions
+    t)))
+
 (defun prettify-symbols--make-keywords ()
   (if prettify-symbols-alist
-      `((,(regexp-opt (mapcar 'car prettify-symbols-alist) t)
-         (0 (prettify-symbols--compose-symbol ',prettify-symbols-alist))))
+      (let ((filtered-alist
+             (seq-filter
+              (lambda (elt)
+                (prettify-symbols--composition-displayable-p (cdr elt)))
+              prettify-symbols-alist)))
+        `((,(regexp-opt (mapcar 'car filtered-alist) t)
+           (0 (prettify-symbols--compose-symbol ',filtered-alist)))))
     nil))
 
 (defvar-local prettify-symbols--keywords nil)