diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc
index 85211565eb4..0bcc6a2d0ab 100644
--- a/gcc/config/rs6000/rs6000.cc
+++ b/gcc/config/rs6000/rs6000.cc
@@ -14706,7 +14706,12 @@ print_operand_address (FILE *file, rtx x)
 	fprintf (file, "@%s(%s)", SMALL_DATA_RELOC,
 		 reg_names[SMALL_DATA_REG]);
       else
-	gcc_assert (!TARGET_TOC);
+	{
+	  /* Do not support getting address directly from TOC, emit error.
+	     No more work is needed for !TARGET_TOC. */
+	  if (TARGET_TOC)
+	    output_operand_lossage ("%%a requires an address of memory");
+	}
     }
   else if (GET_CODE (x) == PLUS && REG_P (XEXP (x, 0))
 	   && REG_P (XEXP (x, 1)))
diff --git a/gcc/testsuite/gcc.target/powerpc/pr96866-1.c b/gcc/testsuite/gcc.target/powerpc/pr96866-1.c
new file mode 100644
index 00000000000..72e59a19753
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr96866-1.c
@@ -0,0 +1,18 @@
+/* The "%a" modifier can't get the address of extern symbol directly from TOC
+   with -fPIC, even the symbol is propagated for "X" constraint under -O2. */
+/* { dg-options "-fPIC -O2 -mno-pcrel" } */
+
+/* It's to verify no ICE here, ignore error messages about invalid 'asm'.  */
+/* { dg-excess-errors "pr96866-1.c" } */
+
+int x[2];
+
+int __attribute__ ((noipa))
+f1 (void)
+{
+  int n;
+  int *p = x;
+  *p++;
+  __asm__ volatile("ld %0, %a1" : "=r"(n) : "X"(p));
+  return n;
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/pr96866-2.c b/gcc/testsuite/gcc.target/powerpc/pr96866-2.c
new file mode 100644
index 00000000000..72bb15fa04f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr96866-2.c
@@ -0,0 +1,13 @@
+/* The "%a" modifier can't get the address of extern symbol directly from TOC
+   with -fPIC. */
+/* { dg-options "-fPIC -O2 -mno-pcrel" } */
+
+/* It's to verify no ICE here, ignore error messages about invalid 'asm'.  */
+/* { dg-excess-errors "pr96866-2.c" } */
+
+void
+f (void)
+{
+  extern int x;
+  __asm__ volatile("#%a0" ::"X"(&x));
+}