libgcc: aarch64: Add SME runtime support

The call ABI for SME (Scalable Matrix Extension) requires a number of
helper routines which are added to libgcc so they are tied to the
compiler version instead of the libc version. See
https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines

The routines are in shared libgcc and static libgcc eh, even though
they are not related to exception handling.  This is to avoid linking
a copy of the routines into dynamic linked binaries, because TPIDR2_EL0
block can be extended in the future which is better to handle in a
single place per process.

The support routines have to decide if SME is accessible or not. Linux
tells userspace if SME is accessible via AT_HWCAP2, otherwise a new
__aarch64_sme_accessible symbol was introduced that a libc can define.
Due to libgcc and libc build order, the symbol availability cannot be
checked so for __aarch64_sme_accessible an unistd.h feature test macro
is used while such detection mechanism is not available for __getauxval
so we rely on configure checks based on the target triplet.

Asm helper code is added to make writing the routines easier.

libgcc/ChangeLog:

	* config/aarch64/t-aarch64: Add sources to the build.
	* config/aarch64/__aarch64_have_sme.c: New file.
	* config/aarch64/__arm_sme_state.S: New file.
	* config/aarch64/__arm_tpidr2_restore.S: New file.
	* config/aarch64/__arm_tpidr2_save.S: New file.
	* config/aarch64/__arm_za_disable.S: New file.
	* config/aarch64/aarch64-asm.h: New file.
	* config/aarch64/libgcc-sme.ver: New file.
This commit is contained in:
Szabolcs Nagy 2022-11-15 14:08:55 +00:00
parent dbbfb52b0e
commit 328c17af77
8 changed files with 517 additions and 0 deletions

View file

@ -0,0 +1,75 @@
/* Initializer for SME support.
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 3, or (at your
option) any later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
#include "auto-target.h"
#ifndef inhibit_libc
/* For libc feature test macros. */
# include <unistd.h>
#endif
#if __ARM_FEATURE_SME
/* Avoid runtime SME detection if libgcc is built with SME. */
# define HAVE_SME_CONST const
# define HAVE_SME_VALUE 1
#elif HAVE___GETAUXVAL
/* SME access detection on Linux. */
# define HAVE_SME_CONST
# define HAVE_SME_VALUE 0
# define HAVE_SME_CTOR sme_accessible ()
# define AT_HWCAP2 26
# define HWCAP2_SME (1 << 23)
unsigned long int __getauxval (unsigned long int);
static _Bool
sme_accessible (void)
{
unsigned long hwcap2 = __getauxval (AT_HWCAP2);
return (hwcap2 & HWCAP2_SME) != 0;
}
#elif __LIBC___AARCH64_SME_ACCESSIBLE
/* Alternative SME access detection. */
# define HAVE_SME_CONST
# define HAVE_SME_VALUE 0
# define HAVE_SME_CTOR __aarch64_sme_accessible ()
_Bool __aarch64_sme_accessible (void);
#else
# define HAVE_SME_CONST const
# define HAVE_SME_VALUE 0
#endif
/* Define the symbol gating SME support in libgcc. */
HAVE_SME_CONST _Bool __aarch64_have_sme
__attribute__((visibility("hidden"), nocommon)) = HAVE_SME_VALUE;
#ifdef HAVE_SME_CTOR
/* Use a higher priority to ensure it runs before user constructors
with priority 100. */
static void __attribute__((constructor (90)))
init_have_sme (void)
{
__aarch64_have_sme = HAVE_SME_CTOR;
}
#endif

View file

@ -0,0 +1,55 @@
/* Support routine for SME.
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 3, or (at your
option) any later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
#include "aarch64-asm.h"
/* Query SME state. Call ABI:
- Private ZA, streaming-compatible.
- x2-x15, x19-x29, sp and fp regs are call preserved.
- Takes no argument.
- Returns SME state in x0 and TPIDR2_EL0 in x1. */
.hidden __aarch64_have_sme
variant_pcs (__arm_sme_state)
ENTRY (__arm_sme_state)
/* Check if SME is available. */
adrp x1, __aarch64_have_sme
ldrb w1, [x1, :lo12:__aarch64_have_sme]
cbz w1, L(nosme)
/* Expose the bottom 2 bits of svcr (SM, ZA) in x0 and set the
top 2 bits indicating that SME and TPIDR2_EL0 are available. */
.inst 0xd53b4240 /* mrs x0, svcr */
.inst 0xd53bd0a1 /* mrs x1, tpidr2_el0 */
and x0, x0, 3
orr x0, x0, 0xc000000000000000
ret
L(nosme):
mov x0, 0
mov x1, 0
ret
END (__arm_sme_state)

View file

@ -0,0 +1,89 @@
/* Support routine for SME.
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 3, or (at your
option) any later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
#include "aarch64-asm.h"
/* Used for lazy ZA restore. Call ABI:
- Shared ZA, streaming-compatible.
- x0 is a pointer to a TPIDR2 block.
- x0-x13, x19-x29, sp and fp regs are call preserved.
- Does not return a value.
- Can abort on failure (then registers are not preserved). */
variant_pcs (__arm_tpidr2_restore)
ENTRY (__arm_tpidr2_restore)
.inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */
cbnz x14, L(fail)
/* check reserved bytes. */
ldrh w15, [x0, 10]
ldr w16, [x0, 12]
orr w15, w15, w16
cbnz w15, L(fail)
ldr x16, [x0]
cbz x16, L(end)
ldrh w17, [x0, 8]
cbz w17, L(end)
/* x0: blk, x14: 0, x15: 0,
x16: za_save_buffer, x17: num_za_save_slices. */
L(restore_loop):
.inst 0xe1006200 /* ldr za[w15, 0], [x16] */
.inst 0xe1006201 /* ldr za[w15, 1], [x16, 1, mul vl] */
.inst 0xe1006202 /* ldr za[w15, 2], [x16, 2, mul vl] */
.inst 0xe1006203 /* ldr za[w15, 3], [x16, 3, mul vl] */
.inst 0xe1006204 /* ldr za[w15, 4], [x16, 4, mul vl] */
.inst 0xe1006205 /* ldr za[w15, 5], [x16, 5, mul vl] */
.inst 0xe1006206 /* ldr za[w15, 6], [x16, 6, mul vl] */
.inst 0xe1006207 /* ldr za[w15, 7], [x16, 7, mul vl] */
.inst 0xe1006208 /* ldr za[w15, 8], [x16, 8, mul vl] */
.inst 0xe1006209 /* ldr za[w15, 9], [x16, 9, mul vl] */
.inst 0xe100620a /* ldr za[w15, 10], [x16, 10, mul vl] */
.inst 0xe100620b /* ldr za[w15, 11], [x16, 11, mul vl] */
.inst 0xe100620c /* ldr za[w15, 12], [x16, 12, mul vl] */
.inst 0xe100620d /* ldr za[w15, 13], [x16, 13, mul vl] */
.inst 0xe100620e /* ldr za[w15, 14], [x16, 14, mul vl] */
.inst 0xe100620f /* ldr za[w15, 15], [x16, 15, mul vl] */
add w15, w15, 16
.inst 0x04305a10 /* addsvl x16, x16, 16 */
cmp w17, w15
bhi L(restore_loop)
L(end):
ret
L(fail):
PACIASP
stp x29, x30, [sp, -32]!
.cfi_adjust_cfa_offset 32
.cfi_rel_offset x29, 0
.cfi_rel_offset x30, 8
mov x29, sp
.inst 0x04e0e3f0 /* cntd x16 */
str x16, [sp, 16]
.cfi_rel_offset 46, 16
.inst 0xd503467f /* smstop */
bl abort
END (__arm_tpidr2_restore)

View file

@ -0,0 +1,101 @@
/* Support routine for SME.
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 3, or (at your
option) any later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
#include "aarch64-asm.h"
/* Used for lazy ZA save. Call ABI:
- Private ZA, streaming-compatible.
- x0-x13, x19-x29, sp and fp regs are call preserved.
- Takes no argument.
- Does not return a value.
- Can abort on failure (then registers are not preserved). */
.hidden __aarch64_have_sme
variant_pcs (__arm_tpidr2_save)
ENTRY (__arm_tpidr2_save)
/* Check if SME is available. */
adrp x14, __aarch64_have_sme
ldrb w14, [x14, :lo12:__aarch64_have_sme]
cbz w14, L(end)
.inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */
cbz x14, L(end)
/* check reserved bytes. */
ldrh w15, [x14, 10]
ldr w16, [x14, 12]
orr w15, w15, w16
cbnz w15, L(fail)
ldr x16, [x14]
cbz x16, L(end)
ldrh w17, [x14, 8]
cbz w17, L(end)
/* x14: tpidr2, x15: 0,
x16: za_save_buffer, x17: num_za_save_slices. */
L(save_loop):
.inst 0xe1206200 /* str za[w15, 0], [x16] */
.inst 0xe1206201 /* str za[w15, 1], [x16, 1, mul vl] */
.inst 0xe1206202 /* str za[w15, 2], [x16, 2, mul vl] */
.inst 0xe1206203 /* str za[w15, 3], [x16, 3, mul vl] */
.inst 0xe1206204 /* str za[w15, 4], [x16, 4, mul vl] */
.inst 0xe1206205 /* str za[w15, 5], [x16, 5, mul vl] */
.inst 0xe1206206 /* str za[w15, 6], [x16, 6, mul vl] */
.inst 0xe1206207 /* str za[w15, 7], [x16, 7, mul vl] */
.inst 0xe1206208 /* str za[w15, 8], [x16, 8, mul vl] */
.inst 0xe1206209 /* str za[w15, 9], [x16, 9, mul vl] */
.inst 0xe120620a /* str za[w15, 10], [x16, 10, mul vl] */
.inst 0xe120620b /* str za[w15, 11], [x16, 11, mul vl] */
.inst 0xe120620c /* str za[w15, 12], [x16, 12, mul vl] */
.inst 0xe120620d /* str za[w15, 13], [x16, 13, mul vl] */
.inst 0xe120620e /* str za[w15, 14], [x16, 14, mul vl] */
.inst 0xe120620f /* str za[w15, 15], [x16, 15, mul vl] */
add w15, w15, 16
.inst 0x04305a10 /* addsvl x16, x16, 16 */
cmp w17, w15
bhi L(save_loop)
L(end):
ret
L(fail):
PACIASP
stp x29, x30, [sp, -32]!
.cfi_adjust_cfa_offset 32
.cfi_rel_offset x29, 0
.cfi_rel_offset x30, 8
mov x29, sp
.inst 0x04e0e3f0 /* cntd x16 */
str x16, [sp, 16]
.cfi_rel_offset 46, 16
.inst 0xd503467f /* smstop */
bl abort
END (__arm_tpidr2_save)
/* Hidden alias used by __arm_za_disable. */
.global __libgcc_arm_tpidr2_save
.hidden __libgcc_arm_tpidr2_save
.set __libgcc_arm_tpidr2_save, __arm_tpidr2_save

View file

@ -0,0 +1,65 @@
/* Support routine for SME.
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 3, or (at your
option) any later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
#include "aarch64-asm.h"
/* Disable ZA. Call ABI:
- Private ZA, streaming-compatible.
- x0-x13, x19-x29, sp and fp regs are call preserved.
- Takes no argument.
- Does not return a value.
- Can abort on failure (then registers are not preserved). */
.hidden __aarch64_have_sme
.hidden __libgcc_arm_tpidr2_save
variant_pcs (__arm_za_disable)
ENTRY (__arm_za_disable)
/* Check if SME is available. */
adrp x14, __aarch64_have_sme
ldrb w14, [x14, :lo12:__aarch64_have_sme]
cbz w14, L(end)
.inst 0xd53bd0ae /* mrs x14, tpidr2_el0 */
cbz x14, L(end)
PACIASP
stp x29, x30, [sp, -16]!
.cfi_adjust_cfa_offset 16
.cfi_rel_offset x29, 0
.cfi_rel_offset x30, 8
mov x29, sp
bl __libgcc_arm_tpidr2_save
.inst 0xd51bd0bf /* msr tpidr2_el0, xzr */
.inst 0xd503447f /* smstop za */
ldp x29, x30, [sp], 16
.cfi_adjust_cfa_offset -16
.cfi_restore x29
.cfi_restore x30
AUTIASP
L(end):
ret
END (__arm_za_disable)

View file

@ -0,0 +1,98 @@
/* AArch64 asm definitions.
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 3, or (at your
option) any later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.
You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
#include "auto-target.h"
#define L(label) .L ## label
/* Marking variant PCS symbol references is important for PLT calls
otherwise it is for documenting the PCS in the symbol table. */
#ifdef HAVE_AS_VARIANT_PCS
# define variant_pcs(name) .variant_pcs name
#else
# define variant_pcs(name)
#endif
/* GNU_PROPERTY_AARCH64_* macros from elf.h for use in asm code. */
#define FEATURE_1_AND 0xc0000000
#define FEATURE_1_BTI 1
#define FEATURE_1_PAC 2
/* Supported features based on the code generation options. */
#if defined(__ARM_FEATURE_BTI_DEFAULT)
# define BTI_FLAG FEATURE_1_BTI
# define BTI_C hint 34
#else
# define BTI_FLAG 0
# define BTI_C
#endif
#if __ARM_FEATURE_PAC_DEFAULT & 3
# define PAC_FLAG FEATURE_1_PAC
# define PACIASP hint 25; .cfi_window_save
# define AUTIASP hint 29; .cfi_window_save
#else
# define PAC_FLAG 0
# define PACIASP
# define AUTIASP
#endif
/* Add a NT_GNU_PROPERTY_TYPE_0 note. */
#define GNU_PROPERTY(type, value) \
.section .note.gnu.property, "a"; \
.p2align 3; \
.word 4; \
.word 16; \
.word 5; \
.asciz "GNU"; \
.word type; \
.word 4; \
.word value; \
.word 0; \
.previous
#if defined(__linux__) || defined(__FreeBSD__)
/* Do not require executable stack. */
.section .note.GNU-stack, "", %progbits
.previous
/* Add GNU property note if built with branch protection. */
# if (BTI_FLAG|PAC_FLAG) != 0
GNU_PROPERTY (FEATURE_1_AND, BTI_FLAG|PAC_FLAG)
# endif
#endif
#define ENTRY_ALIGN(name, align) \
.global name; \
.type name,%function; \
.balign align; \
name: \
.cfi_startproc; \
BTI_C
#define ENTRY(name) ENTRY_ALIGN(name, 16)
#define END(name) \
.cfi_endproc; \
.size name, .-name

View file

@ -0,0 +1,24 @@
# Copyright (C) 2023 Free Software Foundation, Inc.
#
# This file is part of GCC.
#
# GCC is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GCC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GCC; see the file COPYING3. If not see
# <http://www.gnu.org/licenses/>.
GCC_14.0 {
__arm_sme_state
__arm_tpidr2_restore
__arm_tpidr2_save
__arm_za_disable
}

View file

@ -19,3 +19,13 @@
# <http://www.gnu.org/licenses/>.
LIB2ADD += $(srcdir)/config/aarch64/sync-cache.c
# Add sme runtime to shared libgcc
LIB2ADDEH += \
$(srcdir)/config/aarch64/__aarch64_have_sme.c \
$(srcdir)/config/aarch64/__arm_sme_state.S \
$(srcdir)/config/aarch64/__arm_tpidr2_restore.S \
$(srcdir)/config/aarch64/__arm_tpidr2_save.S \
$(srcdir)/config/aarch64/__arm_za_disable.S
SHLIB_MAPFILES += $(srcdir)/config/aarch64/libgcc-sme.ver