libgcc: support heap-based trampolines

Add support for heap-based trampolines on x86_64-linux, aarch64-linux,
and x86_64-darwin. Implement the __builtin_nested_func_ptr_created and
__builtin_nested_func_ptr_deleted functions for these targets.

Co-Authored-By: Maxim Blinov <maxim.blinov@embecosm.com>
Co-Authored-By: Iain Sandoe <iain@sandoe.co.uk>
Co-Authored-By: Francois-Xavier Coudert <fxcoudert@gcc.gnu.org>

libgcc/ChangeLog:

	* libgcc2.h (__builtin_nested_func_ptr_created): Declare.
	(__builtin_nested_func_ptr_deleted): Declare.
	* libgcc-std.ver.in: Add the new symbols.
	* config/aarch64/heap-trampoline.c: Implement heap-based
	trampolines for aarch64.
	* config/aarch64/t-heap-trampoline: Add rule to build
	config/aarch64/heap-trampoline.c
	* config/i386/heap-trampoline.c: Implement heap-based
	trampolines for x86_64.
	* config/i386/t-heap-trampoline: Add rule to build
	config/i386/heap-trampoline.cc
	* config.host: Handle --enable-heap-trampolines on
	x86_64-*-linux*, aarch64-*-linux*, x86_64-*-darwin*.

libgcc/config.host

@@ -423,6 +423,7 @@ aarch64*-*-linux*)
 	tmake_file="${tmake_file} ${cpu_type}/t-lse t-slibgcc-libgcc"
 	tmake_file="${tmake_file} ${cpu_type}/t-softfp t-softfp t-crtfm"
 	tmake_file="${tmake_file} t-dfprules"
+	tmake_file="${tmake_file} ${cpu_type}/t-heap-trampoline"
 	;;
 aarch64*-*-vxworks7*)
 	extra_parts="$extra_parts crtfastmath.o"
@@ -697,6 +698,7 @@ x86_64-*-darwin*)
 	tmake_file="$tmake_file i386/t-crtpc t-crtfm i386/t-msabi"
 	tm_file="$tm_file i386/darwin-lib.h"
 	extra_parts="$extra_parts crtprec32.o crtprec64.o crtprec80.o crtfastmath.o"
+	tmake_file="${tmake_file} i386/t-heap-trampoline"
 	;;
 i[34567]86-*-elfiamcu)
 	tmake_file="$tmake_file i386/t-crtstuff t-softfp-sfdftf i386/32/t-softfp i386/32/t-iamcu i386/t-softfp t-softfp t-dfprules"
@@ -763,6 +765,7 @@ x86_64-*-linux*)
 	tmake_file="${tmake_file} i386/t-crtpc t-crtfm i386/t-crtstuff t-dfprules"
 	tm_file="${tm_file} i386/elf-lib.h"
 	md_unwind_header=i386/linux-unwind.h
+	tmake_file="${tmake_file} i386/t-heap-trampoline"
 	;;
 x86_64-*-kfreebsd*-gnu)
 	extra_parts="$extra_parts crtprec32.o crtprec64.o crtprec80.o crtfastmath.o"

libgcc/config/aarch64/heap-trampoline.c

@@ -0,0 +1,172 @@
+/* Copyright The GNU Toolchain Authors. */
+
+#include <unistd.h>
+#include <sys/mman.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#if __APPLE__
+/* For pthread_jit_write_protect_np */
+#include <pthread.h>
+#endif
+
+void *allocate_trampoline_page (void);
+int get_trampolines_per_page (void);
+struct tramp_ctrl_data *allocate_tramp_ctrl (struct tramp_ctrl_data *parent);
+void *allocate_trampoline_page (void);
+
+void __builtin_nested_func_ptr_created (void *chain, void *func, void **dst);
+void __builtin_nested_func_ptr_deleted (void);
+
+#if defined(__gnu_linux__)
+static const uint32_t aarch64_trampoline_insns[] = {
+  0xd503245f, /* hint    34 */
+  0x580000b1, /* ldr     x17, .+20 */
+  0x580000d2, /* ldr     x18, .+24 */
+  0xd61f0220, /* br      x17 */
+  0xd5033f9f, /* dsb     sy */
+  0xd5033fdf /* isb */
+};
+
+#elif __APPLE__
+static const uint32_t aarch64_trampoline_insns[] = {
+  0xd503245f, /* hint    34 */
+  0x580000b1, /* ldr     x17, .+20 */
+  0x580000d0, /* ldr     x16, .+24 */
+  0xd61f0220, /* br      x17 */
+  0xd5033f9f, /* dsb     sy */
+  0xd5033fdf /* isb */
+};
+
+#else
+#error "Unsupported AArch64 platform for heap trampolines"
+#endif
+
+struct aarch64_trampoline {
+  uint32_t insns[6];
+  void *func_ptr;
+  void *chain_ptr;
+};
+
+struct tramp_ctrl_data
+{
+  struct tramp_ctrl_data *prev;
+
+  int free_trampolines;
+
+  /* This will be pointing to an executable mmap'ed page.  */
+  struct aarch64_trampoline *trampolines;
+};
+
+int
+get_trampolines_per_page (void)
+{
+  return getpagesize() / sizeof(struct aarch64_trampoline);
+}
+
+static _Thread_local struct tramp_ctrl_data *tramp_ctrl_curr = NULL;
+
+void *
+allocate_trampoline_page (void)
+{
+  void *page;
+
+#if defined(__gnu_linux__)
+  page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
+	       MAP_ANON | MAP_PRIVATE, 0, 0);
+#elif __APPLE__
+  page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
+	       MAP_ANON | MAP_PRIVATE | MAP_JIT, 0, 0);
+#else
+  page = MAP_FAILED;
+#endif
+
+  return page;
+}
+
+struct tramp_ctrl_data *
+allocate_tramp_ctrl (struct tramp_ctrl_data *parent)
+{
+  struct tramp_ctrl_data *p = malloc (sizeof (struct tramp_ctrl_data));
+  if (p == NULL)
+    return NULL;
+
+  p->trampolines = allocate_trampoline_page ();
+
+  if (p->trampolines == MAP_FAILED)
+    return NULL;
+
+  p->prev = parent;
+  p->free_trampolines = get_trampolines_per_page();
+
+  return p;
+}
+
+void
+__builtin_nested_func_ptr_created (void *chain, void *func, void **dst)
+{
+  if (tramp_ctrl_curr == NULL)
+    {
+      tramp_ctrl_curr = allocate_tramp_ctrl (NULL);
+      if (tramp_ctrl_curr == NULL)
+	abort ();
+    }
+
+  if (tramp_ctrl_curr->free_trampolines == 0)
+    {
+      void *tramp_ctrl = allocate_tramp_ctrl (tramp_ctrl_curr);
+      if (!tramp_ctrl)
+	abort ();
+
+      tramp_ctrl_curr = tramp_ctrl;
+    }
+
+  struct aarch64_trampoline *trampoline
+    = &tramp_ctrl_curr->trampolines[get_trampolines_per_page ()
+				    - tramp_ctrl_curr->free_trampolines];
+
+#if __APPLE__
+  /* Disable write protection for the MAP_JIT regions in this thread (see
+     https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon) */
+  pthread_jit_write_protect_np (0);
+#endif
+
+  memcpy (trampoline->insns, aarch64_trampoline_insns,
+	  sizeof(aarch64_trampoline_insns));
+  trampoline->func_ptr = func;
+  trampoline->chain_ptr = chain;
+
+#if __APPLE__
+  /* Re-enable write protection.  */
+  pthread_jit_write_protect_np (1);
+#endif
+
+  tramp_ctrl_curr->free_trampolines -= 1;
+
+  __builtin___clear_cache ((void *)trampoline->insns,
+			   ((void *)trampoline->insns + sizeof(trampoline->insns)));
+
+  *dst = &trampoline->insns;
+}
+
+void
+__builtin_nested_func_ptr_deleted (void)
+{
+  if (tramp_ctrl_curr == NULL)
+    abort ();
+
+  tramp_ctrl_curr->free_trampolines += 1;
+
+  if (tramp_ctrl_curr->free_trampolines == get_trampolines_per_page ())
+    {
+      if (tramp_ctrl_curr->prev == NULL)
+	return;
+
+      munmap (tramp_ctrl_curr->trampolines, getpagesize());
+      struct tramp_ctrl_data *prev = tramp_ctrl_curr->prev;
+      free (tramp_ctrl_curr);
+      tramp_ctrl_curr = prev;
+    }
+}

libgcc/config/aarch64/t-heap-trampoline

@@ -0,0 +1,19 @@
+# Copyright The GNU Toolchain Authors.
+
+# This file is part of GCC.
+#
+# GCC is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GCC is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+LIB2ADD += $(srcdir)/config/aarch64/heap-trampoline.c

libgcc/config/i386/heap-trampoline.c

@@ -0,0 +1,172 @@
+/* Copyright The GNU Toolchain Authors. */
+
+#include <unistd.h>
+#include <sys/mman.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#if __APPLE__ && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
+/* For pthread_jit_write_protect_np */
+#include <pthread.h>
+#endif
+
+void *allocate_trampoline_page (void);
+int get_trampolines_per_page (void);
+struct tramp_ctrl_data *allocate_tramp_ctrl (struct tramp_ctrl_data *parent);
+void *allocate_trampoline_page (void);
+
+void __builtin_nested_func_ptr_created (void *chain, void *func, void **dst);
+void __builtin_nested_func_ptr_deleted (void);
+
+static const uint8_t trampoline_insns[] = {
+  /* movabs $<chain>,%r11  */
+  0x49, 0xbb,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+  /* movabs $<func>,%r10  */
+  0x49, 0xba,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+  /* rex.WB jmpq *%r11  */
+  0x41, 0xff, 0xe3
+};
+
+union ix86_trampoline {
+  uint8_t insns[sizeof(trampoline_insns)];
+
+  struct __attribute__((packed)) fields {
+    uint8_t insn_0[2];
+    void *func_ptr;
+    uint8_t insn_1[2];
+    void *chain_ptr;
+    uint8_t insn_2[3];
+  } fields;
+};
+
+struct tramp_ctrl_data
+{
+  struct tramp_ctrl_data *prev;
+
+  int free_trampolines;
+
+  /* This will be pointing to an executable mmap'ed page.  */
+  union ix86_trampoline *trampolines;
+};
+
+int
+get_trampolines_per_page (void)
+{
+  return getpagesize() / sizeof(union ix86_trampoline);
+}
+
+static _Thread_local struct tramp_ctrl_data *tramp_ctrl_curr = NULL;
+
+void *
+allocate_trampoline_page (void)
+{
+  void *page;
+
+#if defined(__gnu_linux__)
+  page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
+	       MAP_ANON | MAP_PRIVATE, 0, 0);
+#elif __APPLE__
+# if  __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
+  page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
+	       MAP_ANON | MAP_PRIVATE | MAP_JIT, 0, 0);
+# else
+  page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
+	       MAP_ANON | MAP_PRIVATE, 0, 0);
+# endif
+#else
+  page = MAP_FAILED;
+#endif
+
+  return page;
+}
+
+struct tramp_ctrl_data *
+allocate_tramp_ctrl (struct tramp_ctrl_data *parent)
+{
+  struct tramp_ctrl_data *p = malloc (sizeof (struct tramp_ctrl_data));
+  if (p == NULL)
+    return NULL;
+
+  p->trampolines = allocate_trampoline_page ();
+
+  if (p->trampolines == MAP_FAILED)
+    return NULL;
+
+  p->prev = parent;
+  p->free_trampolines = get_trampolines_per_page();
+
+  return p;
+}
+
+void
+__builtin_nested_func_ptr_created (void *chain, void *func, void **dst)
+{
+  if (tramp_ctrl_curr == NULL)
+    {
+      tramp_ctrl_curr = allocate_tramp_ctrl (NULL);
+      if (tramp_ctrl_curr == NULL)
+	abort ();
+    }
+
+  if (tramp_ctrl_curr->free_trampolines == 0)
+    {
+      void *tramp_ctrl = allocate_tramp_ctrl (tramp_ctrl_curr);
+      if (!tramp_ctrl)
+	abort ();
+
+      tramp_ctrl_curr = tramp_ctrl;
+    }
+
+  union ix86_trampoline *trampoline
+    = &tramp_ctrl_curr->trampolines[get_trampolines_per_page ()
+				    - tramp_ctrl_curr->free_trampolines];
+
+#if __APPLE__ && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
+  /* Disable write protection for the MAP_JIT regions in this thread (see
+     https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon) */
+  pthread_jit_write_protect_np (0);
+#endif
+
+  memcpy (trampoline->insns, trampoline_insns,
+	  sizeof(trampoline_insns));
+  trampoline->fields.func_ptr = func;
+  trampoline->fields.chain_ptr = chain;
+
+#if __APPLE__ && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
+  /* Re-enable write protection.  */
+  pthread_jit_write_protect_np (1);
+#endif
+
+  tramp_ctrl_curr->free_trampolines -= 1;
+
+  __builtin___clear_cache ((void *)trampoline->insns,
+			   ((void *)trampoline->insns + sizeof(trampoline->insns)));
+
+  *dst = &trampoline->insns;
+}
+
+void
+__builtin_nested_func_ptr_deleted (void)
+{
+  if (tramp_ctrl_curr == NULL)
+    abort ();
+
+  tramp_ctrl_curr->free_trampolines += 1;
+
+  if (tramp_ctrl_curr->free_trampolines == get_trampolines_per_page ())
+    {
+      if (tramp_ctrl_curr->prev == NULL)
+	return;
+
+      munmap (tramp_ctrl_curr->trampolines, getpagesize());
+      struct tramp_ctrl_data *prev = tramp_ctrl_curr->prev;
+      free (tramp_ctrl_curr);
+      tramp_ctrl_curr = prev;
+    }
+}

libgcc/config/i386/t-heap-trampoline

@@ -0,0 +1,19 @@
+# Copyright The GNU Toolchain Authors.
+
+# This file is part of GCC.
+#
+# GCC is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GCC is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+LIB2ADD += $(srcdir)/config/i386/heap-trampoline.c

libgcc/libgcc-std.ver.in

@@ -1943,6 +1943,9 @@ GCC_4.8.0 {
 GCC_7.0.0 {
   __PFX__divmoddi4
   __PFX__divmodti4
+
+  __builtin_nested_func_ptr_created
+  __builtin_nested_func_ptr_deleted
 }
 
 %inherit GCC_14.0.0 GCC_7.0.0

libgcc/libgcc2.h

@@ -29,6 +29,9 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #pragma GCC visibility push(default)
 #endif
 
+extern void __builtin_nested_func_ptr_created (void *, void *, void **);
+extern void __builtin_nested_func_ptr_deleted (void);
+
 extern int __gcc_bcmp (const unsigned char *, const unsigned char *, size_t);
 extern void __clear_cache (void *, void *);
 extern void __eprintf (const char *, const char *, unsigned int, const char *)