diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in index 6c4dc79ab715..5ef94380ec02 100644 --- a/libgcc/Makefile.in +++ b/libgcc/Makefile.in @@ -1048,8 +1048,9 @@ crtbeginT$(objext): $(srcdir)/crtstuff.c crtoffloadbegin$(objext): $(srcdir)/offloadstuff.c $(crt_compile) $(CRTSTUFF_T_CFLAGS) -c $< -DCRT_BEGIN +# crtoffloadend contains a constructor with calls to libgomp, so build as PIC. crtoffloadend$(objext): $(srcdir)/offloadstuff.c - $(crt_compile) $(CRTSTUFF_T_CFLAGS) -c $< -DCRT_END + $(crt_compile) $(CRTSTUFF_T_CFLAGS) $(PICFLAG) -c $< -DCRT_END crtoffloadtable$(objext): $(srcdir)/offloadstuff.c $(crt_compile) $(CRTSTUFF_T_CFLAGS) -c $< -DCRT_TABLE diff --git a/libgcc/offloadstuff.c b/libgcc/offloadstuff.c index 4e1c4d41dd57..6796aa2a83c1 100644 --- a/libgcc/offloadstuff.c +++ b/libgcc/offloadstuff.c @@ -63,6 +63,19 @@ const void *const __offload_vars_end[0] __attribute__ ((__used__, visibility ("hidden"), section (OFFLOAD_VAR_TABLE_SECTION_NAME))) = { }; +extern void GOMP_offload_register_ver (unsigned, const void *, int, + const void *); +extern const void *const __OFFLOAD_TABLE__[0] __attribute__ ((weak)); +static void __attribute__((constructor)) +init_non_offload (void) +{ + /* If an OpenMP program has no offloading, post-offload_register callbacks + that need to run will require a call to GOMP_offload_register_ver, in + order to properly trigger those callbacks during init. */ + if (__OFFLOAD_TABLE__ == NULL) + GOMP_offload_register_ver (0, NULL, 0, NULL); +} + #elif defined CRT_TABLE extern const void *const __offload_func_table[]; diff --git a/libgfortran/gfortran.map b/libgfortran/gfortran.map index db9b86cb183f..abb4def5b808 100644 --- a/libgfortran/gfortran.map +++ b/libgfortran/gfortran.map @@ -1765,3 +1765,8 @@ GFORTRAN_13 { __ieee_exceptions_MOD_ieee_get_modes; __ieee_exceptions_MOD_ieee_set_modes; } GFORTRAN_12; + +GFORTRAN_14 { + global: + _gfortran_mem_allocators_init; +} GFORTRAN_13; diff --git a/libgfortran/libgfortran.h b/libgfortran/libgfortran.h index b972b0aa0cd5..535a3d67b25f 100644 --- a/libgfortran/libgfortran.h +++ b/libgfortran/libgfortran.h @@ -878,6 +878,11 @@ internal_proto(xrealloc); extern void xfree (void *); internal_proto(xfree); +#if !defined(LIBGFOR_MINIMAL) && defined(__linux__) +extern void mem_allocators_init (void *, void *, void *, void *); +export_proto(mem_allocators_init); +#endif + /* environ.c */ extern void init_variables (void); diff --git a/libgfortran/runtime/main.c b/libgfortran/runtime/main.c index bcdcb20a1236..b676ccc1df9e 100644 --- a/libgfortran/runtime/main.c +++ b/libgfortran/runtime/main.c @@ -61,9 +61,16 @@ get_args (int *argc, char ***argv) /* Initialize the runtime library. */ -static void __attribute__((constructor)) -init (void) +static void +do_init (void) { +#if !defined(LIBGFOR_MINIMAL) && defined(__linux__) + static bool do_init_ran = false; + if (do_init_ran) + return; + do_init_ran = true; +#endif + /* Must be first */ init_variables (); @@ -82,5 +89,37 @@ init (void) static void __attribute__((destructor)) cleanup (void) { +#if !defined(LIBGFOR_MINIMAL) && defined(__linux__) + static bool cleanup_ran = false; + if (cleanup_ran) + return; + cleanup_ran = true; +#endif + close_units (); } + +#if !defined(LIBGFOR_MINIMAL) && defined(__linux__) +extern void __attribute__((weak)) +GOMP_post_offload_register_callback (void (*func)(void)); + +extern void __attribute__((weak)) +GOMP_pre_gomp_target_fini_callback (void (*func)(void)); +#endif + +static void __attribute__((constructor)) +init (void) +{ +#if !defined(LIBGFOR_MINIMAL) && defined(__linux__) + if (GOMP_post_offload_register_callback) + { + GOMP_post_offload_register_callback (do_init); + GOMP_pre_gomp_target_fini_callback (cleanup); + return; + } +#endif + + /* If libgomp is not present, then we can go ahead and call do_init + directly. */ + do_init (); +} diff --git a/libgfortran/runtime/memory.c b/libgfortran/runtime/memory.c index a1539836c9c2..59b53843ffd8 100644 --- a/libgfortran/runtime/memory.c +++ b/libgfortran/runtime/memory.c @@ -26,6 +26,28 @@ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see #include "libgfortran.h" #include +#if !defined(LIBGFOR_MINIMAL) && defined(__linux__) +static void * (*gfortran_malloc)(size_t) = malloc; +static void * (*gfortran_calloc)(size_t, size_t) = calloc; +static void * (*gfortran_realloc)(void *, size_t) = realloc; +static void (*gfortran_free)(void *) = free; + +void +mem_allocators_init (void *malloc_ptr, void *calloc_ptr, + void *realloc_ptr, void *free_ptr) +{ + gfortran_malloc = malloc_ptr; + gfortran_calloc = calloc_ptr; + gfortran_realloc = realloc_ptr; + gfortran_free = free_ptr; +} + +#else +#define gfortran_malloc malloc +#define gfortran_calloc calloc +#define gfortran_realloc realloc +#define gfortran_free free +#endif void * xmalloc (size_t n) @@ -35,7 +57,7 @@ xmalloc (size_t n) if (n == 0) n = 1; - p = malloc (n); + p = gfortran_malloc (n); if (p == NULL) os_error ("Memory allocation failed"); @@ -57,7 +79,7 @@ xmallocarray (size_t nmemb, size_t size) os_error ("Integer overflow in xmallocarray"); } - p = malloc (prod); + p = gfortran_malloc (prod); if (!p) os_error ("Memory allocation failed in xmallocarray"); @@ -73,7 +95,7 @@ xcalloc (size_t nmemb, size_t size) if (!nmemb || !size) nmemb = size = 1; - void *p = calloc (nmemb, size); + void *p = gfortran_calloc (nmemb, size); if (!p) os_error ("Allocating cleared memory failed"); @@ -86,7 +108,7 @@ xrealloc (void *ptr, size_t size) if (size == 0) size = 1; - void *newp = realloc (ptr, size); + void *newp = gfortran_realloc (ptr, size); if (!newp) os_error ("Memory allocation failure in xrealloc"); @@ -96,5 +118,5 @@ xrealloc (void *ptr, size_t size) void xfree (void *ptr) { - free (ptr); + gfortran_free (ptr); } diff --git a/libgomp/libgomp.map b/libgomp/libgomp.map index a9621c0a68c4..694a8f1a3598 100644 --- a/libgomp/libgomp.map +++ b/libgomp/libgomp.map @@ -423,6 +423,12 @@ GOMP_5.1.1 { GOMP_taskwait_depend_nowait; } GOMP_5.1; +GOMP_5.1.2 { + global: + GOMP_post_offload_register_callback; + GOMP_pre_gomp_target_fini_callback; +} GOMP_5.1.1; + OACC_2.0 { global: acc_get_num_devices; diff --git a/libgomp/target.c b/libgomp/target.c index 6dd78e1e63dd..20a7f3776c2d 100644 --- a/libgomp/target.c +++ b/libgomp/target.c @@ -3153,6 +3153,70 @@ gomp_requires_to_name (char *buf, size_t size, int requires_mask) (p == buf ? "" : ", ")); } +/* Macro to define a callback set with a name, and routine to register + a callback function into set. */ +#define GOMP_DEFINE_CALLBACK_SET(name) \ + static unsigned int num_ ## name ## _callbacks = 0; \ + static void (*name ## _callbacks[4])(void); \ + void GOMP_ ## name ## _callback (void (*fn)(void)) \ + { \ + if (num_ ## name ## _callbacks \ + < (sizeof (name ## _callbacks) \ + / sizeof (name ## _callbacks[0]))) \ + { \ + name ## _callbacks[num_ ## name ## _callbacks] = fn; \ + num_ ## name ## _callbacks += 1; \ + } \ + } +GOMP_DEFINE_CALLBACK_SET(post_offload_register) +GOMP_DEFINE_CALLBACK_SET(pre_gomp_target_fini) +#undef GOMP_DEFINE_CALLBACK_SET + +/* Routines to insert into libgfortran, under unified_shared_memory. */ +static void * +libgfortran_malloc_usm (size_t size) +{ + return omp_alloc (size, ompx_unified_shared_mem_alloc); +} + +static void * +libgfortran_calloc_usm (size_t n, size_t size) +{ + return omp_calloc (n, size, ompx_unified_shared_mem_alloc); +} + +static void * +libgfortran_realloc_usm (void *ptr, size_t size) +{ + return omp_realloc (ptr, size, ompx_unified_shared_mem_alloc, + ompx_unified_shared_mem_alloc); +} + +static void +libgfortran_free_usm (void *ptr) +{ + omp_free (ptr, ompx_unified_shared_mem_alloc); +} + +extern void __attribute__((weak)) +_gfortran_mem_allocators_init (void *, void *, void *, void *); + +static void +gomp_libgfortran_omp_allocators_init (int omp_requires_mask) +{ + static bool init = false; + if (init) + return; + init = true; + + if ((omp_requires_mask & GOMP_REQUIRES_UNIFIED_SHARED_MEMORY) + && _gfortran_mem_allocators_init != NULL) + _gfortran_mem_allocators_init (libgfortran_malloc_usm, + libgfortran_calloc_usm, + libgfortran_realloc_usm, + libgfortran_free_usm); +} + /* This function should be called from every offload image while loading. It gets the descriptor of the host func and var tables HOST_TABLE, TYPE of the target, and DATA. */ @@ -3163,6 +3227,9 @@ GOMP_offload_register_ver (unsigned version, const void *host_table, { int i; + if (host_table == NULL) + goto end; + if (GOMP_VERSION_LIB (version) > GOMP_VERSION) gomp_fatal ("Library too old for offload (version %u < %u)", GOMP_VERSION, GOMP_VERSION_LIB (version)); @@ -3229,6 +3296,14 @@ GOMP_offload_register_ver (unsigned version, const void *host_table, num_offload_images++; gomp_mutex_unlock (®ister_lock); + + /* Call into libgfortran to initialize OpenMP memory allocators. */ + gomp_libgfortran_omp_allocators_init (omp_requires_mask); + + end: + for (int i = 0; i < num_post_offload_register_callbacks; i++) + post_offload_register_callbacks[i] (); + num_post_offload_register_callbacks = 0; } /* Legacy entry point. */ @@ -3341,7 +3416,7 @@ gomp_unload_device (struct gomp_device_descr *devicep) if (devicep->state == GOMP_DEVICE_INITIALIZED) { unsigned i; - + /* Unload from device all images registered at the moment. */ for (i = 0; i < num_offload_images; i++) { @@ -6507,6 +6582,13 @@ gomp_target_init (void) devices = devs; if (atexit (gomp_target_fini) != 0) gomp_fatal ("atexit failed"); + + /* Register 'pre_gomp_target_fini' callbacks to run before gomp_target_fini + during finalization. */ + for (int i = 0; i < num_pre_gomp_target_fini_callbacks; i++) + if (atexit (pre_gomp_target_fini_callbacks[i]) != 0) + gomp_fatal ("atexit failed"); + num_pre_gomp_target_fini_callbacks = 0; } #else /* PLUGIN_SUPPORT */ diff --git a/libgomp/testsuite/libgomp.fortran/target-unified_shared_memory-1.f90 b/libgomp/testsuite/libgomp.fortran/target-unified_shared_memory-1.f90 new file mode 100644 index 000000000000..f82b1c7527cd --- /dev/null +++ b/libgomp/testsuite/libgomp.fortran/target-unified_shared_memory-1.f90 @@ -0,0 +1,13 @@ +! { dg-do run } + +program requires_unified_shared_memory + character(32) :: str + !$omp requires unified_shared_memory + + str = trim (str) + + !$omp target + block + end block + +end program requires_unified_shared_memory