LLVM OpenMP 22.0.0git
|
Go to the source code of this file.
Classes | |
class | kmp_tgt_allocator_t |
Support OMP 6.0 target memory management Expected offload runtime entries. More... | |
class | kmp_tgt_memspace_list_t |
Maintain a list of target memory spaces that are identified with the requested information. More... | |
struct | kmp_mem_desc |
struct | kmp_mem_descr |
Macros | |
#define | IS_POWER_OF_TWO(n) (((n) & ((n)-1)) == 0) |
#define | KMP_IS_TARGET_MEM_SPACE(MS) |
#define | KMP_IS_TARGET_MEM_ALLOC(MA) |
#define | KMP_IS_PREDEF_MEM_SPACE(MS) |
Typedefs | |
typedef struct kmp_mem_desc | kmp_mem_desc_t |
typedef struct kmp_mem_descr | kmp_mem_descr_t |
#define IS_POWER_OF_TWO | ( | n | ) | (((n) & ((n)-1)) == 0) |
Definition at line 1121 of file kmp_alloc.cpp.
#define KMP_IS_PREDEF_MEM_SPACE | ( | MS | ) |
Definition at line 1279 of file kmp_alloc.cpp.
#define KMP_IS_TARGET_MEM_ALLOC | ( | MA | ) |
Definition at line 1274 of file kmp_alloc.cpp.
#define KMP_IS_TARGET_MEM_SPACE | ( | MS | ) |
Definition at line 1269 of file kmp_alloc.cpp.
typedef struct kmp_mem_desc kmp_mem_desc_t |
typedef struct kmp_mem_descr kmp_mem_descr_t |
Definition at line 2396 of file kmp_alloc.cpp.
Definition at line 2486 of file kmp_alloc.cpp.
References ___kmp_allocate_align(), __kmp_align_alloc, KE_TRACE, KMP_SRC_LOC_PARM, and size.
Definition at line 2401 of file kmp_alloc.cpp.
References _malloc_src_loc, alignment, KE_TRACE, KMP_DEBUG_ASSERT, KMP_FATAL, KMP_MB, KMP_SRC_LOC_PARM, malloc_src_loc, kmp_mem_descr::ptr_aligned, kmp_mem_descr::ptr_allocated, size, kmp_mem_descr::size_aligned, and kmp_mem_descr::size_allocated.
Referenced by ___kmp_allocate(), and ___kmp_page_allocate().
Definition at line 2512 of file kmp_alloc.cpp.
References _free_src_loc, CACHE_LINE, free_src_loc, KE_TRACE, KMP_ASSERT, KMP_DEBUG_ASSERT, KMP_MB, KMP_SRC_LOC_PARM, kmp_mem_descr::ptr_aligned, kmp_mem_descr::ptr_allocated, kmp_mem_descr::size_aligned, and kmp_mem_descr::size_allocated.
Definition at line 2499 of file kmp_alloc.cpp.
References ___kmp_allocate_align(), KE_TRACE, KMP_SRC_LOC_PARM, and size.
void * ___kmp_thread_calloc | ( | kmp_info_t * | th, |
size_t | nelem, | ||
size_t elsize | KMP_SRC_LOC_DECL | ||
) |
Definition at line 1208 of file kmp_alloc.cpp.
References KE_TRACE, and KMP_SRC_LOC_PARM.
void ___kmp_thread_free | ( | kmp_info_t * | th, |
void *ptr | KMP_SRC_LOC_DECL | ||
) |
Definition at line 1227 of file kmp_alloc.cpp.
References KE_TRACE, and KMP_SRC_LOC_PARM.
void * ___kmp_thread_malloc | ( | kmp_info_t * | th, |
size_t size | KMP_SRC_LOC_DECL | ||
) |
Definition at line 1199 of file kmp_alloc.cpp.
References KE_TRACE, KMP_SRC_LOC_PARM, and size.
void * ___kmp_thread_realloc | ( | kmp_info_t * | th, |
void * | ptr, | ||
size_t size | KMP_SRC_LOC_DECL | ||
) |
Definition at line 1218 of file kmp_alloc.cpp.
References KE_TRACE, KMP_SRC_LOC_PARM, and size.
void ___kmpc_free | ( | int | gtid, |
void * | ptr, | ||
omp_allocator_handle_t | allocator | ||
) |
Definition at line 2293 of file kmp_alloc.cpp.
References __kmp_hwloc_available, __kmp_memkind_available, __kmp_target_mem_available, __kmp_tgt_allocator, __kmp_thread_free, __kmp_thread_from_gtid(), __kmp_threads, kmp_mem_desc::allocator, CCAST, kmp_allocator_t::fb_data, KMP_DEBUG_ASSERT, KMP_IS_TARGET_MEM_ALLOC, KMP_IS_TARGET_MEM_SPACE, kmp_max_mem_alloc, kmp_max_mem_space, kmp_mk_free, kmp_target_free_device, kmp_target_free_host, kmp_target_free_shared, kmp_target_unlock_mem, KMP_TEST_THEN_ADD64, llvm_omp_target_device_mem_alloc, llvm_omp_target_host_mem_alloc, llvm_omp_target_shared_mem_alloc, kmp_allocator_t::memkind, kmp_allocator_t::memspace, mk_dax_kmem_all, mk_default, mk_hbw_preferred, kmp_tgt_allocator_t::omp_free(), omp_high_bw_mem_alloc, omp_large_cap_mem_alloc, kmp_allocator_t::pinned, kmp_allocator_t::pool_size, kmp_allocator_t::pool_used, kmp_mem_desc::ptr_align, kmp_mem_desc::ptr_alloc, RCAST, kmp_mem_desc::size_a, and void.
Referenced by __kmp_realloc(), __kmpc_free(), KMP_API_NAME_GOMP_FREE(), and omp_free().
void * __kmp_alloc | ( | int | gtid, |
size_t | algn, | ||
size_t | size, | ||
omp_allocator_handle_t | allocator | ||
) |
Definition at line 1910 of file kmp_alloc.cpp.
References __kmp_alloc(), __kmp_hwloc_available, __kmp_init_serial, __kmp_memkind_available, __kmp_target_mem_available, __kmp_tgt_allocator, __kmp_thread_from_gtid(), __kmp_thread_malloc, __kmp_threads, addr, kmp_allocator_t::alignment, alignment, kmp_mem_desc::allocator, kmp_allocator_t::fb, kmp_allocator_t::fb_data, KE_TRACE, KMP_ASSERT, KMP_DEBUG_ASSERT, KMP_INFORM, KMP_IS_TARGET_MEM_ALLOC, KMP_IS_TARGET_MEM_SPACE, kmp_max_mem_alloc, kmp_max_mem_space, KMP_MB, kmp_mk_alloc, kmp_target_alloc_device, kmp_target_alloc_host, kmp_target_alloc_shared, kmp_target_lock_mem, KMP_TEST_THEN_ADD64, KMP_WARNING, llvm_omp_target_host_mem_alloc, llvm_omp_target_host_mem_space, llvm_omp_target_shared_mem_alloc, llvm_omp_target_shared_mem_space, kmp_allocator_t::memkind, kmp_allocator_t::memspace, mk_dax_kmem_all, mk_default, mk_hbw_preferred, kmp_tgt_allocator_t::omp_alloc(), omp_atv_abort_fb, omp_atv_allocator_fb, omp_atv_default_mem_fb, omp_atv_environment, omp_atv_interleaved, omp_cgroup_mem_alloc, omp_const_mem_alloc, omp_default_mem_alloc, omp_high_bw_mem_alloc, omp_large_cap_mem_alloc, omp_low_lat_mem_alloc, omp_null_allocator, omp_pteam_mem_alloc, omp_thread_mem_alloc, kmp_allocator_t::pinned, kmp_allocator_t::pool_size, kmp_allocator_t::pool_used, kmp_mem_desc::ptr_align, kmp_mem_desc::ptr_alloc, RCAST, size, kmp_mem_desc::size_a, and kmp_mem_desc::size_orig.
Referenced by __kmp_alloc(), __kmp_calloc(), __kmp_realloc(), __kmpc_aligned_alloc(), __kmpc_alloc(), KMP_API_NAME_GOMP_ALLOC(), omp_aligned_alloc(), and omp_alloc().
void * __kmp_calloc | ( | int | gtid, |
size_t | algn, | ||
size_t | nmemb, | ||
size_t | size, | ||
omp_allocator_handle_t | allocator | ||
) |
Definition at line 2227 of file kmp_alloc.cpp.
References __kmp_alloc(), __kmp_init_serial, __kmp_threads, kmp_allocator_t::fb, KMP_ASSERT, KMP_DEBUG_ASSERT, omp_atv_abort_fb, omp_null_allocator, RCAST, and size.
Referenced by __kmpc_calloc(), omp_aligned_calloc(), and omp_calloc().
void __kmp_fini_memkind | ( | ) |
Definition at line 1523 of file kmp_alloc.cpp.
References __kmp_memkind_available, h_memkind, KE_TRACE, kmp_mk_alloc, kmp_mk_check, kmp_mk_free, mk_dax_kmem, mk_dax_kmem_all, mk_dax_kmem_preferred, mk_default, mk_hbw, mk_hbw_hugetlb, mk_hbw_interleave, mk_hbw_preferred, mk_hbw_preferred_hugetlb, mk_hugetlb, and mk_interleave.
Referenced by __kmp_fini_allocator().
void __kmp_fini_target_mem | ( | ) |
Finalize target memory support.
Definition at line 1639 of file kmp_alloc.cpp.
References __kmp_tgt_memspace_list, and kmp_tgt_memspace_list_t::fini().
Referenced by __kmp_fini_allocator().
omp_allocator_handle_t __kmp_get_devices_allocator | ( | int | ndevs, |
const int * | devs, | ||
omp_memspace_handle_t | memspace, | ||
int | host | ||
) |
Definition at line 1805 of file kmp_alloc.cpp.
References __kmp_entry_gtid, __kmp_get_devices_memspace(), __kmp_init_serial, __kmp_serial_initialize(), __kmpc_init_allocator(), kmp_max_mem_space, omp_null_allocator, and omp_null_mem_space.
Referenced by FTN_GET_DEVICE_ALLOCATOR(), FTN_GET_DEVICE_AND_HOST_ALLOCATOR(), FTN_GET_DEVICES_ALL_ALLOCATOR(), FTN_GET_DEVICES_ALLOCATOR(), and FTN_GET_DEVICES_AND_HOST_ALLOCATOR().
omp_memspace_handle_t __kmp_get_devices_memspace | ( | int | ndevs, |
const int * | devs, | ||
omp_memspace_handle_t | memspace, | ||
int | host | ||
) |
Definition at line 1792 of file kmp_alloc.cpp.
References __kmp_init_serial, __kmp_serial_initialize(), __kmp_tgt_memspace_list, kmp_tgt_memspace_list_t::get_memspace(), kmp_max_mem_space, and omp_null_mem_space.
Referenced by __kmp_get_devices_allocator(), FTN_GET_DEVICE_AND_HOST_MEMSPACE(), FTN_GET_DEVICE_MEMSPACE(), FTN_GET_DEVICES_ALL_MEMSPACE(), FTN_GET_DEVICES_AND_HOST_MEMSPACE(), and FTN_GET_DEVICES_MEMSPACE().
int __kmp_get_memspace_num_resources | ( | omp_memspace_handle_t | memspace | ) |
Definition at line 1821 of file kmp_alloc.cpp.
References __kmp_init_serial, __kmp_serial_initialize(), kmp_max_mem_space, kmp_memspace_t::num_resources, and omp_null_mem_space.
Referenced by FTN_GET_MEMSPACE_NUM_RESOURCES().
omp_memspace_handle_t __kmp_get_submemspace | ( | omp_memspace_handle_t | memspace, |
int | num_resources, | ||
int * | resources | ||
) |
Definition at line 1832 of file kmp_alloc.cpp.
References __kmp_allocate, __kmp_free, __kmp_init_serial, __kmp_serial_initialize(), __kmp_tgt_memspace_list, kmp_tgt_memspace_list_t::get_memspace(), i, kmp_max_mem_space, kmp_memspace_t::num_resources, omp_null_mem_space, and kmp_memspace_t::resources.
Referenced by FTN_GET_SUBMEMSPACE().
void __kmp_init_memkind | ( | ) |
Definition at line 1461 of file kmp_alloc.cpp.
References __kmp_memkind_available, h_memkind, int, KE_TRACE, kmp_mk_alloc, kmp_mk_check, kmp_mk_free, kmp_mk_lib_name, mk_dax_kmem, mk_dax_kmem_all, mk_dax_kmem_preferred, mk_default, mk_hbw, mk_hbw_hugetlb, mk_hbw_interleave, mk_hbw_preferred, mk_hbw_preferred_hugetlb, mk_hugetlb, mk_interleave, and void.
Referenced by __kmp_init_allocator().
void __kmp_init_target_mem | ( | ) |
Definition at line 1616 of file kmp_alloc.cpp.
References __kmp_target_mem_available, __kmp_tgt_allocator, __kmp_tgt_memspace_list, kmp_tgt_allocator_t::init(), kmp_tgt_memspace_list_t::init(), KMP_DLSYM, kmp_target_alloc_device, kmp_target_alloc_host, kmp_target_alloc_shared, kmp_target_free_device, kmp_target_free_host, kmp_target_free_shared, kmp_target_lock_mem, and kmp_target_unlock_mem.
Referenced by __kmp_init_allocator().
void * __kmp_realloc | ( | int | gtid, |
void * | ptr, | ||
size_t | size, | ||
omp_allocator_handle_t | allocator, | ||
omp_allocator_handle_t | free_allocator | ||
) |
Definition at line 2256 of file kmp_alloc.cpp.
References ___kmpc_free(), __kmp_alloc(), __kmp_init_serial, KMP_DEBUG_ASSERT, KMP_MEMCPY, kmp_mem_desc::ptr_align, size, kmp_mem_desc::size_a, and kmp_mem_desc::size_orig.
Referenced by __kmpc_realloc(), and omp_realloc().
void * __kmpc_aligned_alloc | ( | int | gtid, |
size_t | algn, | ||
size_t | size, | ||
omp_allocator_handle_t | allocator | ||
) |
Definition at line 1874 of file kmp_alloc.cpp.
References __kmp_alloc(), KE_TRACE, and size.
void * __kmpc_alloc | ( | int | gtid, |
size_t | size, | ||
omp_allocator_handle_t | allocator | ||
) |
Definition at line 1867 of file kmp_alloc.cpp.
References __kmp_alloc(), KE_TRACE, and size.
void * __kmpc_calloc | ( | int | gtid, |
size_t | nmemb, | ||
size_t | size, | ||
omp_allocator_handle_t | allocator | ||
) |
Definition at line 1883 of file kmp_alloc.cpp.
References __kmp_calloc(), KE_TRACE, and size.
void __kmpc_destroy_allocator | ( | int | gtid, |
omp_allocator_handle_t | allocator | ||
) |
Definition at line 1777 of file kmp_alloc.cpp.
References __kmp_free, and kmp_max_mem_alloc.
Referenced by __kmp_cleanup(), and FTN_DESTROY_ALLOCATOR().
void __kmpc_free | ( | int | gtid, |
void * | ptr, | ||
omp_allocator_handle_t | allocator | ||
) |
Definition at line 1902 of file kmp_alloc.cpp.
References ___kmpc_free(), and KE_TRACE.
omp_allocator_handle_t __kmpc_get_default_allocator | ( | int | gtid | ) |
Definition at line 1788 of file kmp_alloc.cpp.
References __kmp_threads.
Referenced by FTN_GET_DEFAULT_ALLOCATOR().
omp_allocator_handle_t __kmpc_init_allocator | ( | int | gtid, |
omp_memspace_handle_t | ms, | ||
int | ntraits, | ||
omp_alloctrait_t | traits[] | ||
) |
Definition at line 1641 of file kmp_alloc.cpp.
References __kmp_allocate, __kmp_free, __kmp_hwloc_available, __kmp_memkind_available, __kmp_target_mem_available, __kmp_type_convert(), kmp_allocator_t::alignment, kmp_allocator_t::atomic_scope, kmp_allocator_t::fb, kmp_allocator_t::fb_data, i, IS_POWER_OF_TWO, key, KMP_ASSERT, KMP_ASSERT2, KMP_DEBUG_ASSERT, KMP_IS_PREDEF_MEM_SPACE, KMP_IS_TARGET_MEM_SPACE, kmp_max_mem_space, kmp_allocator_t::memkind, kmp_allocator_t::memspace, mk_dax_kmem, mk_dax_kmem_all, mk_default, mk_hbw_interleave, mk_hbw_preferred, mk_interleave, omp_atk_access, omp_atk_alignment, omp_atk_atomic_scope, omp_atk_fallback, omp_atk_fb_data, omp_atk_part_size, omp_atk_partition, omp_atk_pin_device, omp_atk_pinned, omp_atk_pool_size, omp_atk_preferred_device, omp_atk_sync_hint, omp_atk_target_access, omp_atv_abort_fb, omp_atv_allocator_fb, omp_atv_blocked, omp_atv_default_mem_fb, omp_atv_device, omp_atv_environment, omp_atv_interleaved, omp_atv_nearest, omp_atv_null_fb, omp_atv_single, omp_default_mem_alloc, omp_high_bw_mem_space, omp_large_cap_mem_space, omp_null_allocator, kmp_allocator_t::part_size, kmp_allocator_t::partition, kmp_allocator_t::pin_device, kmp_allocator_t::pinned, kmp_allocator_t::pool_size, kmp_allocator_t::preferred_device, RCAST, kmp_allocator_t::target_access, omp_alloctrait_t::value, and value.
Referenced by __kmp_get_devices_allocator(), __kmp_stg_parse_allocator(), and FTN_INIT_ALLOCATOR().
void * __kmpc_realloc | ( | int | gtid, |
void * | ptr, | ||
size_t | size, | ||
omp_allocator_handle_t | allocator, | ||
omp_allocator_handle_t | free_allocator | ||
) |
Definition at line 1892 of file kmp_alloc.cpp.
References __kmp_realloc(), KE_TRACE, and size.
void __kmpc_set_default_allocator | ( | int | gtid, |
omp_allocator_handle_t | allocator | ||
) |
Definition at line 1782 of file kmp_alloc.cpp.
References __kmp_threads, omp_default_mem_alloc, and omp_null_allocator.
Referenced by FTN_SET_DEFAULT_ALLOCATOR().
void * kmpc_aligned_malloc | ( | size_t | size, |
size_t | alignment | ||
) |
Definition at line 1123 of file kmp_alloc.cpp.
References __kmp_entry_thread(), alignment, IS_POWER_OF_TWO, KMP_DEBUG_ASSERT, and size.
Referenced by FTN_ALIGNED_MALLOC().
void * kmpc_calloc | ( | size_t | nelem, |
size_t | elsize | ||
) |
Definition at line 1145 of file kmp_alloc.cpp.
References __kmp_entry_thread().
Referenced by FTN_CALLOC().
Definition at line 1186 of file kmp_alloc.cpp.
References __kmp_get_thread, __kmp_init_serial, and KMP_ASSERT.
Referenced by FTN_KFREE().
void * kmpc_malloc | ( | size_t | size | ) |
Definition at line 1110 of file kmp_alloc.cpp.
References __kmp_entry_thread(), and size.
Referenced by FTN_MALLOC().
Definition at line 1156 of file kmp_alloc.cpp.
References __kmp_entry_thread(), __kmp_get_thread, KMP_ASSERT, result, and size.
Referenced by FTN_REALLOC().
|
static |
Definition at line 1267 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), __kmp_alloc(), __kmp_init_target_mem(), and __kmpc_init_allocator().
class kmp_tgt_allocator_t __kmp_tgt_allocator |
Referenced by ___kmpc_free(), __kmp_alloc(), __kmp_init_target_mem(), and kmp_tgt_memspace_list_t::get_memspace().
class kmp_tgt_memspace_list_t __kmp_tgt_memspace_list |
Referenced by __kmp_fini_target_mem(), __kmp_get_devices_memspace(), __kmp_get_submemspace(), and __kmp_init_target_mem().
|
constexpr |
Definition at line 1864 of file kmp_alloc.cpp.
Referenced by ___kmp_allocate_align(), __kmp_alloc(), aligned_by(), FTN_ALIGNED_MALLOC(), KMP_API_NAME_GOMP_ALLOC(), kmpc_aligned_malloc(), and test_kmp_aligned_malloc().
|
static |
Definition at line 1239 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), and __kmp_init_memkind().
Definition at line 1242 of file kmp_alloc.cpp.
Referenced by __kmp_alloc(), __kmp_fini_memkind(), and __kmp_init_memkind().
Definition at line 1246 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), and __kmp_init_memkind().
Definition at line 1244 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), __kmp_fini_memkind(), and __kmp_init_memkind().
|
static |
Definition at line 1238 of file kmp_alloc.cpp.
Referenced by __kmp_init_memkind().
Definition at line 1261 of file kmp_alloc.cpp.
Referenced by __kmp_alloc(), and __kmp_init_target_mem().
Definition at line 1259 of file kmp_alloc.cpp.
Referenced by __kmp_alloc(), and __kmp_init_target_mem().
Definition at line 1260 of file kmp_alloc.cpp.
Referenced by __kmp_alloc(), and __kmp_init_target_mem().
Definition at line 1266 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), and __kmp_init_target_mem().
Definition at line 1264 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), and __kmp_init_target_mem().
Definition at line 1265 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), and __kmp_init_target_mem().
|
static |
Definition at line 1262 of file kmp_alloc.cpp.
Referenced by __kmp_alloc(), and __kmp_init_target_mem().
Definition at line 1263 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), and __kmp_init_target_mem().
|
static |
Definition at line 1256 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), __kmp_init_memkind(), and __kmpc_init_allocator().
|
static |
Definition at line 1257 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), __kmp_alloc(), __kmp_fini_memkind(), __kmp_init_memkind(), and __kmpc_init_allocator().
|
static |
Definition at line 1258 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), and __kmp_init_memkind().
|
static |
Definition at line 1248 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), __kmp_alloc(), __kmp_fini_memkind(), __kmp_init_memkind(), and __kmpc_init_allocator().
|
static |
Definition at line 1250 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), and __kmp_init_memkind().
|
static |
Definition at line 1254 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), and __kmp_init_memkind().
|
static |
Definition at line 1251 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), __kmp_init_memkind(), and __kmpc_init_allocator().
|
static |
Definition at line 1252 of file kmp_alloc.cpp.
Referenced by ___kmpc_free(), __kmp_alloc(), __kmp_fini_memkind(), __kmp_init_memkind(), and __kmpc_init_allocator().
|
static |
Definition at line 1255 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), and __kmp_init_memkind().
|
static |
Definition at line 1253 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), and __kmp_init_memkind().
|
static |
Definition at line 1249 of file kmp_alloc.cpp.
Referenced by __kmp_fini_memkind(), __kmp_init_memkind(), and __kmpc_init_allocator().