24#define MAX_MESSAGE 512
40 if ((env = getenv(
"KMP_INITIAL_THREAD_BIND")) != NULL &&
44 KC_TRACE(10, (
"__kmpc_begin: middle initialization called\n"));
48 KC_TRACE(10, (
"__kmpc_begin: called\n"));
67 KC_TRACE(10, (
"__kmpc_end: called\n"));
72#if KMP_OS_WINDOWS && OMPT_SUPPORT
103 KC_TRACE(10, (
"__kmpc_global_thread_num: T#%d\n", gtid));
124 (
"__kmpc_global_num_threads: num_threads = %d\n",
__kmp_all_nth));
136 KC_TRACE(10, (
"__kmpc_bound_thread_num: called\n"));
146 KC_TRACE(10, (
"__kmpc_bound_num_threads: called\n"));
168 if (__kmp_par_range == 0) {
175 semi2 = strchr(semi2,
';');
179 semi2 = strchr(semi2 + 1,
';');
183 if (__kmp_par_range_filename[0]) {
184 const char *
name = semi2 - 1;
188 if ((*
name ==
'/') || (*
name ==
';')) {
191 if (strncmp(__kmp_par_range_filename,
name, semi2 -
name)) {
192 return __kmp_par_range < 0;
195 semi3 = strchr(semi2 + 1,
';');
196 if (__kmp_par_range_routine[0]) {
197 if ((semi3 != NULL) && (semi3 > semi2) &&
198 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
199 return __kmp_par_range < 0;
202 if (
KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
203 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
204 return __kmp_par_range > 0;
206 return __kmp_par_range < 0;
234 KA_TRACE(20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
235 global_tid, num_threads));
241 KA_TRACE(20, (
"__kmpc_pop_num_threads: enter\n"));
247 KA_TRACE(20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
266#if (KMP_STATS_ENABLED)
270 if (previous_state == stats_state_e::SERIAL_REGION) {
271 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
286 va_start(ap, microtask);
289 ompt_frame_t *ompt_frame;
292 ompt_frame = &master_th->th.th_current_task->ompt_task_info.frame;
295 OMPT_STORE_RETURN_ADDRESS(gtid);
319 ompt_frame->enter_frame = ompt_data_none;
325 if (previous_state == stats_state_e::SERIAL_REGION) {
326 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
356 void *exit_frame_ptr;
397 (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
398 global_tid, num_teams, num_threads));
417 if (thread_limit > 0)
418 thread->th.th_current_task->td_icvs.task_thread_limit = thread_limit;
440 KA_TRACE(20, (
"__kmpc_push_num_teams_51: enter T#%d num_teams_lb=%d"
441 " num_teams_ub=%d num_threads=%d\n",
442 global_tid, num_teams_lb, num_teams_ub, num_threads));
463 va_start(ap, microtask);
468 if (previous_state == stats_state_e::SERIAL_REGION) {
469 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_teams_overhead);
476 this_thr->th.th_teams_microtask = microtask;
477 this_thr->th.th_teams_level =
478 this_thr->th.th_team->t.t_level;
481 kmp_team_t *parent_team = this_thr->th.th_team;
484 parent_team->
t.t_implicit_task_taskdata[tid]
487 OMPT_STORE_RETURN_ADDRESS(gtid);
492 if (this_thr->th.th_teams_size.nteams == 0) {
513 this_thr->th.th_cg_roots = tmp->
up;
514 KA_TRACE(100, (
"__kmpc_fork_teams: Thread %p popping node %p and moving up"
515 " to node %p. cg_nthreads was %d\n",
516 this_thr, tmp, this_thr->th.th_cg_roots, tmp->
cg_nthreads));
524 this_thr->th.th_current_task->td_icvs.thread_limit =
525 this_thr->th.th_cg_roots->cg_thread_limit;
527 this_thr->th.th_teams_microtask = NULL;
528 this_thr->th.th_teams_level = 0;
529 *(
kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
532 if (previous_state == stats_state_e::SERIAL_REGION) {
533 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
565 OMPT_STORE_RETURN_ADDRESS(global_tid);
583 (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
598 serial_team = this_thr->th.th_serial_team;
616 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
617 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = ompt_data_none;
620 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
621 OMPT_CUR_TASK_INFO(this_thr)->thread_num, ompt_task_implicit);
625 ompt_data_t *parent_task_data;
630 &(serial_team->
t.ompt_team_info.parallel_data), parent_task_data,
631 ompt_parallel_invoker_program | ompt_parallel_team,
632 OMPT_LOAD_RETURN_ADDRESS(global_tid));
635 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
641 top = serial_team->
t.t_control_stack_top;
643 copy_icvs(&serial_team->
t.t_threads[0]->th.th_current_task->td_icvs, top);
644 serial_team->
t.t_control_stack_top = top->
next;
652 serial_team->
t.t_dispatch->th_disp_buffer;
653 serial_team->
t.t_dispatch->th_disp_buffer =
654 serial_team->
t.t_dispatch->th_disp_buffer->next;
657 this_thr->th.th_def_allocator = serial_team->
t.t_def_allocator;
659 --serial_team->
t.t_serialized;
660 if (serial_team->
t.t_serialized == 0) {
664#if KMP_ARCH_X86 || KMP_ARCH_X86_64
665 if (__kmp_inherit_fp_control && serial_team->
t.t_fp_control_saved) {
666 __kmp_clear_x87_fpu_status_word();
667 __kmp_load_x87_fpu_control_word(&serial_team->
t.t_x87_fpu_control_word);
668 __kmp_load_mxcsr(&serial_team->
t.t_mxcsr);
674 if (ompd_state & OMPD_ENABLE_BP)
675 ompd_bp_parallel_end();
678 this_thr->th.th_team = serial_team->
t.t_parent;
679 this_thr->th.th_info.ds.ds_tid = serial_team->
t.t_master_tid;
682 this_thr->th.th_team_nproc = serial_team->
t.t_parent->t.t_nproc;
683 this_thr->th.th_team_master =
684 serial_team->
t.t_parent->t.t_threads[0];
685 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
688 this_thr->th.th_dispatch =
689 &this_thr->th.th_team->t.t_dispatch[serial_team->
t.t_master_tid];
691 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
692 this_thr->th.th_current_task->td_flags.executing = 1;
696 this_thr->th.th_task_team =
697 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
699 (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / "
701 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
703#if KMP_AFFINITY_SUPPORTED
704 if (this_thr->th.th_team->t.t_level == 0 && __kmp_affinity.flags.reset) {
710 KA_TRACE(20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting "
711 "depth of serial team %p to %d\n",
712 global_tid, serial_team, serial_team->
t.t_serialized));
716 serial_team->
t.t_level--;
721 this_thr->th.ompt_thread_info.state =
722 ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
723 : ompt_state_work_parallel);
736 KC_TRACE(10, (
"__kmpc_flush: called\n"));
741#if OMPT_SUPPORT && OMPT_OPTIONAL
759 KC_TRACE(10, (
"__kmpc_barrier: called T#%d\n", global_tid));
775 ompt_frame_t *ompt_frame;
778 if (ompt_frame->enter_frame.ptr == NULL)
781 OMPT_STORE_RETURN_ADDRESS(global_tid);
792#if OMPT_SUPPORT && OMPT_OPTIONAL
794 ompt_frame->enter_frame = ompt_data_none;
809 KC_TRACE(10, (
"__kmpc_master: called T#%d\n", global_tid));
823#if OMPT_SUPPORT && OMPT_OPTIONAL
831 ompt_scope_begin, &(team->
t.ompt_team_info.parallel_data),
832 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
839#if KMP_USE_DYNAMIC_LOCK
864 KC_TRACE(10, (
"__kmpc_end_master: called T#%d\n", global_tid));
869#if OMPT_SUPPORT && OMPT_OPTIONAL
875 ompt_scope_end, &(team->
t.ompt_team_info.parallel_data),
876 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
898 KC_TRACE(10, (
"__kmpc_masked: called T#%d\n", global_tid));
913#if OMPT_SUPPORT && OMPT_OPTIONAL
919 ompt_scope_begin, &(team->
t.ompt_team_info.parallel_data),
920 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
927#if KMP_USE_DYNAMIC_LOCK
952 KC_TRACE(10, (
"__kmpc_end_masked: called T#%d\n", global_tid));
956#if OMPT_SUPPORT && OMPT_OPTIONAL
962 ompt_scope_end, &(team->
t.ompt_team_info.parallel_data),
963 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
985 KC_TRACE(10, (
"__kmpc_ordered: called T#%d\n", gtid));
994 __kmp_itt_ordered_prep(gtid);
1000#if OMPT_SUPPORT && OMPT_OPTIONAL
1004 OMPT_STORE_RETURN_ADDRESS(gtid);
1007 lck = (ompt_wait_id_t)(uintptr_t)&team->
t.t_ordered.dt.t_value;
1009 th->th.ompt_thread_info.wait_id =
lck;
1010 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
1013 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1016 ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin,
lck,
1022 if (th->th.th_dispatch->th_deo_fcn != 0)
1023 (*th->th.th_dispatch->th_deo_fcn)(>id, &cid,
loc);
1027#if OMPT_SUPPORT && OMPT_OPTIONAL
1030 th->th.ompt_thread_info.state = ompt_state_work_parallel;
1031 th->th.ompt_thread_info.wait_id = 0;
1036 ompt_mutex_ordered, (ompt_wait_id_t)(uintptr_t)
lck, codeptr_ra);
1042 __kmp_itt_ordered_start(gtid);
1057 KC_TRACE(10, (
"__kmpc_end_ordered: called T#%d\n", gtid));
1061 __kmp_itt_ordered_end(gtid);
1067 if (th->th.th_dispatch->th_dxo_fcn != 0)
1068 (*th->th.th_dispatch->th_dxo_fcn)(>id, &cid,
loc);
1072#if OMPT_SUPPORT && OMPT_OPTIONAL
1073 OMPT_STORE_RETURN_ADDRESS(gtid);
1078 ->
t.t_ordered.dt.t_value,
1079 OMPT_LOAD_RETURN_ADDRESS(gtid));
1084#if KMP_USE_DYNAMIC_LOCK
1086static __forceinline
void
1088 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
1092 kmp_indirect_lock_t **
lck;
1093 lck = (kmp_indirect_lock_t **)
crit;
1094 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
1095 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
1096 KMP_SET_I_LOCK_LOCATION(ilk,
loc);
1099 (
"__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
1101 __kmp_itt_critical_creating(ilk->lock,
loc);
1106 __kmp_itt_critical_destroyed(ilk->lock);
1116#define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \
1118 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1119 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1120 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1121 if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1122 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \
1124 KMP_FSYNC_PREPARE(l); \
1125 KMP_INIT_YIELD(spins); \
1126 kmp_backoff_t backoff = __kmp_spin_backoff_params; \
1128 if (TCR_4(__kmp_nth) > \
1129 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
1132 KMP_YIELD_SPIN(spins); \
1134 __kmp_spin_backoff(&backoff); \
1136 KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1137 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)); \
1139 KMP_FSYNC_ACQUIRED(l); \
1143#define KMP_TEST_TAS_LOCK(lock, gtid, rc) \
1145 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1146 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1147 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1148 rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \
1149 __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \
1153#define KMP_RELEASE_TAS_LOCK(lock, gtid) \
1154 { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); }
1158#include <sys/syscall.h>
1168#define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \
1170 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1171 kmp_int32 gtid_code = (gtid + 1) << 1; \
1173 KMP_FSYNC_PREPARE(ftx); \
1174 kmp_int32 poll_val; \
1175 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \
1176 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1177 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \
1178 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \
1180 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \
1182 KMP_LOCK_BUSY(1, futex))) { \
1185 poll_val |= KMP_LOCK_BUSY(1, futex); \
1188 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \
1189 NULL, NULL, 0)) != 0) { \
1194 KMP_FSYNC_ACQUIRED(ftx); \
1198#define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \
1200 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1201 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1202 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \
1203 KMP_FSYNC_ACQUIRED(ftx); \
1211#define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \
1213 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1215 KMP_FSYNC_RELEASING(ftx); \
1216 kmp_int32 poll_val = \
1217 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \
1218 if (KMP_LOCK_STRIP(poll_val) & 1) { \
1219 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \
1220 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \
1223 KMP_YIELD_OVERSUB(); \
1247 __kmp_itt_critical_creating(
lck);
1263 __kmp_itt_critical_destroyed(
lck);
1290#if KMP_USE_DYNAMIC_LOCK
1291#if OMPT_SUPPORT && OMPT_OPTIONAL
1292 OMPT_STORE_RETURN_ADDRESS(global_tid);
1297#if OMPT_SUPPORT && OMPT_OPTIONAL
1298 ompt_state_t prev_state = ompt_state_undefined;
1303 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1334 __kmp_itt_critical_acquiring(
lck);
1336#if OMPT_SUPPORT && OMPT_OPTIONAL
1337 OMPT_STORE_RETURN_ADDRESS(gtid);
1338 void *codeptr_ra = NULL;
1342 prev_state = ti.
state;
1344 ti.
state = ompt_state_wait_critical;
1347 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1350 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1351 (ompt_wait_id_t)(uintptr_t)
lck, codeptr_ra);
1360 __kmp_itt_critical_acquired(
lck);
1362#if OMPT_SUPPORT && OMPT_OPTIONAL
1365 ti.
state = prev_state;
1371 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)
lck, codeptr_ra);
1378 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1382#if KMP_USE_DYNAMIC_LOCK
1385static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1387#define KMP_TSX_LOCK(seq) lockseq_##seq
1389#define KMP_TSX_LOCK(seq) __kmp_user_lock_seq
1392#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1393#define KMP_CPUINFO_RTM (__kmp_cpuinfo.flags.rtm)
1395#define KMP_CPUINFO_RTM 0
1399 if (hint & kmp_lock_hint_hle)
1400 return KMP_TSX_LOCK(hle);
1401 if (hint & kmp_lock_hint_rtm)
1402 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_queuing) : __kmp_user_lock_seq;
1403 if (hint & kmp_lock_hint_adaptive)
1404 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1407 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1408 return __kmp_user_lock_seq;
1409 if ((hint & omp_lock_hint_speculative) &&
1410 (hint & omp_lock_hint_nonspeculative))
1411 return __kmp_user_lock_seq;
1414 if (hint & omp_lock_hint_contended)
1415 return lockseq_queuing;
1418 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1422 if (hint & omp_lock_hint_speculative)
1423 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_spin) : __kmp_user_lock_seq;
1425 return __kmp_user_lock_seq;
1428#if OMPT_SUPPORT && OMPT_OPTIONAL
1429#if KMP_USE_DYNAMIC_LOCK
1430static kmp_mutex_impl_t
1431__ompt_get_mutex_impl_type(
void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1433 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1438 return kmp_mutex_impl_queuing;
1441 return kmp_mutex_impl_spin;
1444 case locktag_rtm_spin:
1445 return kmp_mutex_impl_speculative;
1448 return kmp_mutex_impl_none;
1450 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1453 switch (ilock->type) {
1455 case locktag_adaptive:
1456 case locktag_rtm_queuing:
1457 return kmp_mutex_impl_speculative;
1459 case locktag_nested_tas:
1460 return kmp_mutex_impl_spin;
1462 case locktag_nested_futex:
1464 case locktag_ticket:
1465 case locktag_queuing:
1467 case locktag_nested_ticket:
1468 case locktag_nested_queuing:
1469 case locktag_nested_drdpa:
1470 return kmp_mutex_impl_queuing;
1472 return kmp_mutex_impl_none;
1477static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
1480 return kmp_mutex_impl_spin;
1487 return kmp_mutex_impl_queuing;
1490 case lk_rtm_queuing:
1493 return kmp_mutex_impl_speculative;
1496 return kmp_mutex_impl_none;
1519#if OMPT_SUPPORT && OMPT_OPTIONAL
1520 ompt_state_t prev_state = ompt_state_undefined;
1523 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1528 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1531 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)
crit;
1534 kmp_dyna_lockseq_t lockseq = __kmp_map_hint_to_lock(hint);
1536 if (KMP_IS_D_LOCK(lockseq)) {
1539 KMP_GET_D_TAG(lockseq));
1541 __kmp_init_indirect_csptr(
crit,
loc, global_tid, KMP_GET_I_TAG(lockseq));
1547 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1551 __kmp_map_hint_to_lock(hint));
1554 __kmp_itt_critical_acquiring(
lck);
1556#if OMPT_SUPPORT && OMPT_OPTIONAL
1560 prev_state = ti.
state;
1562 ti.
state = ompt_state_wait_critical;
1567 ompt_mutex_critical, (
unsigned int)hint,
1568 __ompt_get_mutex_impl_type(
crit), (ompt_wait_id_t)(uintptr_t)
lck,
1573#if KMP_USE_INLINED_TAS
1575 KMP_ACQUIRE_TAS_LOCK(
lck, global_tid);
1577#elif KMP_USE_INLINED_FUTEX
1579 KMP_ACQUIRE_FUTEX_LOCK(
lck, global_tid);
1583 KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
1586 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1590 __kmp_map_hint_to_lock(hint));
1593 __kmp_itt_critical_acquiring(
lck);
1595#if OMPT_SUPPORT && OMPT_OPTIONAL
1599 prev_state = ti.
state;
1601 ti.
state = ompt_state_wait_critical;
1606 ompt_mutex_critical, (
unsigned int)hint,
1607 __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)(uintptr_t)
lck,
1612 KMP_I_LOCK_FUNC(ilk, set)(
lck, global_tid);
1617 __kmp_itt_critical_acquired(
lck);
1619#if OMPT_SUPPORT && OMPT_OPTIONAL
1622 ti.
state = prev_state;
1628 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
1634 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1652 KC_TRACE(10, (
"__kmpc_end_critical: called T#%d\n", global_tid));
1654#if KMP_USE_DYNAMIC_LOCK
1655 int locktag = KMP_EXTRACT_D_TAG(
crit);
1663 __kmp_itt_critical_releasing(
lck);
1665#if KMP_USE_INLINED_TAS
1667 KMP_RELEASE_TAS_LOCK(
lck, global_tid);
1669#elif KMP_USE_INLINED_FUTEX
1671 KMP_RELEASE_FUTEX_LOCK(
lck, global_tid);
1675 KMP_D_LOCK_FUNC(
lck, unset)((kmp_dyna_lock_t *)
lck, global_tid);
1678 kmp_indirect_lock_t *ilk =
1679 (kmp_indirect_lock_t *)
TCR_PTR(*((kmp_indirect_lock_t **)
crit));
1686 __kmp_itt_critical_releasing(
lck);
1688 KMP_I_LOCK_FUNC(ilk, unset)(
lck, global_tid);
1713 __kmp_itt_critical_releasing(
lck);
1721#if OMPT_SUPPORT && OMPT_OPTIONAL
1724 OMPT_STORE_RETURN_ADDRESS(global_tid);
1727 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)
lck,
1728 OMPT_LOAD_RETURN_ADDRESS(0));
1733 KA_TRACE(15, (
"__kmpc_end_critical: done T#%d\n", global_tid));
1747 KC_TRACE(10, (
"__kmpc_barrier_master: called T#%d\n", global_tid));
1759 ompt_frame_t *ompt_frame;
1762 if (ompt_frame->enter_frame.ptr == NULL)
1765 OMPT_STORE_RETURN_ADDRESS(global_tid);
1771#if OMPT_SUPPORT && OMPT_OPTIONAL
1773 ompt_frame->enter_frame = ompt_data_none;
1777 return (
status != 0) ? 0 : 1;
1790 KC_TRACE(10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid));
1807 KC_TRACE(10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1823 ompt_frame_t *ompt_frame;
1826 if (ompt_frame->enter_frame.ptr == NULL)
1829 OMPT_STORE_RETURN_ADDRESS(global_tid);
1835#if OMPT_SUPPORT && OMPT_OPTIONAL
1837 ompt_frame->enter_frame = ompt_data_none;
1878#if OMPT_SUPPORT && OMPT_OPTIONAL
1887 ompt_work_single_executor, ompt_scope_begin,
1888 &(team->
t.ompt_team_info.parallel_data),
1889 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1895 ompt_work_single_other, ompt_scope_begin,
1896 &(team->
t.ompt_team_info.parallel_data),
1897 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1900 ompt_work_single_other, ompt_scope_end,
1901 &(team->
t.ompt_team_info.parallel_data),
1902 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1926#if OMPT_SUPPORT && OMPT_OPTIONAL
1933 ompt_work_single_executor, ompt_scope_end,
1934 &(team->
t.ompt_team_info.parallel_data),
1935 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
1950 KE_TRACE(10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1952#if OMPT_SUPPORT && OMPT_OPTIONAL
1954 ompt_work_t ompt_work_type = ompt_work_loop;
1960 ompt_work_type = ompt_work_loop;
1962 ompt_work_type = ompt_work_sections;
1964 ompt_work_type = ompt_work_distribute;
1972 ompt_work_type, ompt_scope_end, &(team_info->
parallel_data),
2046 if (buffer &&
size) {
2060#if KMP_AFFINITY_SUPPORTED
2062 __kmp_affinity.flags.reset) {
2070 char const *format) {
2072 size_t num_required;
2079#if KMP_AFFINITY_SUPPORTED
2081 __kmp_affinity.flags.reset) {
2087 if (buffer && buf_size) {
2089 capture_buf.
used + 1);
2092 return num_required;
2106 int gtid, tid, bt = arg;
2137#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2144 return __kmp_aux_set_affinity_mask_proc(proc,
mask);
2149#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2156 return __kmp_aux_unset_affinity_mask_proc(proc,
mask);
2161#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2168 return __kmp_aux_get_affinity_mask_proc(proc,
mask);
2218 void *cpy_data,
void (*cpy_func)(
void *,
void *),
2221 KC_TRACE(10, (
"__kmpc_copyprivate: called T#%d\n", gtid));
2237 *data_ptr = cpy_data;
2240 ompt_frame_t *ompt_frame;
2243 if (ompt_frame->enter_frame.ptr == NULL)
2246 OMPT_STORE_RETURN_ADDRESS(gtid);
2255 (*cpy_func)(cpy_data, *data_ptr);
2261 OMPT_STORE_RETURN_ADDRESS(gtid);
2268#if OMPT_SUPPORT && OMPT_OPTIONAL
2270 ompt_frame->enter_frame = ompt_data_none;
2296 KC_TRACE(10, (
"__kmpc_copyprivate_light: called T#%d\n", gtid));
2311 *data_ptr = cpy_data;
2314 ompt_frame_t *ompt_frame;
2317 if (ompt_frame->enter_frame.ptr == NULL)
2319 OMPT_STORE_RETURN_ADDRESS(gtid);
2333#define INIT_LOCK __kmp_init_user_lock_with_checks
2334#define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
2335#define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
2336#define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
2337#define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
2338#define ACQUIRE_NESTED_LOCK_TIMED \
2339 __kmp_acquire_nested_user_lock_with_checks_timed
2340#define RELEASE_LOCK __kmp_release_user_lock_with_checks
2341#define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
2342#define TEST_LOCK __kmp_test_user_lock_with_checks
2343#define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
2344#define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
2345#define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
2350#if KMP_USE_DYNAMIC_LOCK
2353static __forceinline
void __kmp_init_lock_with_hint(
ident_t *
loc,
void **
lock,
2354 kmp_dyna_lockseq_t seq) {
2355 if (KMP_IS_D_LOCK(seq)) {
2356 KMP_INIT_D_LOCK(
lock, seq);
2361 KMP_INIT_I_LOCK(
lock, seq);
2363 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(
lock);
2364 __kmp_itt_lock_creating(ilk->lock,
loc);
2370static __forceinline
void
2372 kmp_dyna_lockseq_t seq) {
2375 if (seq == lockseq_hle || seq == lockseq_rtm_queuing ||
2376 seq == lockseq_rtm_spin || seq == lockseq_adaptive)
2377 seq = __kmp_user_lock_seq;
2381 seq = lockseq_nested_tas;
2385 seq = lockseq_nested_futex;
2388 case lockseq_ticket:
2389 seq = lockseq_nested_ticket;
2391 case lockseq_queuing:
2392 seq = lockseq_nested_queuing;
2395 seq = lockseq_nested_drdpa;
2398 seq = lockseq_nested_queuing;
2400 KMP_INIT_I_LOCK(
lock, seq);
2402 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(
lock);
2403 __kmp_itt_lock_creating(ilk->lock,
loc);
2412 KMP_FATAL(LockIsUninitialized,
"omp_init_lock_with_hint");
2415 __kmp_init_lock_with_hint(
loc, user_lock, __kmp_map_hint_to_lock(hint));
2417#if OMPT_SUPPORT && OMPT_OPTIONAL
2419 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2424 ompt_mutex_lock, (omp_lock_hint_t)hint,
2425 __ompt_get_mutex_impl_type(user_lock),
2426 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2433 void **user_lock, uintptr_t hint) {
2436 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock_with_hint");
2439 __kmp_init_nest_lock_with_hint(
loc, user_lock, __kmp_map_hint_to_lock(hint));
2441#if OMPT_SUPPORT && OMPT_OPTIONAL
2443 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2448 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2449 __ompt_get_mutex_impl_type(user_lock),
2450 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2459#if KMP_USE_DYNAMIC_LOCK
2463 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
2465 __kmp_init_lock_with_hint(
loc, user_lock, __kmp_user_lock_seq);
2467#if OMPT_SUPPORT && OMPT_OPTIONAL
2469 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2474 ompt_mutex_lock, omp_lock_hint_none,
2475 __ompt_get_mutex_impl_type(user_lock),
2476 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2482 static char const *
const func =
"omp_init_lock";
2487 if (user_lock == NULL) {
2510#if OMPT_SUPPORT && OMPT_OPTIONAL
2512 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2517 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2518 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2523 __kmp_itt_lock_creating(
lck);
2531#if KMP_USE_DYNAMIC_LOCK
2535 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
2537 __kmp_init_nest_lock_with_hint(
loc, user_lock, __kmp_user_lock_seq);
2539#if OMPT_SUPPORT && OMPT_OPTIONAL
2541 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2546 ompt_mutex_nest_lock, omp_lock_hint_none,
2547 __ompt_get_mutex_impl_type(user_lock),
2548 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2554 static char const *
const func =
"omp_init_nest_lock";
2559 if (user_lock == NULL) {
2567 (
sizeof(
lck->tas.lk.poll) +
sizeof(
lck->tas.lk.depth_locked) <=
2573 (
sizeof(
lck->futex.lk.poll) +
sizeof(
lck->futex.lk.depth_locked) <=
2585#if OMPT_SUPPORT && OMPT_OPTIONAL
2587 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2592 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2593 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2598 __kmp_itt_lock_creating(
lck);
2605#if KMP_USE_DYNAMIC_LOCK
2609 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2610 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->
lock;
2614 __kmp_itt_lock_destroyed(
lck);
2616#if OMPT_SUPPORT && OMPT_OPTIONAL
2618 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2623 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2626 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2644#if OMPT_SUPPORT && OMPT_OPTIONAL
2646 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2651 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2656 __kmp_itt_lock_destroyed(
lck);
2678#if KMP_USE_DYNAMIC_LOCK
2681 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2682 __kmp_itt_lock_destroyed(ilk->lock);
2684#if OMPT_SUPPORT && OMPT_OPTIONAL
2686 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2691 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2694 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2701 (
sizeof(
lck->tas.lk.poll) +
sizeof(
lck->tas.lk.depth_locked) <=
2707 (
sizeof(
lck->futex.lk.poll) +
sizeof(
lck->futex.lk.depth_locked) <=
2716#if OMPT_SUPPORT && OMPT_OPTIONAL
2718 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2723 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2728 __kmp_itt_lock_destroyed(
lck);
2734 (
sizeof(
lck->tas.lk.poll) +
sizeof(
lck->tas.lk.depth_locked) <=
2740 (
sizeof(
lck->futex.lk.poll) +
sizeof(
lck->futex.lk.depth_locked) <=
2753#if KMP_USE_DYNAMIC_LOCK
2754 int tag = KMP_EXTRACT_D_TAG(user_lock);
2756 __kmp_itt_lock_acquiring(
2760#if OMPT_SUPPORT && OMPT_OPTIONAL
2762 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2767 ompt_mutex_lock, omp_lock_hint_none,
2768 __ompt_get_mutex_impl_type(user_lock),
2769 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2772#if KMP_USE_INLINED_TAS
2774 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2776#elif KMP_USE_INLINED_FUTEX
2778 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2782 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2787#if OMPT_SUPPORT && OMPT_OPTIONAL
2790 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2813 __kmp_itt_lock_acquiring(
lck);
2815#if OMPT_SUPPORT && OMPT_OPTIONAL
2817 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2822 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2823 (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
2830 __kmp_itt_lock_acquired(
lck);
2833#if OMPT_SUPPORT && OMPT_OPTIONAL
2836 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
2844#if KMP_USE_DYNAMIC_LOCK
2849#if OMPT_SUPPORT && OMPT_OPTIONAL
2851 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2857 ompt_mutex_nest_lock, omp_lock_hint_none,
2858 __ompt_get_mutex_impl_type(user_lock),
2859 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2863 int acquire_status =
2864 KMP_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid);
2865 (
void)acquire_status;
2870#if OMPT_SUPPORT && OMPT_OPTIONAL
2876 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
2883 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2894 (
sizeof(
lck->tas.lk.poll) +
sizeof(
lck->tas.lk.depth_locked) <=
2900 (
sizeof(
lck->futex.lk.poll) +
sizeof(
lck->futex.lk.depth_locked) <=
2910 __kmp_itt_lock_acquiring(
lck);
2912#if OMPT_SUPPORT && OMPT_OPTIONAL
2914 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2920 ompt_mutex_nest_lock, omp_lock_hint_none,
2921 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)
lck,
2930 __kmp_itt_lock_acquired(
lck);
2933#if OMPT_SUPPORT && OMPT_OPTIONAL
2939 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
2945 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
2955#if KMP_USE_DYNAMIC_LOCK
2957 int tag = KMP_EXTRACT_D_TAG(user_lock);
2961#if KMP_USE_INLINED_TAS
2963 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
2965#elif KMP_USE_INLINED_FUTEX
2967 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
2971 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2974#if OMPT_SUPPORT && OMPT_OPTIONAL
2976 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2981 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2994#if KMP_OS_LINUX && \
2995 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
3003#if OMPT_SUPPORT && OMPT_OPTIONAL
3005 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3010 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3030 __kmp_itt_lock_releasing(
lck);
3035#if OMPT_SUPPORT && OMPT_OPTIONAL
3037 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3042 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3051#if KMP_USE_DYNAMIC_LOCK
3056 int release_status =
3057 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
3058 (
void)release_status;
3060#if OMPT_SUPPORT && OMPT_OPTIONAL
3062 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3070 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
3076 ompt_scope_end, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3088 (
sizeof(
lck->tas.lk.poll) +
sizeof(
lck->tas.lk.depth_locked) <=
3090#if KMP_OS_LINUX && \
3091 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
3098#if OMPT_SUPPORT && OMPT_OPTIONAL
3104#if OMPT_SUPPORT && OMPT_OPTIONAL
3110#if OMPT_SUPPORT && OMPT_OPTIONAL
3112 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3120 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3125 ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3137 (
sizeof(
lck->futex.lk.poll) +
sizeof(
lck->futex.lk.depth_locked) <=
3147 __kmp_itt_lock_releasing(
lck);
3152#if OMPT_SUPPORT && OMPT_OPTIONAL
3154 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3162 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3167 ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3179#if KMP_USE_DYNAMIC_LOCK
3181 int tag = KMP_EXTRACT_D_TAG(user_lock);
3185#if OMPT_SUPPORT && OMPT_OPTIONAL
3187 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3192 ompt_mutex_test_lock, omp_lock_hint_none,
3193 __ompt_get_mutex_impl_type(user_lock),
3194 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3197#if KMP_USE_INLINED_TAS
3199 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
3201#elif KMP_USE_INLINED_FUTEX
3203 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
3207 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
3213#if OMPT_SUPPORT && OMPT_OPTIONAL
3216 ompt_mutex_test_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3247 __kmp_itt_lock_acquiring(
lck);
3249#if OMPT_SUPPORT && OMPT_OPTIONAL
3251 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3256 ompt_mutex_test_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
3257 (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3264 __kmp_itt_lock_acquired(
lck);
3266 __kmp_itt_lock_cancelled(
lck);
3269#if OMPT_SUPPORT && OMPT_OPTIONAL
3272 ompt_mutex_test_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3285#if KMP_USE_DYNAMIC_LOCK
3290#if OMPT_SUPPORT && OMPT_OPTIONAL
3292 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3297 ompt_mutex_test_nest_lock, omp_lock_hint_none,
3298 __ompt_get_mutex_impl_type(user_lock),
3299 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3302 rc = KMP_D_LOCK_FUNC(user_lock,
test)((kmp_dyna_lock_t *)user_lock, gtid);
3310#if OMPT_SUPPORT && OMPT_OPTIONAL
3316 ompt_mutex_test_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
3323 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3336 (
sizeof(
lck->tas.lk.poll) +
sizeof(
lck->tas.lk.depth_locked) <=
3342 (
sizeof(
lck->futex.lk.poll) +
sizeof(
lck->futex.lk.depth_locked) <=
3352 __kmp_itt_lock_acquiring(
lck);
3355#if OMPT_SUPPORT && OMPT_OPTIONAL
3357 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3363 ompt_mutex_test_nest_lock, omp_lock_hint_none,
3364 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)
lck,
3372 __kmp_itt_lock_acquired(
lck);
3374 __kmp_itt_lock_cancelled(
lck);
3377#if OMPT_SUPPORT && OMPT_OPTIONAL
3383 ompt_mutex_test_nest_lock, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3389 ompt_mutex_scope_begin, (ompt_wait_id_t)(uintptr_t)
lck, codeptr);
3408#define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \
3409 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod))
3411#define __KMP_GET_REDUCTION_METHOD(gtid) \
3412 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method)
3418static __forceinline
void
3428#if KMP_USE_DYNAMIC_LOCK
3430 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)
crit;
3433 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3435 KMP_GET_D_TAG(__kmp_user_lock_seq));
3437 __kmp_init_indirect_csptr(
crit,
loc, global_tid,
3438 KMP_GET_I_TAG(__kmp_user_lock_seq));
3444 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3450 KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
3452 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3458 KMP_I_LOCK_FUNC(ilk, set)(
lck, global_tid);
3482static __forceinline
void
3488#if KMP_USE_DYNAMIC_LOCK
3490 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3494 KMP_D_LOCK_FUNC(
lck, unset)((kmp_dyna_lock_t *)
lck, global_tid);
3496 kmp_indirect_lock_t *ilk =
3497 (kmp_indirect_lock_t *)
TCR_PTR(*((kmp_indirect_lock_t **)
crit));
3500 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3523static __forceinline
int
3529 if (th->th.th_teams_microtask) {
3530 *team_p = team = th->th.th_team;
3531 if (team->
t.t_level == th->th.th_teams_level) {
3535 th->th.th_info.ds.ds_tid = team->
t.t_master_tid;
3536 th->th.th_team = team->
t.t_parent;
3537 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3538 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3539 *task_state = th->th.th_task_state;
3540 th->th.th_task_state = 0;
3548static __forceinline
void
3551 th->th.th_info.ds.ds_tid = 0;
3552 th->th.th_team = team;
3553 th->th.th_team_nproc = team->
t.t_nproc;
3554 th->th.th_task_team = team->
t.t_task_team[task_state];
3576 size_t reduce_size,
void *reduce_data,
3577 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3585 int teams_swapped = 0, task_state;
3586 KA_TRACE(10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3601#if KMP_USE_DYNAMIC_LOCK
3629 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func,
lck);
3682 ompt_frame_t *ompt_frame;
3685 if (ompt_frame->enter_frame.ptr == NULL)
3688 OMPT_STORE_RETURN_ADDRESS(global_tid);
3695 global_tid,
FALSE, reduce_size, reduce_data, reduce_func);
3696 retval = (retval != 0) ? (0) : (1);
3697#if OMPT_SUPPORT && OMPT_OPTIONAL
3699 ompt_frame->enter_frame = ompt_data_none;
3716 if (teams_swapped) {
3721 (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3722 global_tid, packed_reduction_method, retval));
3740 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3781 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3782 global_tid, packed_reduction_method));
3805 size_t reduce_size,
void *reduce_data,
3806 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3813 int teams_swapped = 0, task_state;
3815 KA_TRACE(10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid));
3830#if KMP_USE_DYNAMIC_LOCK
3842 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func,
lck);
3871 ompt_frame_t *ompt_frame;
3874 if (ompt_frame->enter_frame.ptr == NULL)
3877 OMPT_STORE_RETURN_ADDRESS(global_tid);
3885 global_tid,
TRUE, reduce_size, reduce_data, reduce_func);
3886 retval = (retval != 0) ? (0) : (1);
3887#if OMPT_SUPPORT && OMPT_OPTIONAL
3889 ompt_frame->enter_frame = ompt_data_none;
3906 if (teams_swapped) {
3911 (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
3912 global_tid, packed_reduction_method, retval));
3932 int teams_swapped = 0, task_state;
3934 KA_TRACE(10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid));
3953 ompt_frame_t *ompt_frame;
3956 if (ompt_frame->enter_frame.ptr == NULL)
3959 OMPT_STORE_RETURN_ADDRESS(global_tid);
3965#if OMPT_SUPPORT && OMPT_OPTIONAL
3967 ompt_frame->enter_frame = ompt_data_none;
3979 ompt_frame_t *ompt_frame;
3982 if (ompt_frame->enter_frame.ptr == NULL)
3985 OMPT_STORE_RETURN_ADDRESS(global_tid);
3991#if OMPT_SUPPORT && OMPT_OPTIONAL
3993 ompt_frame->enter_frame = ompt_data_none;
4000 ompt_frame_t *ompt_frame;
4003 if (ompt_frame->enter_frame.ptr == NULL)
4006 OMPT_STORE_RETURN_ADDRESS(global_tid);
4013#if OMPT_SUPPORT && OMPT_OPTIONAL
4015 ompt_frame->enter_frame = ompt_data_none;
4031 if (teams_swapped) {
4038 KA_TRACE(10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n",
4039 global_tid, packed_reduction_method));
4044#undef __KMP_GET_REDUCTION_METHOD
4045#undef __KMP_SET_REDUCTION_METHOD
4059 return thread->th.th_current_task->td_task_id;
4074 parent_task = thread->th.th_current_task->
td_parent;
4075 return (parent_task == NULL ? 0 : parent_task->
td_task_id);
4103 (
"__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
4104 gtid, num_dims, !team->
t.t_serialized));
4108 if (team->
t.t_serialized) {
4109 KA_TRACE(20, (
"__kmpc_doacross_init() exit: serialized team\n"));
4120 th,
sizeof(
kmp_int64) * (4 * num_dims + 1));
4131 for (
j = 1;
j < num_dims; ++
j) {
4134 if (dims[
j].st == 1) {
4136 range_length = dims[
j].
up - dims[
j].
lo + 1;
4138 if (dims[
j].st > 0) {
4140 range_length = (
kmp_uint64)(dims[
j].up - dims[
j].lo) / dims[
j].
st + 1;
4155 if (dims[0].st == 1) {
4156 trace_count = dims[0].
up - dims[0].
lo + 1;
4157 }
else if (dims[0].st > 0) {
4159 trace_count = (
kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].
st + 1;
4162 trace_count = (
kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].
st) + 1;
4164 for (
j = 1;
j < num_dims; ++
j) {
4186 if (flags == NULL) {
4189 (size_t)trace_count / 8 + 8;
4209 KA_TRACE(20, (
"__kmpc_doacross_init() exit: T#%d\n", gtid));
4223 KA_TRACE(20, (
"__kmpc_doacross_wait() enter: called T#%d\n", gtid));
4224 if (team->
t.t_serialized) {
4225 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: serialized team\n"));
4230 pr_buf = th->th.th_dispatch;
4236#
if OMPT_SUPPORT && OMPT_OPTIONAL
4240 if (vec[0] < lo || vec[0] > up) {
4241 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4242 "bounds [%lld,%lld]\n",
4243 gtid, vec[0], lo, up));
4246 iter_number = vec[0] - lo;
4247 }
else if (st > 0) {
4248 if (vec[0] < lo || vec[0] > up) {
4249 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4250 "bounds [%lld,%lld]\n",
4251 gtid, vec[0], lo, up));
4254 iter_number = (
kmp_uint64)(vec[0] - lo) / st;
4256 if (vec[0] > lo || vec[0] < up) {
4257 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4258 "bounds [%lld,%lld]\n",
4259 gtid, vec[0], lo, up));
4262 iter_number = (
kmp_uint64)(lo - vec[0]) / (-st);
4264#if OMPT_SUPPORT && OMPT_OPTIONAL
4265 deps[0].variable.value = iter_number;
4266 deps[0].dependence_type = ompt_dependence_type_sink;
4268 for (
i = 1;
i < num_dims; ++
i) {
4276 if (vec[
i] < lo || vec[
i] > up) {
4277 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4278 "bounds [%lld,%lld]\n",
4279 gtid, vec[
i], lo, up));
4283 }
else if (st > 0) {
4284 if (vec[
i] < lo || vec[
i] > up) {
4285 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4286 "bounds [%lld,%lld]\n",
4287 gtid, vec[
i], lo, up));
4292 if (vec[
i] > lo || vec[
i] < up) {
4293 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4294 "bounds [%lld,%lld]\n",
4295 gtid, vec[
i], lo, up));
4300 iter_number = iter + ln * iter_number;
4301#if OMPT_SUPPORT && OMPT_OPTIONAL
4302 deps[
i].variable.value = iter;
4303 deps[
i].dependence_type = ompt_dependence_type_sink;
4306 shft = iter_number % 32;
4313#if OMPT_SUPPORT && OMPT_OPTIONAL
4316 &(OMPT_CUR_TASK_INFO(th)->task_data), deps, (
kmp_uint32)num_dims);
4320 (
"__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
4321 gtid, (iter_number << 5) + shft));
4335 KA_TRACE(20, (
"__kmpc_doacross_post() enter: called T#%d\n", gtid));
4336 if (team->
t.t_serialized) {
4337 KA_TRACE(20, (
"__kmpc_doacross_post() exit: serialized team\n"));
4343 pr_buf = th->th.th_dispatch;
4348#
if OMPT_SUPPORT && OMPT_OPTIONAL
4352 iter_number = vec[0] - lo;
4353 }
else if (st > 0) {
4354 iter_number = (
kmp_uint64)(vec[0] - lo) / st;
4356 iter_number = (
kmp_uint64)(lo - vec[0]) / (-st);
4358#if OMPT_SUPPORT && OMPT_OPTIONAL
4359 deps[0].variable.value = iter_number;
4360 deps[0].dependence_type = ompt_dependence_type_source;
4362 for (
i = 1;
i < num_dims; ++
i) {
4370 }
else if (st > 0) {
4375 iter_number = iter + ln * iter_number;
4376#if OMPT_SUPPORT && OMPT_OPTIONAL
4377 deps[
i].variable.value = iter;
4378 deps[
i].dependence_type = ompt_dependence_type_source;
4381#if OMPT_SUPPORT && OMPT_OPTIONAL
4384 &(OMPT_CUR_TASK_INFO(th)->task_data), deps, (
kmp_uint32)num_dims);
4387 shft = iter_number % 32;
4393 KA_TRACE(20, (
"__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4394 (iter_number << 5) + shft));
4404 KA_TRACE(20, (
"__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4405 if (team->
t.t_serialized) {
4406 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: serialized team %p\n", team));
4411 if (num_done == th->th.th_team_nproc) {
4430 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: T#%d\n", gtid));
4486 (ompt_severity_t)severity, message,
KMP_STRLEN(message),
4502 KMP_WARNING(UserDirectedWarning, src_loc, message);
4504 KMP_FATAL(UserDirectedError, src_loc, message);
4512#if OMPT_SUPPORT && OMPT_OPTIONAL
4517 ompt_work_scope, ompt_scope_begin,
4518 &(team->
t.ompt_team_info.parallel_data),
4519 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
4528#if OMPT_SUPPORT && OMPT_OPTIONAL
4533 ompt_work_scope, ompt_scope_end,
4534 &(team->
t.ompt_team_info.parallel_data),
4535 &(team->
t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
4541#ifdef KMP_USE_VERSION_SYMBOLS
4550#ifdef omp_set_affinity_format
4551#undef omp_set_affinity_format
4553#ifdef omp_get_affinity_format
4554#undef omp_get_affinity_format
4556#ifdef omp_display_affinity
4557#undef omp_display_affinity
4559#ifdef omp_capture_affinity
4560#undef omp_capture_affinity
A simple pure header implementation of VLA that aims to replace uses of actual VLA,...
@ KMP_IDENT_WORK_LOOP
To mark a static loop in OMPT callbacks.
@ KMP_IDENT_WORK_SECTIONS
To mark a sections directive in OMPT callbacks.
@ KMP_IDENT_AUTOPAR
Entry point generated by auto-parallelization.
@ KMP_IDENT_WORK_DISTRIBUTE
To mark a distribute construct in OMPT callbacks.
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
void __kmpc_fork_call_if(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, kmp_int32 cond, void *args)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
The type for a microtask which gets passed to __kmpc_fork_call().
void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams_lb, kmp_int32 num_teams_ub, kmp_int32 num_threads)
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
void __kmpc_end(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void * __kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid, void *cpy_data)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_masked(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, const struct kmp_dim *dims)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
kmp_int32 __kmpc_masked(ident_t *loc, kmp_int32 global_tid, kmp_int32 filter)
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
__itt_string_handle * name
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int mask
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t size
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d int
int __kmp_pause_resource(kmp_pause_status_t level)
void * omp_allocator_handle_t
void __kmp_aux_set_defaults(char const *str, size_t len)
void __kmp_set_schedule(int gtid, kmp_sched_t new_sched, int chunk)
void __kmp_teams_master(int gtid)
#define UNPACK_REDUCTION_BARRIER(packed_reduction_method)
int __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size, void *reduce_data, void(*reduce)(void *, void *))
struct KMP_ALIGN_CACHE dispatch_private_info dispatch_private_info_t
void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al)
void __kmp_push_num_teams_51(ident_t *loc, int gtid, int num_teams_lb, int num_teams_ub, int num_threads)
#define __kmp_assign_root_init_mask()
int __kmp_dflt_max_active_levels
static kmp_team_t * __kmp_team_from_gtid(int gtid)
kmp_tasking_mode_t __kmp_tasking_mode
char * __kmp_affinity_format
void * __kmp_alloc(int gtid, size_t align, size_t sz, omp_allocator_handle_t al)
void * __kmp_calloc(int gtid, size_t align, size_t nmemb, size_t sz, omp_allocator_handle_t al)
@ fork_context_intel
Called from Intel generated code.
void __kmp_exit_single(int gtid)
int __kmp_get_team_size(int gtid, int level)
kmp_uint32 __kmp_eq_4(kmp_uint32 value, kmp_uint32 checker)
volatile int __kmp_all_nth
kmp_target_offload_kind_t __kmp_target_offload
void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref)
KMP_EXPORT void __kmpc_critical_with_hint(ident_t *, kmp_int32 global_tid, kmp_critical_name *, uint32_t hint)
void __kmp_push_num_threads(ident_t *loc, int gtid, int num_threads)
#define set__dynamic(xthread, xval)
#define __kmp_entry_gtid()
void __kmp_internal_end_library(int gtid)
void __kmp_set_max_active_levels(int gtid, int new_max_active_levels)
static int __kmp_tid_from_gtid(int gtid)
void __kmp_internal_end_thread(int gtid)
void __kmp_user_set_library(enum library_type arg)
KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
int __kmp_gtid_get_specific(void)
volatile int __kmp_init_middle
void __kmp_set_num_threads(int new_nth, int gtid)
PACKED_REDUCTION_METHOD_T __kmp_determine_reduction_method(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void * __kmp_realloc(int gtid, void *ptr, size_t sz, omp_allocator_handle_t al, omp_allocator_handle_t free_al)
void __kmp_end_split_barrier(enum barrier_type bt, int gtid)
int PACKED_REDUCTION_METHOD_T
int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws)
static kmp_info_t * __kmp_entry_thread()
int __kmp_get_ancestor_thread_num(int gtid, int level)
#define __kmp_thread_malloc(th, size)
void __kmp_middle_initialize(void)
static void copy_icvs(kmp_internal_control_t *dst, kmp_internal_control_t *src)
kmp_info_t ** __kmp_threads
#define TEST_REDUCTION_METHOD(packed_reduction_method, which_reduction_block)
static void __kmp_reset_root_init_mask(int gtid)
void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref)
int __kmp_dispatch_num_buffers
#define set__max_active_levels(xthread, xval)
#define KMP_MASTER_GTID(gtid)
void __kmp_parallel_initialize(void)
int(* launch_t)(int gtid)
int __kmp_ignore_mppbeg(void)
volatile int __kmp_init_parallel
enum kmp_sched kmp_sched_t
void __kmp_aux_set_stacksize(size_t arg)
static const size_t KMP_AFFINITY_FORMAT_SIZE
void __kmp_push_num_teams(ident_t *loc, int gtid, int num_teams, int num_threads)
size_t __kmp_aux_capture_affinity(int gtid, const char *format, kmp_str_buf_t *buffer)
int __kmp_fork_call(ident_t *loc, int gtid, enum fork_context_e fork_context, kmp_int32 argc, microtask_t microtask, launch_t invoker, kmp_va_list ap)
int __kmp_env_consistency_check
void __kmp_aux_display_affinity(int gtid, const char *format)
void __kmp_push_proc_bind(ident_t *loc, int gtid, kmp_proc_bind_t proc_bind)
int __kmp_invoke_task_func(int gtid)
#define __kmp_thread_calloc(th, nelem, elsize)
void __kmp_save_internal_controls(kmp_info_t *thread)
int __kmp_invoke_teams_master(int gtid)
static void __kmp_aux_convert_blocktime(int *bt)
void __kmp_serial_initialize(void)
kmp_uint32 __kmp_wait_4(kmp_uint32 volatile *spinner, kmp_uint32 checker, kmp_uint32(*pred)(kmp_uint32, kmp_uint32), void *obj)
void __kmp_resume_if_soft_paused()
KMP_EXPORT void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
static void __kmp_assert_valid_gtid(kmp_int32 gtid)
void __kmp_serialized_parallel(ident_t *id, kmp_int32 gtid)
void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr)
void __kmp_internal_begin(void)
static kmp_info_t * __kmp_thread_from_gtid(int gtid)
#define KMP_MIN_DISP_NUM_BUFF
volatile int __kmp_init_serial
#define KMP_MAX_DISP_NUM_BUFF
int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int npr, int argc, void *argv[])
static void __kmp_type_convert(T1 src, T2 *dest)
void __kmp_join_call(ident_t *loc, int gtid, int exit_teams=0)
int __kmp_ignore_mppend(void)
union KMP_ALIGN_CACHE kmp_info kmp_info_t
void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team, int wait=1)
void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid)
#define __kmp_thread_free(th, ptr)
static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit, ident_t const *loc, kmp_int32 gtid)
void ompc_set_dynamic(int flag)
int kmpc_unset_affinity_mask_proc(int proc, void **mask)
#define __KMP_GET_REDUCTION_METHOD(gtid)
void __kmpc_unset_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void * omp_aligned_calloc(size_t align, size_t nmemb, size_t size, omp_allocator_handle_t allocator)
void __kmpc_destroy_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void __kmpc_doacross_post(ident_t *loc, int gtid, const kmp_int64 *vec)
void * omp_calloc(size_t nmemb, size_t size, omp_allocator_handle_t allocator)
void kmpc_set_disp_num_buffers(int arg)
static __forceinline int __kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p, int *task_state)
int ompc_get_ancestor_thread_num(int level)
void KMP_EXPAND_NAME() ompc_set_affinity_format(char const *format)
void kmpc_set_stacksize_s(size_t arg)
void KMP_EXPAND_NAME() ompc_display_affinity(char const *format)
void __kmpc_scope(ident_t *loc, kmp_int32 gtid, void *reserved)
static __forceinline void __kmp_end_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
int kmpc_set_affinity_mask_proc(int proc, void **mask)
void kmpc_set_library(int arg)
void __kmpc_set_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void kmpc_set_stacksize(int arg)
#define DESTROY_NESTED_LOCK
void ompc_set_schedule(omp_sched_t kind, int modifier)
void __kmpc_error(ident_t *loc, int severity, const char *message)
void * omp_aligned_alloc(size_t align, size_t size, omp_allocator_handle_t allocator)
void kmpc_set_blocktime(int arg)
#define __KMP_SET_REDUCTION_METHOD(gtid, rmethod)
void kmpc_set_defaults(char const *str)
void __kmpc_end_scope(ident_t *loc, kmp_int32 gtid, void *reserved)
void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void omp_free(void *ptr, omp_allocator_handle_t allocator)
void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void * omp_realloc(void *ptr, size_t size, omp_allocator_handle_t allocator, omp_allocator_handle_t free_allocator)
kmp_uint64 __kmpc_get_taskid()
int ompc_get_team_size(int level)
size_t KMP_EXPAND_NAME() ompc_get_affinity_format(char *buffer, size_t size)
int kmpc_get_affinity_mask_proc(int proc, void **mask)
void __kmpc_doacross_fini(ident_t *loc, int gtid)
#define RELEASE_NESTED_LOCK
int __kmpc_test_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void __kmpc_destroy_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
static __forceinline void __kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team, int task_state)
void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid)
int __kmpc_invoke_task_func(int gtid)
void __kmpc_unset_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind)
void __kmpc_init_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
#define ACQUIRE_NESTED_LOCK
void ompc_set_nested(int flag)
size_t KMP_EXPAND_NAME() ompc_capture_affinity(char *buffer, size_t buf_size, char const *format)
static __forceinline void __kmp_enter_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void ompc_set_max_active_levels(int max_active_levels)
kmp_uint64 __kmpc_get_parent_taskid()
void ompc_set_num_threads(int arg)
void * omp_alloc(size_t size, omp_allocator_handle_t allocator)
void __kmpc_doacross_wait(ident_t *loc, int gtid, const kmp_int64 *vec)
int __kmpc_get_target_offload(void)
int __kmpc_pause_resource(kmp_pause_status_t level)
int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
#define KMP_DEBUG_ASSERT(cond)
unsigned long long kmp_uint64
void __kmp_push_sync(int gtid, enum cons_type ct, ident_t const *ident, kmp_user_lock_p lck)
void __kmp_check_sync(int gtid, enum cons_type ct, ident_t const *ident, kmp_user_lock_p lck)
enum cons_type __kmp_pop_workshare(int gtid, enum cons_type ct, ident_t const *ident)
void __kmp_pop_sync(int gtid, enum cons_type ct, ident_t const *ident)
void __kmp_check_barrier(int gtid, enum cons_type ct, ident_t const *ident)
void __kmp_pop_parallel(int gtid, ident_t const *ident)
static volatile kmp_i18n_cat_status_t status
static kmp_bootstrap_lock_t lock
#define USE_ITT_BUILD_ARG(x)
size_t __kmp_base_user_lock_size
enum kmp_lock_kind __kmp_user_lock_kind
kmp_user_lock_p __kmp_user_lock_allocate(void **user_lock, kmp_int32 gtid, kmp_lock_flags_t flags)
void __kmp_user_lock_free(void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck)
kmp_user_lock_p __kmp_lookup_user_lock(void **user_lock, char const *func)
#define INTEL_CRITICAL_SIZE
#define OMP_NEST_LOCK_T_SIZE
#define KMP_CHECK_USER_LOCK_INIT()
static void __kmp_release_user_lock_with_checks(kmp_user_lock_p lck, kmp_int32 gtid)
union kmp_user_lock * kmp_user_lock_p
static int __kmp_acquire_user_lock_with_checks(kmp_user_lock_p lck, kmp_int32 gtid)
#define KMP_LOCK_RELEASED
static void __kmp_destroy_user_lock_with_checks(kmp_user_lock_p lck)
static void __kmp_init_user_lock_with_checks(kmp_user_lock_p lck)
#define kmp_lf_critical_section
static void __kmp_set_user_lock_location(kmp_user_lock_p lck, const ident_t *loc)
#define KMP_LOCK_ACQUIRED_FIRST
#define OMP_CRITICAL_SIZE
#define KMP_LOCK_STILL_HELD
#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv)
void(* microtask_t)(int *gtid, int *npr,...)
#define KMP_TEST_THEN_OR32(p, v)
#define KMP_TEST_THEN_INC32(p)
#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv)
#define KMP_VERSION_OMPC_SYMBOL(apic_name, api_name, ver_num, ver_str)
#define kmp_va_addr_of(ap)
#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv)
unsigned long kmp_uintptr_t
#define KMP_EXPAND_NAME(api_name)
#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv)
static void __kmp_strncpy_truncate(char *buffer, size_t buf_size, char const *src, size_t src_size)
Functions for collecting statistics.
#define KMP_PUSH_PARTITIONED_TIMER(name)
#define KMP_GET_THREAD_STATE()
#define KMP_POP_PARTITIONED_TIMER()
#define KMP_COUNT_BLOCK(n)
#define KMP_SET_THREAD_STATE(state_name)
kmp_str_loc_t __kmp_str_loc_init(char const *psource, bool init_fname)
void __kmp_str_buf_free(kmp_str_buf_t *buffer)
char * __kmp_str_format(char const *format,...)
int __kmp_str_match_true(char const *data)
void __kmp_str_loc_free(kmp_str_loc_t *loc)
void __kmp_str_free(char **str)
#define __kmp_str_buf_init(b)
#define omp_get_affinity_format
#define omp_set_affinity_format
#define omp_display_affinity
#define omp_capture_affinity
int test(unsigned upper_bound)
ompt_callbacks_active_t ompt_enabled
ompt_callbacks_internal_t ompt_callbacks
#define OMPT_GET_RETURN_ADDRESS(level)
#define OMPT_GET_FRAME_ADDRESS(level)
ompt_team_info_t * __ompt_get_teaminfo(int depth, int *size)
int __ompt_get_task_info_internal(int ancestor_level, int *type, ompt_data_t **task_data, ompt_frame_t **task_frame, ompt_data_t **parallel_data, int *thread_num)
ompt_task_info_t * __ompt_get_task_info_object(int depth)
void __ompt_lw_taskteam_unlink(kmp_info_t *thr)
ompt_data_t * __ompt_get_thread_data_internal()
#define OMPT_REDUCTION_BEGIN
#define OMPT_REDUCTION_DECL(this_thr, gtid)
#define OMPT_REDUCTION_END
kmp_int32 doacross_num_done
volatile kmp_int32 doacross_buf_idx
volatile kmp_uint32 * doacross_flags
std::atomic< kmp_int32 > poll
kmp_int32 tt_found_proxy_tasks
kmp_int32 tt_hidden_helper_task_encountered
kmp_int32 th_doacross_buf_idx
volatile kmp_uint32 * th_doacross_flags
kmp_int64 * th_doacross_info
struct kmp_internal_control * next
kmp_taskdata_t * td_parent
ompt_data_t parallel_data