24#define USE_NGO_STORES 1
27#if KMP_MIC && USE_NGO_STORES
29#define ngo_load(src) __m512d Vt = _mm512_load_pd((void *)(src))
30#define ngo_store_icvs(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt)
31#define ngo_store_go(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt)
32#define ngo_sync() __asm__ volatile("lock; addl $0,0(%%rsp)" ::: "memory")
34#define ngo_load(src) ((void)0)
35#define ngo_store_icvs(dst, src) copy_icvs((dst), (src))
36#define ngo_store_go(dst, src) KMP_MEMCPY((dst), (src), CACHE_LINE)
37#define ngo_sync() ((void)0)
47void distributedBarrier::computeVarsForN(
size_t n) {
52 int ncores_per_socket =
58 if (ncores_per_socket <= 0)
59 ncores_per_socket = 1;
78 if (nsockets == 1 ||
num_gos == 1)
109void distributedBarrier::computeGo(
size_t n) {
128void distributedBarrier::resize(
size_t nthr) {
166 go[
j].go.store(next_go);
183void distributedBarrier::init(
size_t nthr) {
200 computeVarsForN(nthr);
211 size_t start,
size_t stop,
size_t inc,
218 for (
size_t thr = start; thr <
stop; thr += inc) {
220 int gtid = other_threads[thr]->th.th_info.ds.ds_gtid;
237 team = this_thr->th.th_team;
238 nproc = this_thr->th.th_team_nproc;
239 other_threads = team->
t.t_threads;
241 my_current_iter =
b->iter[tid].iter;
243 group_leader = ((tid %
b->threads_per_group) == 0);
246 (
"__kmp_dist_barrier_gather: T#%d(%d:%d) enter; barrier type %d\n",
247 gtid, team->
t.t_id, tid, bt));
249#if USE_ITT_BUILD && USE_ITT_NOTIFY
251 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
252 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
253 __itt_get_timestamp();
259 size_t group_start = tid + 1;
260 size_t group_end = tid +
b->threads_per_group;
261 size_t threads_pending = 0;
263 if (group_end > nproc)
268 for (
size_t thr = group_start; thr < group_end; thr++) {
270 threads_pending +=
b->flags[my_current_iter][thr].stillNeed;
275 if (task_team != NULL) {
278 int tasks_completed =
FALSE;
297 }
while (threads_pending > 0);
303 for (
size_t thr = group_start; thr < group_end; thr++) {
304 (*reduce)(this_thr->th.th_local.reduce_data,
305 other_threads[thr]->th.th_local.reduce_data);
311 b->flags[my_next_iter][tid].stillNeed = 1;
314 b->flags[my_current_iter][tid].stillNeed = 0;
318 for (
size_t thr = 0; thr < nproc; thr +=
b->threads_per_group) {
319 threads_pending +=
b->flags[my_current_iter][thr].stillNeed;
324 if (task_team != NULL) {
327 int tasks_completed =
FALSE;
346 }
while (threads_pending > 0);
352 for (
size_t thr =
b->threads_per_group; thr < nproc;
353 thr +=
b->threads_per_group) {
354 (*reduce)(this_thr->th.th_local.reduce_data,
355 other_threads[thr]->th.th_local.reduce_data);
362 b->flags[my_next_iter][tid].stillNeed = 1;
365 b->flags[my_current_iter][tid].stillNeed = 0;
371 (
"__kmp_dist_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
372 gtid, team->
t.t_id, tid, bt));
386 KA_TRACE(20, (
"__kmp_dist_barrier_release: T#%d(%d) enter; barrier type %d\n",
389 thr_bar = &this_thr->th.th_bar[bt].bb;
394 if (this_thr->th.th_used_in_team.load() != 1 &&
395 this_thr->th.th_used_in_team.load() != 3) {
403 this_thr->th.th_used_in_team.load() == 0) {
406#if USE_ITT_BUILD && USE_ITT_NOTIFY
407 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
412 __kmp_itt_task_starting(itt_sync_obj);
418 if (itt_sync_obj != NULL)
420 __kmp_itt_task_finished(itt_sync_obj);
426 if (this_thr->th.th_used_in_team.load() != 1 &&
427 this_thr->th.th_used_in_team.load() != 3)
439 team = this_thr->th.th_team;
443 my_current_iter =
b->iter[tid].iter;
445 my_go_index = tid /
b->threads_per_go;
446 if (this_thr->th.th_used_in_team.load() == 3) {
450 if (
b->go[my_go_index].go.load() != next_go) {
453 &(
b->go[my_go_index].go), next_go, &(
b->sleep[tid].sleep));
456 b->iter[tid].iter == 0);
467 if (this_thr->th.th_used_in_team.load() == 1)
474 group_leader = ((tid %
b->threads_per_group) == 0);
477 for (
size_t go_idx = my_go_index + 1;
478 go_idx < my_go_index +
b->gos_per_group; go_idx++) {
479 b->go[go_idx].go.store(next_go);
485#if KMP_BARRIER_ICV_PUSH
486 if (propagate_icvs) {
489 copy_icvs(&team->
t.t_implicit_task_taskdata[tid].td_icvs,
492 &team->
t.t_implicit_task_taskdata[tid].td_icvs);
498 size_t nproc = this_thr->th.th_team_nproc;
499 size_t group_end = tid +
b->threads_per_group;
500 if (nproc < group_end)
505 team = this_thr->th.th_team;
507 my_current_iter =
b->iter[tid].iter;
509#if KMP_BARRIER_ICV_PUSH
510 if (propagate_icvs) {
513 &team->
t.t_implicit_task_taskdata[tid].td_icvs);
517 for (
size_t go_idx = 0; go_idx <
b->num_gos; go_idx +=
b->gos_per_group) {
518 b->go[go_idx].go.store(next_go);
523 size_t nproc = this_thr->th.th_team_nproc;
525 b->threads_per_group, tid);
529 for (
size_t go_idx = 1; go_idx <
b->gos_per_group; go_idx++) {
530 b->go[go_idx].go.store(next_go);
538 size_t nproc = this_thr->th.th_team_nproc;
539 size_t group_end = tid +
b->threads_per_group;
540 if (nproc < group_end)
550 20, (
"__kmp_dist_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
551 gtid, team->
t.t_id, tid, bt));
555template <
bool cancellable = false>
566 (
"__kmp_linear_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
567 gtid, team->
t.t_id, tid, bt));
568 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
570#if USE_ITT_BUILD && USE_ITT_NOTIFY
572 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
573 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
574 __itt_get_timestamp();
581 (
"__kmp_linear_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d)"
582 "arrived(%p): %llu => %llu\n",
584 team->
t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
594 int nproc = this_thr->th.th_team_nproc;
600 for (
i = 1;
i < nproc; ++
i) {
606 KA_TRACE(20, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) "
607 "arrived(%p) == %llu\n",
610 &other_threads[
i]->th.th_bar[bt].bb.b_arrived, new_state));
615 &other_threads[
i]->th.th_bar[bt].bb.b_arrived, new_state);
623#if USE_ITT_BUILD && USE_ITT_NOTIFY
626 if (__kmp_forkjoin_frames_mode == 2) {
627 this_thr->th.th_bar_min_time =
KMP_MIN(
628 this_thr->th.th_bar_min_time, other_threads[
i]->th.th_bar_min_time);
633 (
"__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
638 (*reduce)(this_thr->th.th_local.reduce_data,
639 other_threads[
i]->th.th_local.reduce_data);
645 KA_TRACE(20, (
"__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d "
646 "arrived(%p) = %llu\n",
647 gtid, team->
t.t_id, tid, team->
t.t_id, &team_bar->
b_arrived,
652 (
"__kmp_linear_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
653 gtid, team->
t.t_id, tid, bt));
657template <
bool cancellable = false>
667 kmp_uint32 nproc = this_thr->th.th_team_nproc;
672 other_threads = team->
t.t_threads;
674 KA_TRACE(20, (
"__kmp_linear_barrier_release: T#%d(%d:%d) primary enter for "
676 gtid, team->
t.t_id, tid, bt));
679#if KMP_BARRIER_ICV_PUSH
682 if (propagate_icvs) {
683 ngo_load(&team->
t.t_implicit_task_taskdata[0].td_icvs);
684 for (
i = 1;
i < nproc; ++
i) {
688 &team->
t.t_implicit_task_taskdata[0].td_icvs);
696 for (
i = 1;
i < nproc; ++
i) {
704 (
"__kmp_linear_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%d) "
705 "go(%p): %u => %u\n",
706 gtid, team->
t.t_id, tid, other_threads[
i]->th.th_info.ds.ds_gtid,
707 team->
t.t_id,
i, &other_threads[
i]->th.th_bar[bt].bb.b_go,
708 other_threads[
i]->th.th_bar[bt].bb.b_go,
716 KA_TRACE(20, (
"__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n",
726#if USE_ITT_BUILD && USE_ITT_NOTIFY
727 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
732 __kmp_itt_task_starting(itt_sync_obj);
738 if (itt_sync_obj != NULL)
740 __kmp_itt_task_finished(itt_sync_obj);
754 (
"__kmp_linear_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
760 (
"__kmp_linear_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
761 gtid, team->
t.t_id, tid, bt));
768 __kmp_linear_barrier_gather_template<false>(
775 return __kmp_linear_barrier_gather_template<true>(
782 __kmp_linear_barrier_release_template<false>(
789 return __kmp_linear_barrier_release_template<true>(
801 kmp_uint32 nproc = this_thr->th.th_team_nproc;
809 20, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
810 gtid, team->
t.t_id, tid, bt));
811 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
813#if USE_ITT_BUILD && USE_ITT_NOTIFY
815 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
816 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
817 __itt_get_timestamp();
822 child_tid = (tid << branch_bits) + 1;
823 if (child_tid < nproc) {
828 kmp_info_t *child_thr = other_threads[child_tid];
832 if (child + 1 <= branch_factor && child_tid + 1 < nproc)
834 &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_arrived);
837 (
"__kmp_tree_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
838 "arrived(%p) == %llu\n",
840 team->
t.t_id, child_tid, &child_bar->b_arrived, new_state));
844#if USE_ITT_BUILD && USE_ITT_NOTIFY
847 if (__kmp_forkjoin_frames_mode == 2) {
848 this_thr->th.th_bar_min_time =
KMP_MIN(this_thr->th.th_bar_min_time,
849 child_thr->th.th_bar_min_time);
854 (
"__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
856 team->
t.t_id, child_tid));
859 (*reduce)(this_thr->th.th_local.reduce_data,
860 child_thr->th.th_local.reduce_data);
865 }
while (child <= branch_factor && child_tid < nproc);
869 kmp_int32 parent_tid = (tid - 1) >> branch_bits;
872 (
"__kmp_tree_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
873 "arrived(%p): %llu => %llu\n",
875 team->
t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
887 team->
t.t_bar[bt].b_arrived = new_state;
890 KA_TRACE(20, (
"__kmp_tree_barrier_gather: T#%d(%d:%d) set team %d "
891 "arrived(%p) = %llu\n",
892 gtid, team->
t.t_id, tid, team->
t.t_id,
893 &team->
t.t_bar[bt].b_arrived, team->
t.t_bar[bt].b_arrived));
896 (
"__kmp_tree_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
897 gtid, team->
t.t_id, tid, bt));
915 KA_TRACE(20, (
"__kmp_tree_barrier_release: T#%d wait go(%p) == %u\n", gtid,
920#if USE_ITT_BUILD && USE_ITT_NOTIFY
921 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
926 __kmp_itt_task_starting(itt_sync_obj);
932 if (itt_sync_obj != NULL)
934 __kmp_itt_task_finished(itt_sync_obj);
948 (
"__kmp_tree_barrier_release: T#%d(%d:%d) set go(%p) = %u\n", gtid,
954 KA_TRACE(20, (
"__kmp_tree_barrier_release: T#%d(%d:%d) primary enter for "
956 gtid, team->
t.t_id, tid, bt));
958 nproc = this_thr->th.th_team_nproc;
959 child_tid = (tid << branch_bits) + 1;
961 if (child_tid < nproc) {
966 kmp_info_t *child_thr = other_threads[child_tid];
970 if (child + 1 <= branch_factor && child_tid + 1 < nproc)
972 &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_go);
975#if KMP_BARRIER_ICV_PUSH
978 if (propagate_icvs) {
980 team->
t.t_threads[child_tid], team,
982 copy_icvs(&team->
t.t_implicit_task_taskdata[child_tid].td_icvs,
983 &team->
t.t_implicit_task_taskdata[0].td_icvs);
988 (
"__kmp_tree_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)"
989 "go(%p): %u => %u\n",
991 team->
t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
998 }
while (child <= branch_factor && child_tid < nproc);
1001 20, (
"__kmp_tree_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
1002 gtid, team->
t.t_id, tid, bt));
1014 kmp_uint32 num_threads = this_thr->th.th_team_nproc;
1022 (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n",
1023 gtid, team->
t.t_id, tid, bt));
1024 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
1026#if USE_ITT_BUILD && USE_ITT_NOTIFY
1028 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
1029 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time =
1030 __itt_get_timestamp();
1036 for (
level = 0, offset = 1; offset < num_threads;
1037 level += branch_bits, offset <<= branch_bits) {
1041 if (((tid >>
level) & (branch_factor - 1)) != 0) {
1042 kmp_int32 parent_tid = tid & ~((1 << (
level + branch_bits)) - 1);
1046 (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) "
1047 "arrived(%p): %llu => %llu\n",
1049 team->
t.t_id, parent_tid, &thr_bar->b_arrived,
1056 p_flag.
set_waiter(other_threads[parent_tid]);
1064 for (child = 1, child_tid = tid + (1 <<
level);
1065 child < branch_factor && child_tid < num_threads;
1066 child++, child_tid += (1 <<
level)) {
1067 kmp_info_t *child_thr = other_threads[child_tid];
1068 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
1072 if (child + 1 < branch_factor && next_child_tid < num_threads)
1074 &other_threads[next_child_tid]->th.th_bar[bt].bb.b_arrived);
1077 (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) "
1078 "arrived(%p) == %llu\n",
1080 team->
t.t_id, child_tid, &child_bar->b_arrived, new_state));
1085#if USE_ITT_BUILD && USE_ITT_NOTIFY
1088 if (__kmp_forkjoin_frames_mode == 2) {
1089 this_thr->th.th_bar_min_time =
KMP_MIN(this_thr->th.th_bar_min_time,
1090 child_thr->th.th_bar_min_time);
1095 (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
1097 team->
t.t_id, child_tid));
1100 (*reduce)(this_thr->th.th_local.reduce_data,
1101 child_thr->th.th_local.reduce_data);
1112 team->
t.t_bar[bt].b_arrived = new_state;
1113 KA_TRACE(20, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) set team %d "
1114 "arrived(%p) = %llu\n",
1115 gtid, team->
t.t_id, tid, team->
t.t_id,
1116 &team->
t.t_bar[bt].b_arrived, team->
t.t_bar[bt].b_arrived));
1119 20, (
"__kmp_hyper_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n",
1120 gtid, team->
t.t_id, tid, bt));
1124#define KMP_REVERSE_HYPER_BAR
1147 KA_TRACE(20, (
"__kmp_hyper_barrier_release: T#%d(%d:%d) primary enter for "
1148 "barrier type %d\n",
1149 gtid, team->
t.t_id, tid, bt));
1150#if KMP_BARRIER_ICV_PUSH
1151 if (propagate_icvs) {
1153 &team->
t.t_implicit_task_taskdata[tid].td_icvs);
1157 KA_TRACE(20, (
"__kmp_hyper_barrier_release: T#%d wait go(%p) == %u\n", gtid,
1162#if USE_ITT_BUILD && USE_ITT_NOTIFY
1163 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) {
1167 __kmp_itt_task_starting(itt_sync_obj);
1173 if (itt_sync_obj != NULL)
1175 __kmp_itt_task_finished(itt_sync_obj);
1189 (
"__kmp_hyper_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
1193 num_threads = this_thr->th.th_team_nproc;
1194 other_threads = team->
t.t_threads;
1196#ifdef KMP_REVERSE_HYPER_BAR
1198 for (
level = 0, offset = 1;
1199 offset < num_threads && (((tid >>
level) & (branch_factor - 1)) == 0);
1200 level += branch_bits, offset <<= branch_bits)
1204 for (
level -= branch_bits, offset >>= branch_bits; offset != 0;
1205 level -= branch_bits, offset >>= branch_bits)
1208 for (
level = 0, offset = 1; offset < num_threads;
1209 level += branch_bits, offset <<= branch_bits)
1212#ifdef KMP_REVERSE_HYPER_BAR
1216 for (child = (child < branch_factor - 1) ? child : branch_factor - 1,
1217 child_tid = tid + (child <<
level);
1218 child >= 1; child--, child_tid -= (1 <<
level))
1220 if (((tid >>
level) & (branch_factor - 1)) != 0)
1225 for (child = 1, child_tid = tid + (1 <<
level);
1226 child < branch_factor && child_tid < num_threads;
1227 child++, child_tid += (1 <<
level))
1230 if (child_tid >= num_threads)
1233 kmp_info_t *child_thr = other_threads[child_tid];
1234 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
1238#ifdef KMP_REVERSE_HYPER_BAR
1239 if (child - 1 >= 1 && next_child_tid < num_threads)
1241 if (child + 1 < branch_factor && next_child_tid < num_threads)
1244 &other_threads[next_child_tid]->th.th_bar[bt].bb.b_go);
1247#if KMP_BARRIER_ICV_PUSH
1249 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1254 (
"__kmp_hyper_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)"
1255 "go(%p): %u => %u\n",
1257 team->
t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
1265#if KMP_BARRIER_ICV_PUSH
1266 if (propagate_icvs &&
1270 copy_icvs(&team->
t.t_implicit_task_taskdata[tid].td_icvs,
1271 &thr_bar->th_fixed_icvs);
1276 (
"__kmp_hyper_barrier_release: T#%d(%d:%d) exit for barrier type %d\n",
1277 gtid, team->
t.t_id, tid, bt));
1295 bool uninitialized = thr_bar->team == NULL;
1296 bool team_changed = team != thr_bar->team;
1297 bool team_sz_changed = nproc != thr_bar->nproc;
1298 bool tid_changed = tid != thr_bar->old_tid;
1299 bool retval =
false;
1301 if (uninitialized || team_sz_changed) {
1305 if (uninitialized || team_sz_changed || tid_changed) {
1306 thr_bar->my_level = thr_bar->depth - 1;
1307 thr_bar->parent_tid = -1;
1311 while (d < thr_bar->depth) {
1314 if (
d == thr_bar->depth - 2) {
1315 thr_bar->parent_tid = 0;
1316 thr_bar->my_level =
d;
1318 }
else if ((rem = tid % thr_bar->skip_per_level[
d + 1]) != 0) {
1321 thr_bar->parent_tid = tid - rem;
1322 thr_bar->my_level =
d;
1329 (thr_bar->skip_per_level[thr_bar->my_level])),
1330 &(thr_bar->offset));
1331 thr_bar->old_tid = tid;
1333 thr_bar->team = team;
1334 thr_bar->parent_bar =
1335 &team->
t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
1337 if (uninitialized || team_changed || tid_changed) {
1338 thr_bar->team = team;
1339 thr_bar->parent_bar =
1340 &team->
t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
1343 if (uninitialized || team_sz_changed || tid_changed) {
1344 thr_bar->nproc = nproc;
1345 thr_bar->leaf_kids = thr_bar->base_leaf_kids;
1346 if (thr_bar->my_level == 0)
1347 thr_bar->leaf_kids = 0;
1348 if (thr_bar->leaf_kids && (
kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc)
1350 thr_bar->leaf_state = 0;
1351 for (
int i = 0;
i < thr_bar->leaf_kids; ++
i)
1352 ((
char *)&(thr_bar->leaf_state))[7 -
i] = 1;
1363 kmp_uint32 nproc = this_thr->th.th_team_nproc;
1367 int level = team->
t.t_level;
1368 if (other_threads[0]
1369 ->th.th_teams_microtask)
1370 if (this_thr->th.th_teams_size.nteams > 1)
1373 thr_bar->use_oncore_barrier = 1;
1375 thr_bar->use_oncore_barrier = 0;
1377 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) enter for "
1378 "barrier type %d\n",
1379 gtid, team->
t.t_id, tid, bt));
1380 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]);
1382#if USE_ITT_BUILD && USE_ITT_NOTIFY
1384 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) {
1385 this_thr->th.th_bar_arrive_time = __itt_get_timestamp();
1392 if (thr_bar->my_level) {
1397 thr_bar->use_oncore_barrier) {
1398 if (thr_bar->leaf_kids) {
1402 ? thr_bar->b_arrived | thr_bar->leaf_state
1403 : team->
t.t_bar[bt].b_arrived | thr_bar->leaf_state;
1404 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) waiting "
1406 gtid, team->
t.t_id, tid));
1412 for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids;
1414 KA_TRACE(100, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
1416 gtid, team->
t.t_id, tid,
1419 (*reduce)(this_thr->th.th_local.reduce_data,
1420 other_threads[child_tid]->th.th_local.reduce_data);
1430 kmp_uint32 last = tid + thr_bar->skip_per_level[
d + 1],
1431 skip = thr_bar->skip_per_level[
d];
1434 for (child_tid = tid + skip; child_tid < (
int)last; child_tid += skip) {
1435 kmp_info_t *child_thr = other_threads[child_tid];
1436 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
1437 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait "
1439 "arrived(%p) == %llu\n",
1440 gtid, team->
t.t_id, tid,
1442 child_tid, &child_bar->b_arrived, new_state));
1446 KA_TRACE(100, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
1448 gtid, team->
t.t_id, tid,
1451 (*reduce)(this_thr->th.th_local.reduce_data,
1452 child_thr->th.th_local.reduce_data);
1459 kmp_uint32 last = tid + thr_bar->skip_per_level[
d + 1],
1460 skip = thr_bar->skip_per_level[
d];
1463 for (child_tid = tid + skip; child_tid < (
int)last; child_tid += skip) {
1464 kmp_info_t *child_thr = other_threads[child_tid];
1465 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
1466 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait "
1468 "arrived(%p) == %llu\n",
1469 gtid, team->
t.t_id, tid,
1471 child_tid, &child_bar->b_arrived, new_state));
1475 KA_TRACE(100, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += "
1477 gtid, team->
t.t_id, tid,
1480 (*reduce)(this_thr->th.th_local.reduce_data,
1481 child_thr->th.th_local.reduce_data);
1490 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) releasing"
1491 " T#%d(%d:%d) arrived(%p): %llu => %llu\n",
1492 gtid, team->
t.t_id, tid,
1494 thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
1500 !thr_bar->use_oncore_barrier) {
1503 other_threads[thr_bar->parent_tid]);
1509 thr_bar->offset + 1);
1510 flag.set_waiter(other_threads[thr_bar->parent_tid]);
1514 team->
t.t_bar[bt].b_arrived = new_state;
1515 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) set team %d "
1516 "arrived(%p) = %llu\n",
1517 gtid, team->
t.t_id, tid, team->
t.t_id,
1518 &team->
t.t_bar[bt].b_arrived, team->
t.t_bar[bt].b_arrived));
1521 KA_TRACE(20, (
"__kmp_hierarchical_barrier_gather: T#%d(%d:%d) exit for "
1522 "barrier type %d\n",
1523 gtid, team->
t.t_id, tid, bt));
1533 bool team_change =
false;
1538 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) primary "
1539 "entered barrier type %d\n",
1540 gtid, team->
t.t_id, tid, bt));
1543 if (!thr_bar->use_oncore_barrier ||
1545 thr_bar->team == NULL) {
1550 TCW_8(thr_bar->b_go,
1557 thr_bar->offset + 1, bt,
1560 if (thr_bar->wait_flag ==
1562 TCW_8(thr_bar->b_go,
1565 (
RCAST(
volatile char *,
1566 &(thr_bar->parent_bar->b_go)))[thr_bar->offset + 1] = 0;
1580 (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) set go(%p) = %u\n",
1585 nproc = this_thr->th.th_team_nproc;
1586 int level = team->
t.t_level;
1587 if (team->
t.t_threads[0]
1588 ->th.th_teams_microtask) {
1590 this_thr->th.th_teams_level ==
level)
1592 if (this_thr->th.th_teams_size.nteams > 1)
1596 thr_bar->use_oncore_barrier = 1;
1598 thr_bar->use_oncore_barrier = 0;
1602 unsigned short int old_leaf_kids = thr_bar->leaf_kids;
1603 kmp_uint64 old_leaf_state = thr_bar->leaf_state;
1610#if KMP_BARRIER_ICV_PUSH
1611 if (propagate_icvs) {
1617 &team->
t.t_implicit_task_taskdata[tid].td_icvs);
1619 thr_bar->use_oncore_barrier) {
1620 if (!thr_bar->my_level)
1623 copy_icvs(&team->
t.t_implicit_task_taskdata[tid].td_icvs,
1624 &thr_bar->parent_bar->th_fixed_icvs);
1627 if (thr_bar->my_level)
1629 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs);
1631 copy_icvs(&team->
t.t_implicit_task_taskdata[tid].td_icvs,
1632 &thr_bar->parent_bar->th_fixed_icvs);
1638 if (thr_bar->my_level) {
1642 thr_bar->use_oncore_barrier) {
1652 for (child_tid = thr_bar->skip_per_level[1]; child_tid < (
int)nproc;
1653 child_tid += thr_bar->skip_per_level[1]) {
1655 &team->
t.t_threads[child_tid]->th.th_bar[bt].bb;
1656 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) "
1657 "releasing T#%d(%d:%d)"
1658 " go(%p): %u => %u\n",
1659 gtid, team->
t.t_id, tid,
1661 child_tid, &child_bar->b_go, child_bar->b_go,
1665 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1669 TCW_8(thr_bar->b_go,
1672 if (thr_bar->leaf_kids) {
1675 old_leaf_kids < thr_bar->leaf_kids) {
1676 if (old_leaf_kids) {
1677 thr_bar->b_go |= old_leaf_state;
1680 last = tid + thr_bar->skip_per_level[1];
1683 for (child_tid = tid + 1 + old_leaf_kids; child_tid < (
int)last;
1685 kmp_info_t *child_thr = team->
t.t_threads[child_tid];
1686 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
1689 (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) releasing"
1690 " T#%d(%d:%d) go(%p): %u => %u\n",
1692 team->
t.t_id, child_tid, &child_bar->b_go, child_bar->b_go,
1700 thr_bar->b_go |= thr_bar->leaf_state;
1704 for (
int d = thr_bar->my_level - 1;
d >= 0;
1706 last = tid + thr_bar->skip_per_level[
d + 1];
1710 for (child_tid = tid + skip; child_tid < (
int)last; child_tid += skip) {
1711 kmp_info_t *child_thr = team->
t.t_threads[child_tid];
1712 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb;
1713 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) "
1714 "releasing T#%d(%d:%d) go(%p): %u => %u\n",
1715 gtid, team->
t.t_id, tid,
1717 child_tid, &child_bar->b_go, child_bar->b_go,
1725#if KMP_BARRIER_ICV_PUSH
1728 copy_icvs(&team->
t.t_implicit_task_taskdata[tid].td_icvs,
1729 &thr_bar->th_fixed_icvs);
1732 KA_TRACE(20, (
"__kmp_hierarchical_barrier_release: T#%d(%d:%d) exit for "
1733 "barrier type %d\n",
1734 gtid, team->
t.t_id, tid, bt));
1755 constexpr operator bool()
const {
return false; }
1766template <
bool cancellable = false>
1768 size_t reduce_size,
void *reduce_data,
1769 void (*reduce)(
void *,
void *)) {
1777#if OMPT_SUPPORT && OMPT_OPTIONAL
1778 ompt_data_t *my_task_data;
1779 ompt_data_t *my_parallel_data;
1780 void *return_address;
1781 ompt_sync_region_t barrier_kind;
1784 KA_TRACE(15, (
"__kmp_barrier: T#%d(%d:%d) has arrived\n", gtid,
1790 my_task_data = OMPT_CUR_TASK_DATA(this_thr);
1791 my_parallel_data = OMPT_CUR_TEAM_DATA(this_thr);
1792 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
1796 barrier_kind, ompt_scope_begin, my_parallel_data, my_task_data,
1801 barrier_kind, ompt_scope_begin, my_parallel_data, my_task_data,
1808 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier;
1812 if (!team->
t.t_serialized) {
1815 void *itt_sync_obj = NULL;
1817 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1818 itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
1824 (
"__kmp_barrier: T#%d(%d:%d) past tasking barrier\n", gtid,
1834 this_thr->th.th_team_bt_intervals =
1835 team->
t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
1836 this_thr->th.th_team_bt_set =
1837 team->
t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
1844 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1845 __kmp_itt_barrier_starting(gtid, itt_sync_obj);
1850 team->
t.t_bar[bt].b_master_arrived += 1;
1852 this_thr->th.th_bar[bt].bb.b_worker_arrived += 1;
1855 if (reduce != NULL) {
1857 this_thr->th.th_local.reduce_data = reduce_data;
1909 team->
t.t_bar[bt].b_team_arrived += 1;
1925 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1926 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
1928#if USE_ITT_BUILD && USE_ITT_NOTIFY
1930 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
1931 __kmp_forkjoin_frames_mode &&
1932 (this_thr->th.th_teams_microtask == NULL ||
1933 this_thr->th.th_teams_size.nteams == 1) &&
1934 team->
t.t_active_level == 1) {
1938 int nproc = this_thr->th.th_team_nproc;
1940 switch (__kmp_forkjoin_frames_mode) {
1942 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
1944 this_thr->th.th_frame_time = cur_time;
1948 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time,
1952 if (__itt_metadata_add_ptr) {
1954 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
1957 this_thr->th.th_bar_arrive_time = 0;
1958 for (
i = 1;
i < nproc; ++
i) {
1959 delta += (cur_time - other_threads[
i]->th.th_bar_arrive_time);
1960 other_threads[
i]->th.th_bar_arrive_time = 0;
1962 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time,
1966 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
1968 this_thr->th.th_frame_time = cur_time;
1976 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
1977 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
1980 if ((
status == 1 || !is_split) && !cancelled) {
2024 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
2025 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
2030 if (this_thr->th.th_task_team != NULL) {
2032 void *itt_sync_obj = NULL;
2033 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
2034 itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1);
2035 __kmp_itt_barrier_starting(gtid, itt_sync_obj);
2040 this_thr->th.th_task_team->tt.tt_found_proxy_tasks ==
TRUE ||
2041 this_thr->th.th_task_team->tt.tt_hidden_helper_task_encountered ==
2047 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
2048 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
2053 KA_TRACE(15, (
"__kmp_barrier: T#%d(%d:%d) is leaving with return value %d\n",
2062 barrier_kind, ompt_scope_end, my_parallel_data, my_task_data,
2067 barrier_kind, ompt_scope_end, my_parallel_data, my_task_data,
2071 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
2076 return (
int)cancelled;
2082 size_t reduce_size,
void *reduce_data,
2083 void (*reduce)(
void *,
void *)) {
2084 return __kmp_barrier_template<>(bt, gtid, is_split, reduce_size, reduce_data,
2088#if defined(KMP_GOMP_COMPAT)
2120 if (!team->
t.t_serialized) {
2170 void *itt_sync_obj = NULL;
2172 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
2177#if ((USE_ITT_BUILD && USE_ITT_NOTIFY) || defined KMP_DEBUG)
2178 int nproc = this_thr->th.th_team_nproc;
2183 team = this_thr->th.th_team;
2187 team_id = team->
t.t_id;
2188 kmp_info_t *master_thread = this_thr->th.th_team_master;
2189 if (master_thread != team->
t.t_threads[0]) {
2200 KA_TRACE(10, (
"__kmp_join_barrier: T#%d(%d:%d) arrived at join barrier\n",
2201 gtid, team_id, tid));
2206 ompt_data_t *my_task_data;
2207 ompt_data_t *my_parallel_data;
2208 void *codeptr = NULL;
2209 int ds_tid = this_thr->th.th_info.ds.ds_tid;
2213 codeptr = team->
t.ompt_team_info.master_return_address;
2214 my_task_data = OMPT_CUR_TASK_DATA(this_thr);
2215 my_parallel_data = OMPT_CUR_TEAM_DATA(this_thr);
2218 ompt_sync_region_barrier_implicit, ompt_scope_begin, my_parallel_data,
2219 my_task_data, codeptr);
2223 ompt_sync_region_barrier_implicit, ompt_scope_begin, my_parallel_data,
2224 my_task_data, codeptr);
2227 this_thr->th.ompt_thread_info.task_data = *OMPT_CUR_TASK_DATA(this_thr);
2229 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier_implicit;
2235 KA_TRACE(10, (
"__kmp_join_barrier: T#%d(%d:%d) past tasking barrier\n",
2236 gtid, team_id, tid));
2240 KA_TRACE(20, (
"__kmp_join_barrier: T#%d, old team = %d, old task_team = "
2241 "%p, th_task_team = %p\n",
2243 team->
t.t_task_team[this_thr->th.th_task_state],
2244 this_thr->th.th_task_team));
2256 this_thr->th.th_team_bt_intervals =
2257 team->
t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
2258 this_thr->th.th_team_bt_set =
2259 team->
t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
2266 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
2267 __kmp_itt_barrier_starting(gtid, itt_sync_obj);
2310#if KMP_STATS_ENABLED
2314 for (
int i = 0;
i < team->
t.t_nproc; ++
i) {
2316 if (team_thread == this_thr)
2318 team_thread->th.th_stats->setIdleFlag();
2320 team_thread->th.th_sleep_loc != NULL)
2325 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
2326 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
2329#if USE_ITT_BUILD && USE_ITT_NOTIFY
2331 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2332 __kmp_forkjoin_frames_mode &&
2333 (this_thr->th.th_teams_microtask == NULL ||
2334 this_thr->th.th_teams_size.nteams == 1) &&
2335 team->
t.t_active_level == 1) {
2339 switch (__kmp_forkjoin_frames_mode) {
2341 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
2345 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1,
2349 if (__itt_metadata_add_ptr) {
2351 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
2354 this_thr->th.th_bar_arrive_time = 0;
2355 for (
int i = 1;
i < nproc; ++
i) {
2356 delta += (cur_time - other_threads[
i]->th.th_bar_arrive_time);
2357 other_threads[
i]->th.th_bar_arrive_time = 0;
2359 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time,
2360 cur_time, delta, 0);
2362 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0,
2364 this_thr->th.th_frame_time = cur_time;
2372 if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
2373 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
2381 (
"__kmp_join_barrier: T#%d(%d:%d) says all %d team threads arrived\n",
2382 gtid, team_id, tid, nproc));
2389 (
"__kmp_join_barrier: T#%d(%d:%d) leaving\n", gtid, team_id, tid));
2399 kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL;
2401 void *itt_sync_obj = NULL;
2405 KA_TRACE(10, (
"__kmp_fork_barrier: T#%d(%d:%d) has arrived\n", gtid,
2406 (team != NULL) ? team->
t.t_id : -1, tid));
2410#if USE_ITT_BUILD && USE_ITT_NOTIFY
2411 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
2414 __kmp_itt_barrier_middle(gtid, itt_sync_obj);
2426 for (
i = 1;
i < team->
t.t_nproc; ++
i) {
2428 (
"__kmp_fork_barrier: T#%d(%d:0) checking T#%d(%d:%d) fork go "
2430 gtid, team->
t.t_id, other_threads[
i]->th.th_info.ds.ds_gtid,
2431 team->
t.t_id, other_threads[
i]->th.th_info.ds.ds_tid,
2451 this_thr->th.th_team_bt_intervals =
2452 team->
t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
2453 this_thr->th.th_team_bt_set =
2454 team->
t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
2492 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
2493 int ds_tid = this_thr->th.th_info.ds.ds_tid;
2494 ompt_data_t *task_data = (team)
2495 ? OMPT_CUR_TASK_DATA(this_thr)
2496 : &(this_thr->th.ompt_thread_info.task_data);
2497 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
2499 void *codeptr = NULL;
2503 codeptr = team ? team->
t.ompt_team_info.master_return_address : NULL;
2506 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
2511 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
2517 ompt_scope_end, NULL, task_data, 0, ds_tid,
2518 ompt_task_implicit);
2525 this_thr->th.th_task_team = NULL;
2527#if USE_ITT_BUILD && USE_ITT_NOTIFY
2528 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
2532 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
2536 KA_TRACE(10, (
"__kmp_fork_barrier: T#%d is leaving early\n", gtid));
2548#if KMP_BARRIER_ICV_PULL
2561 (
"__kmp_fork_barrier: T#%d(%d) is PULLing ICVs\n", gtid, tid));
2564 copy_icvs(&team->
t.t_implicit_task_taskdata[tid].td_icvs,
2565 &team->
t.t_threads[0]
2576#if KMP_AFFINITY_SUPPORTED
2580 if (__kmp_affinity.type == affinity_balanced && team->
t.t_size_changed) {
2581 __kmp_balanced_affinity(this_thr, team->
t.t_nproc);
2584 if (this_thr->th.th_new_place == this_thr->th.th_current_place) {
2585 KA_TRACE(100, (
"__kmp_fork_barrier: T#%d already in correct place %d\n",
2587 this_thr->th.th_current_place));
2589 __kmp_affinity_bind_place(gtid);
2595 if (team->
t.t_display_affinity
2597 || (__kmp_affinity.type == affinity_balanced && team->
t.t_size_changed)
2602 this_thr->th.th_prev_num_threads = team->
t.t_nproc;
2603 this_thr->th.th_prev_level = team->
t.t_level;
2609#if USE_ITT_BUILD && USE_ITT_NOTIFY
2610 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
2614 __kmp_itt_barrier_finished(gtid, itt_sync_obj);
2618 KA_TRACE(10, (
"__kmp_fork_barrier: T#%d(%d:%d) is leaving\n", gtid,
2619 team->
t.t_id, tid));
2632#if KMP_BARRIER_ICV_PULL
2641 KF_TRACE(10, (
"__kmp_setup_icv_copy: PULL: T#%d this_thread=%p team=%p\n", 0,
2642 team->
t.t_threads[0], team));
2643#elif KMP_BARRIER_ICV_PUSH
2646 KF_TRACE(10, (
"__kmp_setup_icv_copy: PUSH: T#%d this_thread=%p team=%p\n", 0,
2647 team->
t.t_threads[0], team));
2654 for (
int f = 1;
f < new_nproc; ++
f) {
2656 KF_TRACE(10, (
"__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
2657 f, team->
t.t_threads[
f], team));
2660 KF_TRACE(10, (
"__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n",
2661 f, team->
t.t_threads[
f], team));
size_t KMP_ALIGN_CACHE gos_per_group
size_t KMP_ALIGN_CACHE num_groups
size_t KMP_ALIGN_CACHE threads_per_group
size_t KMP_ALIGN_CACHE num_gos
size_t KMP_ALIGN_CACHE threads_per_go
size_t KMP_ALIGN_CACHE num_threads
size_t KMP_ALIGN_CACHE max_threads
flags_s * flags[MAX_ITERS]
bool KMP_ALIGN_CACHE fix_threads_per_go
bool wait(kmp_info_t *this_thr, int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj))
bool wait(kmp_info_t *this_thr, int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj))
bool wait(kmp_info_t *this_thr, int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj))
void set_waiter(kmp_info_t *thr)
int get_level(kmp_hw_t type) const
int get_count(int level) const
int calculate_ratio(int level1, int level2) const
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long value
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d int
#define KMP_INTERNAL_MALLOC(sz)
kmp_global_t __kmp_global
void __kmp_teams_master(int gtid)
#define KMP_MAX_BLOCKTIME
#define KMP_INTERNAL_REALLOC(p, sz)
void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team)
#define KMP_NOT_SAFE_TO_REAP
static kmp_team_t * __kmp_team_from_gtid(int gtid)
kmp_bar_pat_e __kmp_barrier_gather_pattern[bs_last_barrier]
kmp_tasking_mode_t __kmp_tasking_mode
void __kmp_abort_thread(void)
void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar)
int __kmp_omp_cancellation
#define KMP_BARRIER_UNUSED_STATE
int __kmp_barrier_gomp_cancel(int gtid)
#define KMP_BARRIER_SLEEP_STATE
static int __kmp_tid_from_gtid(int gtid)
#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr)
#define KMP_CHECK_UPDATE(a, b)
#define KMP_MASTER_TID(tid)
kmp_uint32 __kmp_barrier_release_branch_bits[bs_last_barrier]
#define KMP_BARRIER_OWN_FLAG
void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr, kmp_team_t *team, int tid, int set_curr_task)
static void copy_icvs(kmp_internal_control_t *dst, kmp_internal_control_t *src)
#define KMP_TASKING_ENABLED(task_team)
kmp_info_t ** __kmp_threads
#define KMP_BARRIER_PARENT_FLAG
#define KMP_MASTER_GTID(gtid)
volatile int __kmp_init_parallel
#define __kmp_allocate(size)
void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid)
void __kmp_aux_display_affinity(int gtid, const char *format)
#define KMP_INIT_BARRIER_STATE
kmp_uint32 __kmp_barrier_gather_branch_bits[bs_last_barrier]
#define KMP_BARRIER_NOT_WAITING
static int __kmp_gtid_from_tid(int tid, const kmp_team_t *team)
#define KMP_BARRIER_SWITCHING
int __kmp_display_affinity
#define KMP_BLOCKTIME_INTERVAL(team, tid)
void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team)
int __kmp_atomic_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid, kmp_atomic_flag_64< C, S > *flag, int final_spin, int *thread_finished, kmp_int32 is_constrained)
#define KMP_BARRIER_STATE_BUMP
void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64< C, S > *flag)
static int __kmp_gtid_from_thread(const kmp_info_t *thr)
static void __kmp_type_convert(T1 src, T2 *dest)
struct KMP_ALIGN_CACHE kmp_bstate kmp_bstate_t
kmp_bar_pat_e __kmp_barrier_release_pattern[bs_last_barrier]
union KMP_ALIGN_CACHE kmp_info kmp_info_t
void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team, int wait=1)
kmp_topology_t * __kmp_topology
static bool __kmp_linear_barrier_gather_template(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, void(*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj))
static void __kmp_dist_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj))
static void __kmp_tree_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj))
#define ngo_store_icvs(dst, src)
static bool __kmp_linear_barrier_release_template(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj))
#define ngo_store_go(dst, src)
int __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size, void *reduce_data, void(*reduce)(void *, void *))
void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, kmp_internal_control_t *new_icvs, ident_t *loc)
static void __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, void(*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj))
void __kmp_join_barrier(int gtid)
static void __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, void(*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj))
void __kmp_end_split_barrier(enum barrier_type bt, int gtid)
static bool __kmp_linear_barrier_release_cancellable(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj))
static bool __kmp_linear_barrier_gather_cancellable(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, void(*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj))
static int __kmp_barrier_template(enum barrier_type bt, int gtid, int is_split, size_t reduce_size, void *reduce_data, void(*reduce)(void *, void *))
static void __kmp_linear_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, void(*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj))
void __kmp_print_structure(void)
static void __kmp_hyper_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj))
void __kmp_dist_barrier_wakeup(enum barrier_type bt, kmp_team_t *team, size_t start, size_t stop, size_t inc, size_t tid)
static void __kmp_hierarchical_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj))
static bool __kmp_init_hierarchical_barrier_thread(enum barrier_type bt, kmp_bstate_t *thr_bar, kmp_uint32 nproc, int gtid, int tid, kmp_team_t *team)
void __kmp_fork_barrier(int gtid, int tid)
static void __kmp_dist_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, void(*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj))
static void __kmp_hierarchical_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, void(*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj))
static void __kmp_linear_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj))
#define KMP_OPTIMIZE_FOR_REDUCTIONS
#define KMP_DEBUG_ASSERT(cond)
unsigned long long kmp_uint64
static volatile kmp_i18n_cat_status_t status
#define USE_ITT_BUILD_ARG(x)
void(* microtask_t)(int *gtid, int *npr,...)
#define KMP_TEST_THEN_AND64(p, v)
#define KMP_CACHE_PREFETCH(ADDR)
#define KMP_ATOMIC_ST_RLX(p, v)
#define KMP_ATOMIC_LD_RLX(p)
#define KMP_AFFINITY_SUPPORTED
#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv)
Functions for collecting statistics.
#define KMP_SET_THREAD_STATE_BLOCK(state_name)
#define KMP_TIME_PARTITIONED_BLOCK(name)
#define KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(n)
static void __kmp_null_resume_wrapper(kmp_info_t *thr)
ompt_callbacks_active_t ompt_enabled
ompt_callbacks_internal_t ompt_callbacks
ompt_sync_region_t __ompt_get_barrier_kind(enum barrier_type bt, kmp_info_t *thr)
#define OMPT_REDUCTION_BEGIN
#define OMPT_REDUCTION_DECL(this_thr, gtid)
#define OMPT_REDUCTION_END
is_cancellable & operator=(bool b)
is_cancellable & operator=(bool b)
KMP_ALIGN_CACHE volatile kmp_uint32 tt_active