LLVM OpenMP 20.0.0git
kmp.h
Go to the documentation of this file.
1/*! \file */
2/*
3 * kmp.h -- KPTS runtime header file.
4 */
5
6//===----------------------------------------------------------------------===//
7//
8// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9// See https://llvm.org/LICENSE.txt for license information.
10// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef KMP_H
15#define KMP_H
16
17#include "kmp_config.h"
18
19/* #define BUILD_PARALLEL_ORDERED 1 */
20
21/* This fix replaces gettimeofday with clock_gettime for better scalability on
22 the Altix. Requires user code to be linked with -lrt. */
23//#define FIX_SGI_CLOCK
24
25/* Defines for OpenMP 3.0 tasking and auto scheduling */
26
27#ifndef KMP_STATIC_STEAL_ENABLED
28#define KMP_STATIC_STEAL_ENABLED 1
29#endif
30#define KMP_WEIGHTED_ITERATIONS_SUPPORTED \
31 (KMP_AFFINITY_SUPPORTED && KMP_STATIC_STEAL_ENABLED && \
32 (KMP_ARCH_X86 || KMP_ARCH_X86_64))
33
34#define TASK_CURRENT_NOT_QUEUED 0
35#define TASK_CURRENT_QUEUED 1
36
37#ifdef BUILD_TIED_TASK_STACK
38#define TASK_STACK_EMPTY 0 // entries when the stack is empty
39#define TASK_STACK_BLOCK_BITS 5 // Used in TASK_STACK_SIZE and TASK_STACK_MASK
40// Number of entries in each task stack array
41#define TASK_STACK_BLOCK_SIZE (1 << TASK_STACK_BLOCK_BITS)
42// Mask for determining index into stack block
43#define TASK_STACK_INDEX_MASK (TASK_STACK_BLOCK_SIZE - 1)
44#endif // BUILD_TIED_TASK_STACK
45
46#define TASK_NOT_PUSHED 1
47#define TASK_SUCCESSFULLY_PUSHED 0
48#define TASK_TIED 1
49#define TASK_UNTIED 0
50#define TASK_EXPLICIT 1
51#define TASK_IMPLICIT 0
52#define TASK_PROXY 1
53#define TASK_FULL 0
54#define TASK_DETACHABLE 1
55#define TASK_UNDETACHABLE 0
56
57#define KMP_CANCEL_THREADS
58#define KMP_THREAD_ATTR
59
60// Android does not have pthread_cancel. Undefine KMP_CANCEL_THREADS if being
61// built on Android
62#if defined(__ANDROID__)
63#undef KMP_CANCEL_THREADS
64#endif
65
66// Some WASI targets (e.g., wasm32-wasi-threads) do not support thread
67// cancellation.
68#if KMP_OS_WASI
69#undef KMP_CANCEL_THREADS
70#endif
71
72#if !KMP_OS_WASI
73#include <signal.h>
74#endif
75#include <stdarg.h>
76#include <stddef.h>
77#include <stdio.h>
78#include <stdlib.h>
79#include <string.h>
80#include <limits>
81#include <type_traits>
82/* include <ctype.h> don't use; problems with /MD on Windows* OS NT due to bad
83 Microsoft library. Some macros provided below to replace these functions */
84#ifndef __ABSOFT_WIN
85#include <sys/types.h>
86#endif
87#include <limits.h>
88#include <time.h>
89
90#include <errno.h>
91
92#include "kmp_os.h"
93
94#include "kmp_safe_c_api.h"
95
96#if KMP_STATS_ENABLED
97class kmp_stats_list;
98#endif
99
100#if KMP_USE_HIER_SCHED
101// Only include hierarchical scheduling if affinity is supported
102#undef KMP_USE_HIER_SCHED
103#define KMP_USE_HIER_SCHED KMP_AFFINITY_SUPPORTED
104#endif
105
106// OMPD_SKIP_HWLOC used in libompd/omp-icv.cpp to avoid OMPD depending on hwloc
107#if KMP_USE_HWLOC && KMP_AFFINITY_SUPPORTED && !defined(OMPD_SKIP_HWLOC)
108#include "hwloc.h"
109#ifndef HWLOC_OBJ_NUMANODE
110#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
111#endif
112#ifndef HWLOC_OBJ_PACKAGE
113#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET
114#endif
115#endif
116
117#if KMP_ARCH_X86 || KMP_ARCH_X86_64
118#include <xmmintrin.h>
119#endif
120
121// The below has to be defined before including "kmp_barrier.h".
122#define KMP_INTERNAL_MALLOC(sz) malloc(sz)
123#define KMP_INTERNAL_FREE(p) free(p)
124#define KMP_INTERNAL_REALLOC(p, sz) realloc((p), (sz))
125#define KMP_INTERNAL_CALLOC(n, sz) calloc((n), (sz))
126
127#include "kmp_debug.h"
128#include "kmp_lock.h"
129#include "kmp_version.h"
130#include "kmp_barrier.h"
131#if USE_DEBUGGER
132#include "kmp_debugger.h"
133#endif
134#include "kmp_i18n.h"
135
136#define KMP_HANDLE_SIGNALS ((KMP_OS_UNIX && !KMP_OS_WASI) || KMP_OS_WINDOWS)
137
138#include "kmp_wrapper_malloc.h"
139#if KMP_OS_UNIX
140#include <unistd.h>
141#if !defined NSIG && defined _NSIG
142#define NSIG _NSIG
143#endif
144#endif
145
146#if KMP_OS_LINUX
147#pragma weak clock_gettime
148#endif
149
150#if OMPT_SUPPORT
151#include "ompt-internal.h"
152#endif
153
154#if OMPD_SUPPORT
155#include "ompd-specific.h"
156#endif
157
158#ifndef UNLIKELY
159#define UNLIKELY(x) (x)
160#endif
161
162// Affinity format function
163#include "kmp_str.h"
164
165// 0 - no fast memory allocation, alignment: 8-byte on x86, 16-byte on x64.
166// 3 - fast allocation using sync, non-sync free lists of any size, non-self
167// free lists of limited size.
168#ifndef USE_FAST_MEMORY
169#define USE_FAST_MEMORY 3
170#endif
171
172#ifndef KMP_NESTED_HOT_TEAMS
173#define KMP_NESTED_HOT_TEAMS 0
174#define USE_NESTED_HOT_ARG(x)
175#else
176#if KMP_NESTED_HOT_TEAMS
177#define USE_NESTED_HOT_ARG(x) , x
178#else
179#define USE_NESTED_HOT_ARG(x)
180#endif
181#endif
182
183// Assume using BGET compare_exchange instruction instead of lock by default.
184#ifndef USE_CMP_XCHG_FOR_BGET
185#define USE_CMP_XCHG_FOR_BGET 1
186#endif
187
188// Test to see if queuing lock is better than bootstrap lock for bget
189// #ifndef USE_QUEUING_LOCK_FOR_BGET
190// #define USE_QUEUING_LOCK_FOR_BGET
191// #endif
192
193#define KMP_NSEC_PER_SEC 1000000000L
194#define KMP_USEC_PER_SEC 1000000L
195#define KMP_NSEC_PER_USEC 1000L
196
197/*!
198@ingroup BASIC_TYPES
199@{
200*/
201
202/*!
203Values for bit flags used in the ident_t to describe the fields.
204*/
205enum {
206 /*! Use trampoline for internal microtasks */
208 /*! Use c-style ident structure */
210 /* 0x04 is no longer used */
211 /*! Entry point generated by auto-parallelization */
213 /*! Compiler generates atomic reduction option for kmpc_reduce* */
215 /*! To mark a 'barrier' directive in user code */
217 /*! To Mark implicit barriers. */
222
225
226 /*! To mark a static loop in OMPT callbacks */
228 /*! To mark a sections directive in OMPT callbacks */
230 /*! To mark a distribute construct in OMPT callbacks */
232 /*! Atomic hint; bottom four bits as omp_sync_hint_t. Top four reserved and
233 not currently used. If one day we need more bits, then we can use
234 an invalid combination of hints to mean that another, larger field
235 should be used in a different flag. */
243
244/*!
245 * The ident structure that describes a source location.
246 */
247typedef struct ident {
248 kmp_int32 reserved_1; /**< might be used in Fortran; see above */
249 kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC
250 identifies this union member */
251 kmp_int32 reserved_2; /**< not really used in Fortran any more; see above */
252#if USE_ITT_BUILD
253/* but currently used for storing region-specific ITT */
254/* contextual information. */
255#endif /* USE_ITT_BUILD */
256 kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for C++ */
257 char const *psource; /**< String describing the source location.
258 The string is composed of semi-colon separated fields
259 which describe the source file, the function and a pair
260 of line numbers that delimit the construct. */
261 // Returns the OpenMP version in form major*10+minor (e.g., 50 for 5.0)
263 return (((flags & KMP_IDENT_OPENMP_SPEC_VERSION_MASK) >> 24) & 0xFF);
264 }
266/*!
267@}
268*/
269
270// Some forward declarations.
271typedef union kmp_team kmp_team_t;
274typedef union kmp_team kmp_team_p;
275typedef union kmp_info kmp_info_p;
276typedef union kmp_root kmp_root_p;
277
278template <bool C = false, bool S = true> class kmp_flag_32;
279template <bool C = false, bool S = true> class kmp_flag_64;
280template <bool C = false, bool S = true> class kmp_atomic_flag_64;
281class kmp_flag_oncore;
282
283#ifdef __cplusplus
284extern "C" {
285#endif
286
287/* ------------------------------------------------------------------------ */
288
289/* Pack two 32-bit signed integers into a 64-bit signed integer */
290/* ToDo: Fix word ordering for big-endian machines. */
291#define KMP_PACK_64(HIGH_32, LOW_32) \
292 ((kmp_int64)((((kmp_uint64)(HIGH_32)) << 32) | (kmp_uint64)(LOW_32)))
293
294// Generic string manipulation macros. Assume that _x is of type char *
295#define SKIP_WS(_x) \
296 { \
297 while (*(_x) == ' ' || *(_x) == '\t') \
298 (_x)++; \
299 }
300#define SKIP_DIGITS(_x) \
301 { \
302 while (*(_x) >= '0' && *(_x) <= '9') \
303 (_x)++; \
304 }
305#define SKIP_TOKEN(_x) \
306 { \
307 while ((*(_x) >= '0' && *(_x) <= '9') || (*(_x) >= 'a' && *(_x) <= 'z') || \
308 (*(_x) >= 'A' && *(_x) <= 'Z') || *(_x) == '_') \
309 (_x)++; \
310 }
311#define SKIP_TO(_x, _c) \
312 { \
313 while (*(_x) != '\0' && *(_x) != (_c)) \
314 (_x)++; \
315 }
316
317/* ------------------------------------------------------------------------ */
318
319#define KMP_MAX(x, y) ((x) > (y) ? (x) : (y))
320#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
321
322/* ------------------------------------------------------------------------ */
323/* Enumeration types */
324
329
332
335#ifdef USE_LOAD_BALANCE
336 dynamic_load_balance,
337#endif /* USE_LOAD_BALANCE */
342
343/* external schedule constants, duplicate enum omp_sched in omp.h in order to
344 * not include it here */
345#ifndef KMP_SCHED_TYPE_DEFINED
346#define KMP_SCHED_TYPE_DEFINED
347typedef enum kmp_sched {
348 kmp_sched_lower = 0, // lower and upper bounds are for routine parameter check
349 // Note: need to adjust __kmp_sch_map global array in case enum is changed
350 kmp_sched_static = 1, // mapped to kmp_sch_static_chunked (33)
351 kmp_sched_dynamic = 2, // mapped to kmp_sch_dynamic_chunked (35)
352 kmp_sched_guided = 3, // mapped to kmp_sch_guided_chunked (36)
353 kmp_sched_auto = 4, // mapped to kmp_sch_auto (38)
354 kmp_sched_upper_std = 5, // upper bound for standard schedules
355 kmp_sched_lower_ext = 100, // lower bound of Intel extension schedules
356 kmp_sched_trapezoidal = 101, // mapped to kmp_sch_trapezoidal (39)
357#if KMP_STATIC_STEAL_ENABLED
358 kmp_sched_static_steal = 102, // mapped to kmp_sch_static_steal (44)
359#endif
361 kmp_sched_default = kmp_sched_static, // default scheduling
362 kmp_sched_monotonic = 0x80000000
364#endif
365
366/*!
367 @ingroup WORK_SHARING
368 * Describes the loop schedule to be used for a parallel for loop.
369 */
371 kmp_sch_lower = 32, /**< lower bound for unordered values */
373 kmp_sch_static = 34, /**< static unspecialized */
375 kmp_sch_guided_chunked = 36, /**< guided unspecialized */
377 kmp_sch_auto = 38, /**< auto */
379
380 /* accessible only through KMP_SCHEDULE environment variable */
383 /* accessible only through KMP_SCHEDULE environment variable */
386 /* accessible only through KMP_SCHEDULE environment variable */
388
389 /* static with chunk adjustment (e.g., simd) */
391 kmp_sch_guided_simd = 46, /**< guided with chunk adjustment */
392 kmp_sch_runtime_simd = 47, /**< runtime with chunk adjustment */
393
394 /* accessible only through KMP_SCHEDULE environment variable */
395 kmp_sch_upper, /**< upper bound for unordered values */
396
397 kmp_ord_lower = 64, /**< lower bound for ordered values, must be power of 2 */
399 kmp_ord_static = 66, /**< ordered static unspecialized */
403 kmp_ord_auto = 70, /**< ordered auto */
405 kmp_ord_upper, /**< upper bound for ordered values */
406
407 /* Schedules for Distribute construct */
408 kmp_distribute_static_chunked = 91, /**< distribute static chunked */
409 kmp_distribute_static = 92, /**< distribute static unspecialized */
410
411 /* For the "nomerge" versions, kmp_dispatch_next*() will always return a
412 single iteration/chunk, even if the loop is serialized. For the schedule
413 types listed above, the entire iteration vector is returned if the loop is
414 serialized. This doesn't work for gcc/gcomp sections. */
415 kmp_nm_lower = 160, /**< lower bound for nomerge values */
416
419 kmp_nm_static = 162, /**< static unspecialized */
421 kmp_nm_guided_chunked = 164, /**< guided unspecialized */
423 kmp_nm_auto = 166, /**< auto */
425
426 /* accessible only through KMP_SCHEDULE environment variable */
429 /* accessible only through KMP_SCHEDULE environment variable */
433 172, /* accessible only through OMP_SCHEDULE environment variable */
434
436 kmp_nm_ord_static = 194, /**< ordered static unspecialized */
440 kmp_nm_ord_auto = 198, /**< auto */
442 kmp_nm_upper, /**< upper bound for nomerge values */
443
444 /* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. Since
445 we need to distinguish the three possible cases (no modifier, monotonic
446 modifier, nonmonotonic modifier), we need separate bits for each modifier.
447 The absence of monotonic does not imply nonmonotonic, especially since 4.5
448 says that the behaviour of the "no modifier" case is implementation defined
449 in 4.5, but will become "nonmonotonic" in 5.0.
450
451 Since we're passing a full 32 bit value, we can use a couple of high bits
452 for these flags; out of paranoia we avoid the sign bit.
453
454 These modifiers can be or-ed into non-static schedules by the compiler to
455 pass the additional information. They will be stripped early in the
456 processing in __kmp_dispatch_init when setting up schedules, so most of the
457 code won't ever see schedules with these bits set. */
459 (1 << 29), /**< Set if the monotonic schedule modifier was present */
461 (1 << 30), /**< Set if the nonmonotonic schedule modifier was present */
462
464 (enum sched_type)( \
466#define SCHEDULE_HAS_MONOTONIC(s) (((s)&kmp_sch_modifier_monotonic) != 0)
467#define SCHEDULE_HAS_NONMONOTONIC(s) (((s)&kmp_sch_modifier_nonmonotonic) != 0)
468#define SCHEDULE_HAS_NO_MODIFIERS(s) \
469 (((s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) == 0)
470#define SCHEDULE_GET_MODIFIERS(s) \
471 ((enum sched_type)( \
472 (s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)))
473#define SCHEDULE_SET_MODIFIERS(s, m) \
474 (s = (enum sched_type)((kmp_int32)s | (kmp_int32)m))
475#define SCHEDULE_NONMONOTONIC 0
476#define SCHEDULE_MONOTONIC 1
477
478 kmp_sch_default = kmp_sch_static /**< default scheduling algorithm */
480
481// Apply modifiers on internal kind to standard kind
482static inline void
484 enum sched_type internal_kind) {
485 if (SCHEDULE_HAS_MONOTONIC(internal_kind)) {
486 *kind = (kmp_sched_t)((int)*kind | (int)kmp_sched_monotonic);
487 }
488}
489
490// Apply modifiers on standard kind to internal kind
491static inline void
493 enum sched_type *internal_kind) {
494 if ((int)kind & (int)kmp_sched_monotonic) {
495 *internal_kind = (enum sched_type)((int)*internal_kind |
497 }
498}
499
500// Get standard schedule without modifiers
502 return (kmp_sched_t)((int)kind & ~((int)kmp_sched_monotonic));
503}
504
505/* Type to keep runtime schedule set via OMP_SCHEDULE or omp_set_schedule() */
506typedef union kmp_r_sched {
507 struct {
509 int chunk;
510 };
513
514extern enum sched_type __kmp_sch_map[]; // map OMP 3.0 schedule types with our
515// internal schedule types
516
523
524#if KMP_OS_LINUX
525enum clock_function_type {
526 clock_function_gettimeofday,
527 clock_function_clock_gettime
528};
529#endif /* KMP_OS_LINUX */
530
531#if KMP_MIC_SUPPORTED
532enum mic_type { non_mic, mic1, mic2, mic3, dummy };
533#endif
534
535// OpenMP 3.1 - Nested num threads array
536typedef struct kmp_nested_nthreads_t {
537 int *nth;
538 int size;
539 int used;
541
543
544/* -- fast reduction stuff ------------------------------------------------ */
545
546#undef KMP_FAST_REDUCTION_BARRIER
547#define KMP_FAST_REDUCTION_BARRIER 1
548
549#undef KMP_FAST_REDUCTION_CORE_DUO
550#if KMP_ARCH_X86 || KMP_ARCH_X86_64
551#define KMP_FAST_REDUCTION_CORE_DUO 1
552#endif
553
559 empty_reduce_block = (4 << 8)
561
562// Description of the packed_reduction_method variable:
563// The packed_reduction_method variable consists of two enum types variables
564// that are packed together into 0-th byte and 1-st byte:
565// 0: (packed_reduction_method & 0x000000FF) is a 'enum barrier_type' value of
566// barrier that will be used in fast reduction: bs_plain_barrier or
567// bs_reduction_barrier
568// 1: (packed_reduction_method & 0x0000FF00) is a reduction method that will
569// be used in fast reduction;
570// Reduction method is of 'enum _reduction_method' type and it's defined the way
571// so that the bits of 0-th byte are empty, so no need to execute a shift
572// instruction while packing/unpacking
573
574#if KMP_FAST_REDUCTION_BARRIER
575#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \
576 ((reduction_method) | (barrier_type))
577
578#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \
579 ((enum _reduction_method)((packed_reduction_method) & (0x0000FF00)))
580
581#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) \
582 ((enum barrier_type)((packed_reduction_method) & (0x000000FF)))
583#else
584#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \
585 (reduction_method)
586
587#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \
588 (packed_reduction_method)
589
590#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) (bs_plain_barrier)
591#endif
592
593#define TEST_REDUCTION_METHOD(packed_reduction_method, which_reduction_block) \
594 ((UNPACK_REDUCTION_METHOD(packed_reduction_method)) == \
595 (which_reduction_block))
596
597#if KMP_FAST_REDUCTION_BARRIER
598#define TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER \
599 (PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_reduction_barrier))
600
601#define TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER \
602 (PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_plain_barrier))
603#endif
604
606
607/* -- end of fast reduction stuff ----------------------------------------- */
608
609#if KMP_OS_WINDOWS
610#define USE_CBLKDATA
611#if KMP_MSVC_COMPAT
612#pragma warning(push)
613#pragma warning(disable : 271 310)
614#endif
615#include <windows.h>
616#if KMP_MSVC_COMPAT
617#pragma warning(pop)
618#endif
619#endif
620
621#if KMP_OS_UNIX
622#if !KMP_OS_WASI
623#include <dlfcn.h>
624#endif
625#include <pthread.h>
626#endif
627
628enum kmp_hw_t : int {
644
645typedef enum kmp_hw_core_type_t {
647#if KMP_ARCH_X86 || KMP_ARCH_X86_64
648 KMP_HW_CORE_TYPE_ATOM = 0x20,
649 KMP_HW_CORE_TYPE_CORE = 0x40,
651#else
653#endif
655
656#define KMP_HW_MAX_NUM_CORE_EFFS 8
657
658#define KMP_DEBUG_ASSERT_VALID_HW_TYPE(type) \
659 KMP_DEBUG_ASSERT(type >= (kmp_hw_t)0 && type < KMP_HW_LAST)
660#define KMP_ASSERT_VALID_HW_TYPE(type) \
661 KMP_ASSERT(type >= (kmp_hw_t)0 && type < KMP_HW_LAST)
662
663#define KMP_FOREACH_HW_TYPE(type) \
664 for (kmp_hw_t type = (kmp_hw_t)0; type < KMP_HW_LAST; \
665 type = (kmp_hw_t)((int)type + 1))
666
667const char *__kmp_hw_get_keyword(kmp_hw_t type, bool plural = false);
668const char *__kmp_hw_get_catalog_string(kmp_hw_t type, bool plural = false);
670
671/* Only Linux* OS and Windows* OS support thread affinity. */
672#if KMP_AFFINITY_SUPPORTED
673
674// GROUP_AFFINITY is already defined for _MSC_VER>=1600 (VS2010 and later).
675#if KMP_OS_WINDOWS
676#if _MSC_VER < 1600 && KMP_MSVC_COMPAT
677typedef struct GROUP_AFFINITY {
678 KAFFINITY Mask;
679 WORD Group;
680 WORD Reserved[3];
681} GROUP_AFFINITY;
682#endif /* _MSC_VER < 1600 */
683#if KMP_GROUP_AFFINITY
684extern int __kmp_num_proc_groups;
685#else
686static const int __kmp_num_proc_groups = 1;
687#endif /* KMP_GROUP_AFFINITY */
688typedef DWORD (*kmp_GetActiveProcessorCount_t)(WORD);
689extern kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount;
690
691typedef WORD (*kmp_GetActiveProcessorGroupCount_t)(void);
692extern kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount;
693
694typedef BOOL (*kmp_GetThreadGroupAffinity_t)(HANDLE, GROUP_AFFINITY *);
695extern kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity;
696
697typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *,
698 GROUP_AFFINITY *);
699extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity;
700#endif /* KMP_OS_WINDOWS */
701
702#if KMP_USE_HWLOC && !defined(OMPD_SKIP_HWLOC)
703extern hwloc_topology_t __kmp_hwloc_topology;
704extern int __kmp_hwloc_error;
705#endif
706
707extern size_t __kmp_affin_mask_size;
708#define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0)
709#define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0)
710#define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size)
711#define KMP_CPU_SET_ITERATE(i, mask) \
712 for (i = (mask)->begin(); (int)i != (mask)->end(); i = (mask)->next(i))
713#define KMP_CPU_SET(i, mask) (mask)->set(i)
714#define KMP_CPU_ISSET(i, mask) (mask)->is_set(i)
715#define KMP_CPU_CLR(i, mask) (mask)->clear(i)
716#define KMP_CPU_ZERO(mask) (mask)->zero()
717#define KMP_CPU_ISEMPTY(mask) (mask)->empty()
718#define KMP_CPU_COPY(dest, src) (dest)->copy(src)
719#define KMP_CPU_AND(dest, src) (dest)->bitwise_and(src)
720#define KMP_CPU_COMPLEMENT(max_bit_number, mask) (mask)->bitwise_not()
721#define KMP_CPU_UNION(dest, src) (dest)->bitwise_or(src)
722#define KMP_CPU_EQUAL(dest, src) (dest)->is_equal(src)
723#define KMP_CPU_ALLOC(ptr) (ptr = __kmp_affinity_dispatch->allocate_mask())
724#define KMP_CPU_FREE(ptr) __kmp_affinity_dispatch->deallocate_mask(ptr)
725#define KMP_CPU_ALLOC_ON_STACK(ptr) KMP_CPU_ALLOC(ptr)
726#define KMP_CPU_FREE_FROM_STACK(ptr) KMP_CPU_FREE(ptr)
727#define KMP_CPU_INTERNAL_ALLOC(ptr) KMP_CPU_ALLOC(ptr)
728#define KMP_CPU_INTERNAL_FREE(ptr) KMP_CPU_FREE(ptr)
729#define KMP_CPU_INDEX(arr, i) __kmp_affinity_dispatch->index_mask_array(arr, i)
730#define KMP_CPU_ALLOC_ARRAY(arr, n) \
731 (arr = __kmp_affinity_dispatch->allocate_mask_array(n))
732#define KMP_CPU_FREE_ARRAY(arr, n) \
733 __kmp_affinity_dispatch->deallocate_mask_array(arr)
734#define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) KMP_CPU_ALLOC_ARRAY(arr, n)
735#define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) KMP_CPU_FREE_ARRAY(arr, n)
736#define __kmp_get_system_affinity(mask, abort_bool) \
737 (mask)->get_system_affinity(abort_bool)
738#define __kmp_set_system_affinity(mask, abort_bool) \
739 (mask)->set_system_affinity(abort_bool)
740#define __kmp_get_proc_group(mask) (mask)->get_proc_group()
741
742class KMPAffinity {
743public:
744 class Mask {
745 public:
746 void *operator new(size_t n);
747 void operator delete(void *p);
748 void *operator new[](size_t n);
749 void operator delete[](void *p);
750 virtual ~Mask() {}
751 // Set bit i to 1
752 virtual void set(int i) {}
753 // Return bit i
754 virtual bool is_set(int i) const { return false; }
755 // Set bit i to 0
756 virtual void clear(int i) {}
757 // Zero out entire mask
758 virtual void zero() {}
759 // Check whether mask is empty
760 virtual bool empty() const { return true; }
761 // Copy src into this mask
762 virtual void copy(const Mask *src) {}
763 // this &= rhs
764 virtual void bitwise_and(const Mask *rhs) {}
765 // this |= rhs
766 virtual void bitwise_or(const Mask *rhs) {}
767 // this = ~this
768 virtual void bitwise_not() {}
769 // this == rhs
770 virtual bool is_equal(const Mask *rhs) const { return false; }
771 // API for iterating over an affinity mask
772 // for (int i = mask->begin(); i != mask->end(); i = mask->next(i))
773 virtual int begin() const { return 0; }
774 virtual int end() const { return 0; }
775 virtual int next(int previous) const { return 0; }
776#if KMP_OS_WINDOWS
777 virtual int set_process_affinity(bool abort_on_error) const { return -1; }
778#endif
779 // Set the system's affinity to this affinity mask's value
780 virtual int set_system_affinity(bool abort_on_error) const { return -1; }
781 // Set this affinity mask to the current system affinity
782 virtual int get_system_affinity(bool abort_on_error) { return -1; }
783 // Only 1 DWORD in the mask should have any procs set.
784 // Return the appropriate index, or -1 for an invalid mask.
785 virtual int get_proc_group() const { return -1; }
786 int get_max_cpu() const {
787 int cpu;
788 int max_cpu = -1;
789 KMP_CPU_SET_ITERATE(cpu, this) {
790 if (cpu > max_cpu)
791 max_cpu = cpu;
792 }
793 return max_cpu;
794 }
795 };
796 void *operator new(size_t n);
797 void operator delete(void *p);
798 // Need virtual destructor
799 virtual ~KMPAffinity() = default;
800 // Determine if affinity is capable
801 virtual void determine_capable(const char *env_var) {}
802 // Bind the current thread to os proc
803 virtual void bind_thread(int proc) {}
804 // Factory functions to allocate/deallocate a mask
805 virtual Mask *allocate_mask() { return nullptr; }
806 virtual void deallocate_mask(Mask *m) {}
807 virtual Mask *allocate_mask_array(int num) { return nullptr; }
808 virtual void deallocate_mask_array(Mask *m) {}
809 virtual Mask *index_mask_array(Mask *m, int index) { return nullptr; }
810 static void pick_api();
811 static void destroy_api();
812 enum api_type {
813 NATIVE_OS
814#if KMP_USE_HWLOC
815 ,
816 HWLOC
817#endif
818 };
819 virtual api_type get_api_type() const {
820 KMP_ASSERT(0);
821 return NATIVE_OS;
822 }
823
824private:
825 static bool picked_api;
826};
827
828typedef KMPAffinity::Mask kmp_affin_mask_t;
829extern KMPAffinity *__kmp_affinity_dispatch;
830
831#ifndef KMP_OS_AIX
832class kmp_affinity_raii_t {
833 kmp_affin_mask_t *mask;
834 bool restored;
835
836public:
837 kmp_affinity_raii_t(const kmp_affin_mask_t *new_mask = nullptr)
838 : mask(nullptr), restored(false) {
839 if (KMP_AFFINITY_CAPABLE()) {
840 KMP_CPU_ALLOC(mask);
841 KMP_ASSERT(mask != NULL);
842 __kmp_get_system_affinity(mask, /*abort_on_error=*/true);
843 if (new_mask)
844 __kmp_set_system_affinity(new_mask, /*abort_on_error=*/true);
845 }
846 }
847 void restore() {
848 if (mask && KMP_AFFINITY_CAPABLE() && !restored) {
849 __kmp_set_system_affinity(mask, /*abort_on_error=*/true);
850 KMP_CPU_FREE(mask);
851 }
852 restored = true;
853 }
854 ~kmp_affinity_raii_t() { restore(); }
855};
856#endif // !KMP_OS_AIX
857
858// Declare local char buffers with this size for printing debug and info
859// messages, using __kmp_affinity_print_mask().
860#define KMP_AFFIN_MASK_PRINT_LEN 1024
861
862enum affinity_type {
863 affinity_none = 0,
864 affinity_physical,
865 affinity_logical,
866 affinity_compact,
867 affinity_scatter,
868 affinity_explicit,
869 affinity_balanced,
870 affinity_disabled, // not used outsize the env var parser
871 affinity_default
872};
873
874enum affinity_top_method {
875 affinity_top_method_all = 0, // try all (supported) methods, in order
876#if KMP_ARCH_X86 || KMP_ARCH_X86_64
877 affinity_top_method_apicid,
878 affinity_top_method_x2apicid,
879 affinity_top_method_x2apicid_1f,
880#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
881 affinity_top_method_cpuinfo, // KMP_CPUINFO_FILE is usable on Windows* OS, too
882#if KMP_GROUP_AFFINITY
883 affinity_top_method_group,
884#endif /* KMP_GROUP_AFFINITY */
885 affinity_top_method_flat,
886#if KMP_USE_HWLOC
887 affinity_top_method_hwloc,
888#endif
889 affinity_top_method_default
890};
891
892#define affinity_respect_mask_default (2)
893
894typedef struct kmp_affinity_flags_t {
895 unsigned dups : 1;
896 unsigned verbose : 1;
897 unsigned warnings : 1;
898 unsigned respect : 2;
899 unsigned reset : 1;
900 unsigned initialized : 1;
901 unsigned core_types_gran : 1;
902 unsigned core_effs_gran : 1;
903 unsigned omp_places : 1;
904 unsigned reserved : 22;
905} kmp_affinity_flags_t;
906KMP_BUILD_ASSERT(sizeof(kmp_affinity_flags_t) == 4);
907
908typedef struct kmp_affinity_ids_t {
909 int os_id;
910 int ids[KMP_HW_LAST];
911} kmp_affinity_ids_t;
912
913typedef struct kmp_affinity_attrs_t {
914 int core_type : 8;
915 int core_eff : 8;
916 unsigned valid : 1;
917 unsigned reserved : 15;
918} kmp_affinity_attrs_t;
919#define KMP_AFFINITY_ATTRS_UNKNOWN \
920 { KMP_HW_CORE_TYPE_UNKNOWN, kmp_hw_attr_t::UNKNOWN_CORE_EFF, 0, 0 }
921
922typedef struct kmp_affinity_t {
923 char *proclist;
924 enum affinity_type type;
925 kmp_hw_t gran;
926 int gran_levels;
927 kmp_affinity_attrs_t core_attr_gran;
928 int compact;
929 int offset;
930 kmp_affinity_flags_t flags;
931 unsigned num_masks;
932 kmp_affin_mask_t *masks;
933 kmp_affinity_ids_t *ids;
934 kmp_affinity_attrs_t *attrs;
935 unsigned num_os_id_masks;
936 kmp_affin_mask_t *os_id_masks;
937 const char *env_var;
938} kmp_affinity_t;
939
940#define KMP_AFFINITY_INIT(env) \
941 { \
942 nullptr, affinity_default, KMP_HW_UNKNOWN, -1, KMP_AFFINITY_ATTRS_UNKNOWN, \
943 0, 0, \
944 {TRUE, FALSE, TRUE, affinity_respect_mask_default, FALSE, FALSE, \
945 FALSE, FALSE, FALSE}, \
946 0, nullptr, nullptr, nullptr, 0, nullptr, env \
947 }
948
949extern enum affinity_top_method __kmp_affinity_top_method;
950extern kmp_affinity_t __kmp_affinity;
951extern kmp_affinity_t __kmp_hh_affinity;
952extern kmp_affinity_t *__kmp_affinities[2];
953
954extern void __kmp_affinity_bind_thread(int which);
955
956extern kmp_affin_mask_t *__kmp_affin_fullMask;
957extern kmp_affin_mask_t *__kmp_affin_origMask;
958extern char *__kmp_cpuinfo_file;
959
960#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
961extern int __kmp_first_osid_with_ecore;
962#endif
963
964#endif /* KMP_AFFINITY_SUPPORTED */
965
966// This needs to be kept in sync with the values in omp.h !!!
967typedef enum kmp_proc_bind_t {
973 proc_bind_intel, // use KMP_AFFINITY interface
976
979 int size;
980 int used;
982
985
986extern int __kmp_display_affinity;
987extern char *__kmp_affinity_format;
988static const size_t KMP_AFFINITY_FORMAT_SIZE = 512;
989#if OMPT_SUPPORT
990extern int __kmp_tool;
991extern char *__kmp_tool_libraries;
992#endif // OMPT_SUPPORT
993
994#if KMP_AFFINITY_SUPPORTED
995#define KMP_PLACE_ALL (-1)
996#define KMP_PLACE_UNDEFINED (-2)
997// Is KMP_AFFINITY is being used instead of OMP_PROC_BIND/OMP_PLACES?
998#define KMP_AFFINITY_NON_PROC_BIND \
999 ((__kmp_nested_proc_bind.bind_types[0] == proc_bind_false || \
1000 __kmp_nested_proc_bind.bind_types[0] == proc_bind_intel) && \
1001 (__kmp_affinity.num_masks > 0 || __kmp_affinity.type == affinity_balanced))
1002#endif /* KMP_AFFINITY_SUPPORTED */
1003
1004extern int __kmp_affinity_num_places;
1005
1006typedef enum kmp_cancel_kind_t {
1013
1014// KMP_HW_SUBSET support:
1015typedef struct kmp_hws_item {
1016 int num;
1019
1026extern int __kmp_hws_requested;
1027extern int __kmp_hws_abs_flag; // absolute or per-item number requested
1028
1029/* ------------------------------------------------------------------------ */
1030
1031#define KMP_PAD(type, sz) \
1032 (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
1033
1034// We need to avoid using -1 as a GTID as +1 is added to the gtid
1035// when storing it in a lock, and the value 0 is reserved.
1036#define KMP_GTID_DNE (-2) /* Does not exist */
1037#define KMP_GTID_SHUTDOWN (-3) /* Library is shutting down */
1038#define KMP_GTID_MONITOR (-4) /* Monitor thread ID */
1039#define KMP_GTID_UNKNOWN (-5) /* Is not known */
1040#define KMP_GTID_MIN (-6) /* Minimal gtid for low bound check in DEBUG */
1041
1042/* OpenMP 5.0 Memory Management support */
1043
1044#ifndef __OMP_H
1045// Duplicate type definitions from omp.h
1046typedef uintptr_t omp_uintptr_t;
1047
1048typedef enum {
1058
1059typedef enum {
1080#define omp_atv_default ((omp_uintptr_t)-1)
1081
1091
1092typedef struct {
1096
1112
1113// end of duplicate type definitions from omp.h
1114#endif
1115
1116extern int __kmp_memkind_available;
1117
1119
1120typedef struct kmp_allocator_t {
1122 void **memkind; // pointer to memkind
1130
1133 int ntraits,
1134 omp_alloctrait_t traits[]);
1135extern void __kmpc_destroy_allocator(int gtid, omp_allocator_handle_t al);
1138// external interfaces, may be used by compiler
1139extern void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
1140extern void *__kmpc_aligned_alloc(int gtid, size_t align, size_t sz,
1142extern void *__kmpc_calloc(int gtid, size_t nmemb, size_t sz,
1144extern void *__kmpc_realloc(int gtid, void *ptr, size_t sz,
1146 omp_allocator_handle_t free_al);
1147extern void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
1148// internal interfaces, contain real implementation
1149extern void *__kmp_alloc(int gtid, size_t align, size_t sz,
1151extern void *__kmp_calloc(int gtid, size_t align, size_t nmemb, size_t sz,
1153extern void *__kmp_realloc(int gtid, void *ptr, size_t sz,
1155 omp_allocator_handle_t free_al);
1156extern void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
1157
1158extern void __kmp_init_memkind();
1159extern void __kmp_fini_memkind();
1160extern void __kmp_init_target_mem();
1161
1162/* ------------------------------------------------------------------------ */
1163
1164#if ENABLE_LIBOMPTARGET
1165extern void __kmp_init_target_task();
1166#endif
1167
1168/* ------------------------------------------------------------------------ */
1169
1170#define KMP_UINT64_MAX \
1171 (~((kmp_uint64)1 << ((sizeof(kmp_uint64) * (1 << 3)) - 1)))
1172
1173#define KMP_MIN_NTH 1
1174
1175#ifndef KMP_MAX_NTH
1176#if defined(PTHREAD_THREADS_MAX) && PTHREAD_THREADS_MAX < INT_MAX
1177#define KMP_MAX_NTH PTHREAD_THREADS_MAX
1178#else
1179#ifdef __ve__
1180// VE's pthread supports only up to 64 threads per a VE process.
1181// Please check p. 14 of following documentation for more details.
1182// https://sxauroratsubasa.sakura.ne.jp/documents/veos/en/VEOS_high_level_design.pdf
1183#define KMP_MAX_NTH 64
1184#else
1185#define KMP_MAX_NTH INT_MAX
1186#endif
1187#endif
1188#endif /* KMP_MAX_NTH */
1189
1190#ifdef PTHREAD_STACK_MIN
1191#define KMP_MIN_STKSIZE ((size_t)PTHREAD_STACK_MIN)
1192#else
1193#define KMP_MIN_STKSIZE ((size_t)(32 * 1024))
1194#endif
1195
1196#if KMP_OS_AIX && KMP_ARCH_PPC
1197#define KMP_MAX_STKSIZE 0x10000000 /* 256Mb max size on 32-bit AIX */
1198#else
1199#define KMP_MAX_STKSIZE (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
1200#endif
1201
1202#if KMP_ARCH_X86
1203#define KMP_DEFAULT_STKSIZE ((size_t)(2 * 1024 * 1024))
1204#elif KMP_ARCH_X86_64
1205#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))
1206#define KMP_BACKUP_STKSIZE ((size_t)(2 * 1024 * 1024))
1207#elif KMP_ARCH_VE
1208// Minimum stack size for pthread for VE is 4MB.
1209// https://www.hpc.nec/documents/veos/en/glibc/Difference_Points_glibc.htm
1210#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))
1211#elif KMP_OS_AIX
1212// The default stack size for worker threads on AIX is 4MB.
1213#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))
1214#else
1215#define KMP_DEFAULT_STKSIZE ((size_t)(1024 * 1024))
1216#endif
1217
1218#define KMP_DEFAULT_MALLOC_POOL_INCR ((size_t)(1024 * 1024))
1219#define KMP_MIN_MALLOC_POOL_INCR ((size_t)(4 * 1024))
1220#define KMP_MAX_MALLOC_POOL_INCR \
1221 (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
1222
1223#define KMP_MIN_STKOFFSET (0)
1224#define KMP_MAX_STKOFFSET KMP_MAX_STKSIZE
1225#if KMP_OS_DARWIN
1226#define KMP_DEFAULT_STKOFFSET KMP_MIN_STKOFFSET
1227#else
1228#define KMP_DEFAULT_STKOFFSET CACHE_LINE
1229#endif
1230
1231#define KMP_MIN_STKPADDING (0)
1232#define KMP_MAX_STKPADDING (2 * 1024 * 1024)
1233
1234#define KMP_BLOCKTIME_MULTIPLIER \
1235 (1000000) /* number of blocktime units per second */
1236#define KMP_MIN_BLOCKTIME (0)
1237#define KMP_MAX_BLOCKTIME \
1238 (INT_MAX) /* Must be this for "infinite" setting the work */
1239
1240/* __kmp_blocktime is in microseconds */
1241#define KMP_DEFAULT_BLOCKTIME (__kmp_is_hybrid_cpu() ? (0) : (200000))
1242
1243#if KMP_USE_MONITOR
1244#define KMP_DEFAULT_MONITOR_STKSIZE ((size_t)(64 * 1024))
1245#define KMP_MIN_MONITOR_WAKEUPS (1) // min times monitor wakes up per second
1246#define KMP_MAX_MONITOR_WAKEUPS (1000) // max times monitor can wake up per sec
1247
1248/* Calculate new number of monitor wakeups for a specific block time based on
1249 previous monitor_wakeups. Only allow increasing number of wakeups */
1250#define KMP_WAKEUPS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
1251 (((blocktime) == KMP_MAX_BLOCKTIME) ? (monitor_wakeups) \
1252 : ((blocktime) == KMP_MIN_BLOCKTIME) ? KMP_MAX_MONITOR_WAKEUPS \
1253 : ((monitor_wakeups) > (KMP_BLOCKTIME_MULTIPLIER / (blocktime))) \
1254 ? (monitor_wakeups) \
1255 : (KMP_BLOCKTIME_MULTIPLIER) / (blocktime))
1256
1257/* Calculate number of intervals for a specific block time based on
1258 monitor_wakeups */
1259#define KMP_INTERVALS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \
1260 (((blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1) / \
1261 (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)))
1262#else
1263#define KMP_BLOCKTIME(team, tid) \
1264 (get__bt_set(team, tid) ? get__blocktime(team, tid) : __kmp_dflt_blocktime)
1265#if KMP_OS_UNIX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1266// HW TSC is used to reduce overhead (clock tick instead of nanosecond).
1269#if KMP_COMPILER_ICC || KMP_COMPILER_ICX
1270#define KMP_NOW() ((kmp_uint64)_rdtsc())
1271#else
1272#define KMP_NOW() __kmp_hardware_timestamp()
1273#endif
1274#define KMP_BLOCKTIME_INTERVAL(team, tid) \
1275 ((kmp_uint64)KMP_BLOCKTIME(team, tid) * __kmp_ticks_per_usec)
1276#define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())
1277#else
1278// System time is retrieved sporadically while blocking.
1279extern kmp_uint64 __kmp_now_nsec();
1280#define KMP_NOW() __kmp_now_nsec()
1281#define KMP_BLOCKTIME_INTERVAL(team, tid) \
1282 ((kmp_uint64)KMP_BLOCKTIME(team, tid) * (kmp_uint64)KMP_NSEC_PER_USEC)
1283#define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
1284#endif
1285#endif // KMP_USE_MONITOR
1286
1287#define KMP_MIN_STATSCOLS 40
1288#define KMP_MAX_STATSCOLS 4096
1289#define KMP_DEFAULT_STATSCOLS 80
1290
1291#define KMP_MIN_INTERVAL 0
1292#define KMP_MAX_INTERVAL (INT_MAX - 1)
1293#define KMP_DEFAULT_INTERVAL 0
1294
1295#define KMP_MIN_CHUNK 1
1296#define KMP_MAX_CHUNK (INT_MAX - 1)
1297#define KMP_DEFAULT_CHUNK 1
1298
1299#define KMP_MIN_DISP_NUM_BUFF 1
1300#define KMP_DFLT_DISP_NUM_BUFF 7
1301#define KMP_MAX_DISP_NUM_BUFF 4096
1302
1303#define KMP_MAX_ORDERED 8
1304
1305#define KMP_MAX_FIELDS 32
1306
1307#define KMP_MAX_BRANCH_BITS 31
1308
1309#define KMP_MAX_ACTIVE_LEVELS_LIMIT INT_MAX
1310
1311#define KMP_MAX_DEFAULT_DEVICE_LIMIT INT_MAX
1312
1313#define KMP_MAX_TASK_PRIORITY_LIMIT INT_MAX
1314
1315/* Minimum number of threads before switch to TLS gtid (experimentally
1316 determined) */
1317/* josh TODO: what about OS X* tuning? */
1318#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1319#define KMP_TLS_GTID_MIN 5
1320#else
1321#define KMP_TLS_GTID_MIN INT_MAX
1322#endif
1323
1324#define KMP_MASTER_TID(tid) (0 == (tid))
1325#define KMP_WORKER_TID(tid) (0 != (tid))
1326
1327#define KMP_MASTER_GTID(gtid) (0 == __kmp_tid_from_gtid((gtid)))
1328#define KMP_WORKER_GTID(gtid) (0 != __kmp_tid_from_gtid((gtid)))
1329#define KMP_INITIAL_GTID(gtid) (0 == (gtid))
1330
1331#ifndef TRUE
1332#define FALSE 0
1333#define TRUE (!FALSE)
1334#endif
1335
1336/* NOTE: all of the following constants must be even */
1337
1338#if KMP_OS_WINDOWS
1339#define KMP_INIT_WAIT 64U /* initial number of spin-tests */
1340#define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */
1341#elif KMP_OS_LINUX
1342#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1343#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1344#elif KMP_OS_DARWIN
1345/* TODO: tune for KMP_OS_DARWIN */
1346#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1347#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1348#elif KMP_OS_DRAGONFLY
1349/* TODO: tune for KMP_OS_DRAGONFLY */
1350#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1351#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1352#elif KMP_OS_FREEBSD
1353/* TODO: tune for KMP_OS_FREEBSD */
1354#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1355#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1356#elif KMP_OS_NETBSD
1357/* TODO: tune for KMP_OS_NETBSD */
1358#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1359#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1360#elif KMP_OS_OPENBSD
1361/* TODO: tune for KMP_OS_OPENBSD */
1362#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1363#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1364#elif KMP_OS_HURD
1365/* TODO: tune for KMP_OS_HURD */
1366#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1367#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1368#elif KMP_OS_SOLARIS
1369/* TODO: tune for KMP_OS_SOLARIS */
1370#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1371#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1372#elif KMP_OS_WASI
1373/* TODO: tune for KMP_OS_WASI */
1374#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1375#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1376#elif KMP_OS_AIX
1377/* TODO: tune for KMP_OS_AIX */
1378#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */
1379#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1380#endif
1381
1382#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1383typedef struct kmp_cpuid {
1384 kmp_uint32 eax;
1385 kmp_uint32 ebx;
1386 kmp_uint32 ecx;
1387 kmp_uint32 edx;
1388} kmp_cpuid_t;
1389
1390typedef struct kmp_cpuinfo_flags_t {
1391 unsigned sse2 : 1; // 0 if SSE2 instructions are not supported, 1 otherwise.
1392 unsigned rtm : 1; // 0 if RTM instructions are not supported, 1 otherwise.
1393 unsigned hybrid : 1;
1394 unsigned reserved : 29; // Ensure size of 32 bits
1395} kmp_cpuinfo_flags_t;
1396
1397typedef struct kmp_cpuinfo {
1398 int initialized; // If 0, other fields are not initialized.
1399 int signature; // CPUID(1).EAX
1400 int family; // CPUID(1).EAX[27:20]+CPUID(1).EAX[11:8] (Extended Family+Family)
1401 int model; // ( CPUID(1).EAX[19:16] << 4 ) + CPUID(1).EAX[7:4] ( ( Extended
1402 // Model << 4 ) + Model)
1403 int stepping; // CPUID(1).EAX[3:0] ( Stepping )
1404 kmp_cpuinfo_flags_t flags;
1405 int apic_id;
1406 kmp_uint64 frequency; // Nominal CPU frequency in Hz.
1407 char name[3 * sizeof(kmp_cpuid_t)]; // CPUID(0x80000002,0x80000003,0x80000004)
1408} kmp_cpuinfo_t;
1409
1410extern void __kmp_query_cpuid(kmp_cpuinfo_t *p);
1411
1412#if KMP_OS_UNIX
1413// subleaf is only needed for cache and topology discovery and can be set to
1414// zero in most cases
1415static inline void __kmp_x86_cpuid(int leaf, int subleaf, struct kmp_cpuid *p) {
1416 __asm__ __volatile__("cpuid"
1417 : "=a"(p->eax), "=b"(p->ebx), "=c"(p->ecx), "=d"(p->edx)
1418 : "a"(leaf), "c"(subleaf));
1419}
1420// Load p into FPU control word
1421static inline void __kmp_load_x87_fpu_control_word(const kmp_int16 *p) {
1422 __asm__ __volatile__("fldcw %0" : : "m"(*p));
1423}
1424// Store FPU control word into p
1425static inline void __kmp_store_x87_fpu_control_word(kmp_int16 *p) {
1426 __asm__ __volatile__("fstcw %0" : "=m"(*p));
1427}
1428static inline void __kmp_clear_x87_fpu_status_word() {
1429#if KMP_MIC
1430 // 32-bit protected mode x87 FPU state
1431 struct x87_fpu_state {
1432 unsigned cw;
1433 unsigned sw;
1434 unsigned tw;
1435 unsigned fip;
1436 unsigned fips;
1437 unsigned fdp;
1438 unsigned fds;
1439 };
1440 struct x87_fpu_state fpu_state = {0, 0, 0, 0, 0, 0, 0};
1441 __asm__ __volatile__("fstenv %0\n\t" // store FP env
1442 "andw $0x7f00, %1\n\t" // clear 0-7,15 bits of FP SW
1443 "fldenv %0\n\t" // load FP env back
1444 : "+m"(fpu_state), "+m"(fpu_state.sw));
1445#else
1446 __asm__ __volatile__("fnclex");
1447#endif // KMP_MIC
1448}
1449#if __SSE__
1450static inline void __kmp_load_mxcsr(const kmp_uint32 *p) { _mm_setcsr(*p); }
1451static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = _mm_getcsr(); }
1452#else
1453static inline void __kmp_load_mxcsr(const kmp_uint32 *p) {}
1454static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = 0; }
1455#endif
1456#else
1457// Windows still has these as external functions in assembly file
1458extern void __kmp_x86_cpuid(int mode, int mode2, struct kmp_cpuid *p);
1459extern void __kmp_load_x87_fpu_control_word(const kmp_int16 *p);
1460extern void __kmp_store_x87_fpu_control_word(kmp_int16 *p);
1461extern void __kmp_clear_x87_fpu_status_word();
1462static inline void __kmp_load_mxcsr(const kmp_uint32 *p) { _mm_setcsr(*p); }
1463static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = _mm_getcsr(); }
1464#endif // KMP_OS_UNIX
1465
1466#define KMP_X86_MXCSR_MASK 0xffffffc0 /* ignore status flags (6 lsb) */
1467
1468// User-level Monitor/Mwait
1469#if KMP_HAVE_UMWAIT
1470// We always try for UMWAIT first
1471#if KMP_HAVE_WAITPKG_INTRINSICS
1472#if KMP_HAVE_IMMINTRIN_H
1473#include <immintrin.h>
1474#elif KMP_HAVE_INTRIN_H
1475#include <intrin.h>
1476#endif
1477#endif // KMP_HAVE_WAITPKG_INTRINSICS
1478
1480static inline int __kmp_tpause(uint32_t hint, uint64_t counter) {
1481#if !KMP_HAVE_WAITPKG_INTRINSICS
1482 uint32_t timeHi = uint32_t(counter >> 32);
1483 uint32_t timeLo = uint32_t(counter & 0xffffffff);
1484 char flag;
1485 __asm__ volatile("#tpause\n.byte 0x66, 0x0F, 0xAE, 0xF1\n"
1486 "setb %0"
1487 // The "=q" restraint means any register accessible as rl
1488 // in 32-bit mode: a, b, c, and d;
1489 // in 64-bit mode: any integer register
1490 : "=q"(flag)
1491 : "a"(timeLo), "d"(timeHi), "c"(hint)
1492 :);
1493 return flag;
1494#else
1495 return _tpause(hint, counter);
1496#endif
1497}
1499static inline void __kmp_umonitor(void *cacheline) {
1500#if !KMP_HAVE_WAITPKG_INTRINSICS
1501 __asm__ volatile("# umonitor\n.byte 0xF3, 0x0F, 0xAE, 0x01 "
1502 :
1503 : "a"(cacheline)
1504 :);
1505#else
1506 _umonitor(cacheline);
1507#endif
1508}
1510static inline int __kmp_umwait(uint32_t hint, uint64_t counter) {
1511#if !KMP_HAVE_WAITPKG_INTRINSICS
1512 uint32_t timeHi = uint32_t(counter >> 32);
1513 uint32_t timeLo = uint32_t(counter & 0xffffffff);
1514 char flag;
1515 __asm__ volatile("#umwait\n.byte 0xF2, 0x0F, 0xAE, 0xF1\n"
1516 "setb %0"
1517 // The "=q" restraint means any register accessible as rl
1518 // in 32-bit mode: a, b, c, and d;
1519 // in 64-bit mode: any integer register
1520 : "=q"(flag)
1521 : "a"(timeLo), "d"(timeHi), "c"(hint)
1522 :);
1523 return flag;
1524#else
1525 return _umwait(hint, counter);
1526#endif
1527}
1528#elif KMP_HAVE_MWAIT
1529#if KMP_OS_UNIX
1530#include <pmmintrin.h>
1531#else
1532#include <intrin.h>
1533#endif
1534#if KMP_OS_UNIX
1535__attribute__((target("sse3")))
1536#endif
1537static inline void
1538__kmp_mm_monitor(void *cacheline, unsigned extensions, unsigned hints) {
1539 _mm_monitor(cacheline, extensions, hints);
1540}
1541#if KMP_OS_UNIX
1542__attribute__((target("sse3")))
1543#endif
1544static inline void
1545__kmp_mm_mwait(unsigned extensions, unsigned hints) {
1546 _mm_mwait(extensions, hints);
1547}
1548#endif // KMP_HAVE_UMWAIT
1549
1550#if KMP_ARCH_X86
1551extern void __kmp_x86_pause(void);
1552#elif KMP_MIC
1553// Performance testing on KNC (C0QS-7120 P/A/X/D, 61-core, 16 GB Memory) showed
1554// regression after removal of extra PAUSE from spin loops. Changing
1555// the delay from 100 to 300 showed even better performance than double PAUSE
1556// on Spec OMP2001 and LCPC tasking tests, no regressions on EPCC.
1557static inline void __kmp_x86_pause(void) { _mm_delay_32(300); }
1558#else
1559static inline void __kmp_x86_pause(void) { _mm_pause(); }
1560#endif
1561#define KMP_CPU_PAUSE() __kmp_x86_pause()
1562#elif KMP_ARCH_PPC64
1563#define KMP_PPC64_PRI_LOW() __asm__ volatile("or 1, 1, 1")
1564#define KMP_PPC64_PRI_MED() __asm__ volatile("or 2, 2, 2")
1565#define KMP_PPC64_PRI_LOC_MB() __asm__ volatile("" : : : "memory")
1566#define KMP_CPU_PAUSE() \
1567 do { \
1568 KMP_PPC64_PRI_LOW(); \
1569 KMP_PPC64_PRI_MED(); \
1570 KMP_PPC64_PRI_LOC_MB(); \
1571 } while (0)
1572#else
1573#define KMP_CPU_PAUSE() /* nothing to do */
1574#endif
1575
1576#define KMP_INIT_YIELD(count) \
1577 { (count) = __kmp_yield_init; }
1578
1579#define KMP_INIT_BACKOFF(time) \
1580 { (time) = __kmp_pause_init; }
1581
1582#define KMP_OVERSUBSCRIBED \
1583 (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc))
1584
1585#define KMP_TRY_YIELD \
1586 ((__kmp_use_yield == 1) || (__kmp_use_yield == 2 && (KMP_OVERSUBSCRIBED)))
1587
1588#define KMP_TRY_YIELD_OVERSUB \
1589 ((__kmp_use_yield == 1 || __kmp_use_yield == 2) && (KMP_OVERSUBSCRIBED))
1590
1591#define KMP_YIELD(cond) \
1592 { \
1593 KMP_CPU_PAUSE(); \
1594 if ((cond) && (KMP_TRY_YIELD)) \
1595 __kmp_yield(); \
1596 }
1597
1598#define KMP_YIELD_OVERSUB() \
1599 { \
1600 KMP_CPU_PAUSE(); \
1601 if ((KMP_TRY_YIELD_OVERSUB)) \
1602 __kmp_yield(); \
1603 }
1604
1605// Note the decrement of 2 in the following Macros. With KMP_LIBRARY=turnaround,
1606// there should be no yielding since initial value from KMP_INIT_YIELD() is odd.
1607#define KMP_YIELD_SPIN(count) \
1608 { \
1609 KMP_CPU_PAUSE(); \
1610 if (KMP_TRY_YIELD) { \
1611 (count) -= 2; \
1612 if (!(count)) { \
1613 __kmp_yield(); \
1614 (count) = __kmp_yield_next; \
1615 } \
1616 } \
1617 }
1618
1619// If TPAUSE is available & enabled, use it. If oversubscribed, use the slower
1620// (C0.2) state, which improves performance of other SMT threads on the same
1621// core, otherwise, use the fast (C0.1) default state, or whatever the user has
1622// requested. Uses a timed TPAUSE, and exponential backoff. If TPAUSE isn't
1623// available, fall back to the regular CPU pause and yield combination.
1624#if KMP_HAVE_UMWAIT
1625#define KMP_TPAUSE_MAX_MASK ((kmp_uint64)0xFFFF)
1626#define KMP_YIELD_OVERSUB_ELSE_SPIN(count, time) \
1627 { \
1628 if (__kmp_tpause_enabled) { \
1629 if (KMP_OVERSUBSCRIBED) { \
1630 __kmp_tpause(0, (time)); \
1631 } else { \
1632 __kmp_tpause(__kmp_tpause_hint, (time)); \
1633 } \
1634 (time) = (time << 1 | 1) & KMP_TPAUSE_MAX_MASK; \
1635 } else { \
1636 KMP_CPU_PAUSE(); \
1637 if ((KMP_TRY_YIELD_OVERSUB)) { \
1638 __kmp_yield(); \
1639 } else if (__kmp_use_yield == 1) { \
1640 (count) -= 2; \
1641 if (!(count)) { \
1642 __kmp_yield(); \
1643 (count) = __kmp_yield_next; \
1644 } \
1645 } \
1646 } \
1647 }
1648#else
1649#define KMP_YIELD_OVERSUB_ELSE_SPIN(count, time) \
1650 { \
1651 KMP_CPU_PAUSE(); \
1652 if ((KMP_TRY_YIELD_OVERSUB)) \
1653 __kmp_yield(); \
1654 else if (__kmp_use_yield == 1) { \
1655 (count) -= 2; \
1656 if (!(count)) { \
1657 __kmp_yield(); \
1658 (count) = __kmp_yield_next; \
1659 } \
1660 } \
1661 }
1662#endif // KMP_HAVE_UMWAIT
1663
1664/* ------------------------------------------------------------------------ */
1665/* Support datatypes for the orphaned construct nesting checks. */
1666/* ------------------------------------------------------------------------ */
1667
1668/* When adding to this enum, add its corresponding string in cons_text_c[]
1669 * array in kmp_error.cpp */
1683 ct_masked
1685
1686#define IS_CONS_TYPE_ORDERED(ct) ((ct) == ct_pdo_ordered)
1687
1691 int prev;
1693 name; /* address exclusively for critical section name comparison */
1694};
1695
1700};
1701
1703 char *text;
1706};
1707
1708/* ---------------------------------------------------------------------- */
1709/* ---------------------------------------------------------------------- */
1710
1711#if KMP_OS_WINDOWS
1712typedef HANDLE kmp_thread_t;
1713typedef DWORD kmp_key_t;
1714#endif /* KMP_OS_WINDOWS */
1715
1716#if KMP_OS_UNIX
1717typedef pthread_t kmp_thread_t;
1718typedef pthread_key_t kmp_key_t;
1719#endif
1720
1721extern kmp_key_t __kmp_gtid_threadprivate_key;
1722
1723typedef struct kmp_sys_info {
1724 long maxrss; /* the maximum resident set size utilized (in kilobytes) */
1725 long minflt; /* the number of page faults serviced without any I/O */
1726 long majflt; /* the number of page faults serviced that required I/O */
1727 long nswap; /* the number of times a process was "swapped" out of memory */
1728 long inblock; /* the number of times the file system had to perform input */
1729 long oublock; /* the number of times the file system had to perform output */
1730 long nvcsw; /* the number of times a context switch was voluntarily */
1731 long nivcsw; /* the number of times a context switch was forced */
1733
1734#if USE_ITT_BUILD
1735// We cannot include "kmp_itt.h" due to circular dependency. Declare the only
1736// required type here. Later we will check the type meets requirements.
1737typedef int kmp_itt_mark_t;
1738#define KMP_ITT_DEBUG 0
1739#endif /* USE_ITT_BUILD */
1740
1742
1743/*!
1744@ingroup PARALLEL
1745The type for a microtask which gets passed to @ref __kmpc_fork_call().
1746The arguments to the outlined function are
1747@param global_tid the global thread identity of the thread executing the
1748function.
1749@param bound_tid the local identity of the thread executing the function
1750@param ... pointers to shared variables accessed by the function.
1751*/
1752typedef void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid, ...);
1753typedef void (*kmpc_micro_bound)(kmp_int32 *bound_tid, kmp_int32 *bound_nth,
1754 ...);
1755
1756/*!
1757@ingroup THREADPRIVATE
1758@{
1759*/
1760/* ---------------------------------------------------------------------------
1761 */
1762/* Threadprivate initialization/finalization function declarations */
1763
1764/* for non-array objects: __kmpc_threadprivate_register() */
1765
1766/*!
1767 Pointer to the constructor function.
1768 The first argument is the <tt>this</tt> pointer
1769*/
1770typedef void *(*kmpc_ctor)(void *);
1771
1772/*!
1773 Pointer to the destructor function.
1774 The first argument is the <tt>this</tt> pointer
1775*/
1776typedef void (*kmpc_dtor)(
1777 void * /*, size_t */); /* 2nd arg: magic number for KCC unused by Intel
1778 compiler */
1779/*!
1780 Pointer to an alternate constructor.
1781 The first argument is the <tt>this</tt> pointer.
1782*/
1783typedef void *(*kmpc_cctor)(void *, void *);
1784
1785/* for array objects: __kmpc_threadprivate_register_vec() */
1786/* First arg: "this" pointer */
1787/* Last arg: number of array elements */
1788/*!
1789 Array constructor.
1790 First argument is the <tt>this</tt> pointer
1791 Second argument the number of array elements.
1792*/
1793typedef void *(*kmpc_ctor_vec)(void *, size_t);
1794/*!
1795 Pointer to the array destructor function.
1796 The first argument is the <tt>this</tt> pointer
1797 Second argument the number of array elements.
1798*/
1799typedef void (*kmpc_dtor_vec)(void *, size_t);
1800/*!
1801 Array constructor.
1802 First argument is the <tt>this</tt> pointer
1803 Third argument the number of array elements.
1804*/
1805typedef void *(*kmpc_cctor_vec)(void *, void *,
1806 size_t); /* function unused by compiler */
1807
1808/*!
1809@}
1810*/
1811
1812/* keeps tracked of threadprivate cache allocations for cleanup later */
1813typedef struct kmp_cached_addr {
1814 void **addr; /* address of allocated cache */
1815 void ***compiler_cache; /* pointer to compiler's cache */
1816 void *data; /* pointer to global data */
1817 struct kmp_cached_addr *next; /* pointer to next cached address */
1819
1821 struct private_data *next; /* The next descriptor in the list */
1822 void *data; /* The data buffer for this descriptor */
1823 int more; /* The repeat count for this descriptor */
1824 size_t size; /* The data size for this descriptor */
1825};
1826
1831 void *par_addr; /* par_addr == gbl_addr for PRIMARY thread */
1832 size_t cmn_size;
1833};
1834
1840 union {
1844 union {
1848 union {
1852 size_t vec_len;
1854 size_t cmn_size;
1855};
1856
1857#define KMP_HASH_TABLE_LOG2 9 /* log2 of the hash table size */
1858#define KMP_HASH_TABLE_SIZE \
1859 (1 << KMP_HASH_TABLE_LOG2) /* size of the hash table */
1860#define KMP_HASH_SHIFT 3 /* throw away this many low bits from the address */
1861#define KMP_HASH(x) \
1862 ((((kmp_uintptr_t)x) >> KMP_HASH_SHIFT) & (KMP_HASH_TABLE_SIZE - 1))
1863
1866};
1867
1870};
1871
1872/* ------------------------------------------------------------------------ */
1873
1874#if KMP_USE_HIER_SCHED
1875// Shared barrier data that exists inside a single unit of the scheduling
1876// hierarchy
1877typedef struct kmp_hier_private_bdata_t {
1878 kmp_int32 num_active;
1879 kmp_uint64 index;
1880 kmp_uint64 wait_val[2];
1881} kmp_hier_private_bdata_t;
1882#endif
1883
1884typedef struct kmp_sched_flags {
1885 unsigned ordered : 1;
1886 unsigned nomerge : 1;
1887 unsigned contains_last : 1;
1888 unsigned use_hier : 1; // Used in KMP_USE_HIER_SCHED code
1889 unsigned use_hybrid : 1; // Used in KMP_WEIGHTED_ITERATIONS_SUPPORTED code
1890 unsigned unused : 27;
1892
1894
1895#if KMP_STATIC_STEAL_ENABLED
1898 kmp_int32 ub;
1899 /* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */
1900 kmp_int32 lb;
1901 kmp_int32 st;
1902 kmp_int32 tc;
1903 kmp_lock_t *steal_lock; // lock used for chunk stealing
1904
1905 kmp_uint32 ordered_lower;
1906 kmp_uint32 ordered_upper;
1907
1908 // KMP_ALIGN(32) ensures (if the KMP_ALIGN macro is turned on)
1909 // a) parm3 is properly aligned and
1910 // b) all parm1-4 are on the same cache line.
1911 // Because of parm1-4 are used together, performance seems to be better
1912 // if they are on the same cache line (not measured though).
1913
1914 struct KMP_ALIGN(32) {
1915 kmp_int32 parm1;
1916 kmp_int32 parm2;
1917 kmp_int32 parm3;
1918 kmp_int32 parm4;
1919 };
1920
1921#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
1922 kmp_uint32 pchunks;
1923 kmp_uint32 num_procs_with_pcore;
1924 kmp_int32 first_thread_with_ecore;
1925#endif
1926#if KMP_OS_WINDOWS
1927 kmp_int32 last_upper;
1928#endif /* KMP_OS_WINDOWS */
1930
1931#if CACHE_LINE <= 128
1933#endif
1934
1936 kmp_int64 count; // current chunk number for static & static-steal scheduling
1937 kmp_int64 ub; /* upper-bound */
1938 /* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */
1939 kmp_int64 lb; /* lower-bound */
1940 kmp_int64 st; /* stride */
1941 kmp_int64 tc; /* trip count (number of iterations) */
1942 kmp_lock_t *steal_lock; // lock used for chunk stealing
1943
1944 kmp_uint64 ordered_lower;
1945 kmp_uint64 ordered_upper;
1946 /* parm[1-4] are used in different ways by different scheduling algorithms */
1947
1948 // KMP_ALIGN(32) ensures ( if the KMP_ALIGN macro is turned on )
1949 // a) parm3 is properly aligned and
1950 // b) all parm1-4 are in the same cache line.
1951 // Because of parm1-4 are used together, performance seems to be better
1952 // if they are in the same line (not measured though).
1953 struct KMP_ALIGN(32) {
1954 kmp_int64 parm1;
1955 kmp_int64 parm2;
1956 kmp_int64 parm3;
1957 kmp_int64 parm4;
1958 };
1959
1960#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
1961 kmp_uint64 pchunks;
1962 kmp_uint64 num_procs_with_pcore;
1963 kmp_int64 first_thread_with_ecore;
1964#endif
1965
1966#if KMP_OS_WINDOWS
1967 kmp_int64 last_upper;
1968#endif /* KMP_OS_WINDOWS */
1970
1971#if CACHE_LINE <= 128
1973#endif
1974
1975#else /* KMP_STATIC_STEAL_ENABLED */
1981
1986
1988
1991#if KMP_OS_WINDOWS
1992 kmp_int32 last_upper;
1993#endif /* KMP_OS_WINDOWS */
1995
1997 kmp_int64 lb; /* lower-bound */
1998 kmp_int64 ub; /* upper-bound */
1999 kmp_int64 st; /* stride */
2000 kmp_int64 tc; /* trip count (number of iterations) */
2001
2002 /* parm[1-4] are used in different ways by different scheduling algorithms */
2007
2008 kmp_int64 count; /* current chunk number for static scheduling */
2009
2012#if KMP_OS_WINDOWS
2013 kmp_int64 last_upper;
2014#endif /* KMP_OS_WINDOWS */
2016#endif /* KMP_STATIC_STEAL_ENABLED */
2017
2022 } u;
2023 enum sched_type schedule; /* scheduling algorithm */
2024 kmp_sched_flags_t flags; /* flags (e.g., ordered, nomerge, etc.) */
2025 std::atomic<kmp_uint32> steal_flag; // static_steal only, state of a buffer
2027 // Stack of buffers for nest of serial regions
2029 kmp_int32 type_size; /* the size of types in private_info */
2030#if KMP_USE_HIER_SCHED
2031 kmp_int32 hier_id;
2032 void *parent; /* hierarchical scheduling parent pointer */
2033#endif
2036
2038 /* chunk index under dynamic, number of idle threads under static-steal;
2039 iteration index otherwise */
2043 // Dummy to retain the structure size after making ordered_iteration scalar
2046
2048 /* chunk index under dynamic, number of idle threads under static-steal;
2049 iteration index otherwise */
2053 // Dummy to retain the structure size after making ordered_iteration scalar
2056
2057typedef struct dispatch_shared_info {
2061 } u;
2063 volatile kmp_int32 doacross_buf_idx; // teamwise index
2064 volatile kmp_uint32 *doacross_flags; // shared array of iteration flags (0/1)
2065 kmp_int32 doacross_num_done; // count finished threads
2066#if KMP_USE_HIER_SCHED
2067 void *hier;
2068#endif
2069#if KMP_USE_HWLOC
2070 // When linking with libhwloc, the ORDERED EPCC test slows down on big
2071 // machines (> 48 cores). Performance analysis showed that a cache thrash
2072 // was occurring and this padding helps alleviate the problem.
2073 char padding[64];
2074#endif
2076
2077typedef struct kmp_disp {
2078 /* Vector for ORDERED SECTION */
2079 void (*th_deo_fcn)(int *gtid, int *cid, ident_t *);
2080 /* Vector for END ORDERED SECTION */
2081 void (*th_dxo_fcn)(int *gtid, int *cid, ident_t *);
2082
2085
2088 kmp_int32 th_doacross_buf_idx; // thread's doacross buffer index
2089 volatile kmp_uint32 *th_doacross_flags; // pointer to shared array of flags
2090 kmp_int64 *th_doacross_info; // info on loop bounds
2091#if KMP_USE_INTERNODE_ALIGNMENT
2092 char more_padding[INTERNODE_CACHE_LINE];
2093#endif
2095
2096/* ------------------------------------------------------------------------ */
2097/* Barrier stuff */
2098
2099/* constants for barrier state update */
2100#define KMP_INIT_BARRIER_STATE 0 /* should probably start from zero */
2101#define KMP_BARRIER_SLEEP_BIT 0 /* bit used for suspend/sleep part of state */
2102#define KMP_BARRIER_UNUSED_BIT 1 // bit that must never be set for valid state
2103#define KMP_BARRIER_BUMP_BIT 2 /* lsb used for bump of go/arrived state */
2104
2105#define KMP_BARRIER_SLEEP_STATE (1 << KMP_BARRIER_SLEEP_BIT)
2106#define KMP_BARRIER_UNUSED_STATE (1 << KMP_BARRIER_UNUSED_BIT)
2107#define KMP_BARRIER_STATE_BUMP (1 << KMP_BARRIER_BUMP_BIT)
2108
2109#if (KMP_BARRIER_SLEEP_BIT >= KMP_BARRIER_BUMP_BIT)
2110#error "Barrier sleep bit must be smaller than barrier bump bit"
2111#endif
2112#if (KMP_BARRIER_UNUSED_BIT >= KMP_BARRIER_BUMP_BIT)
2113#error "Barrier unused bit must be smaller than barrier bump bit"
2114#endif
2115
2116// Constants for release barrier wait state: currently, hierarchical only
2117#define KMP_BARRIER_NOT_WAITING 0 // Normal state; worker not in wait_sleep
2118#define KMP_BARRIER_OWN_FLAG \
2119 1 // Normal state; worker waiting on own b_go flag in release
2120#define KMP_BARRIER_PARENT_FLAG \
2121 2 // Special state; worker waiting on parent's b_go flag in release
2122#define KMP_BARRIER_SWITCH_TO_OWN_FLAG \
2123 3 // Special state; tells worker to shift from parent to own b_go
2124#define KMP_BARRIER_SWITCHING \
2125 4 // Special state; worker resets appropriate flag on wake-up
2126
2127#define KMP_NOT_SAFE_TO_REAP \
2128 0 // Thread th_reap_state: not safe to reap (tasking)
2129#define KMP_SAFE_TO_REAP 1 // Thread th_reap_state: safe to reap (not tasking)
2130
2131// The flag_type describes the storage used for the flag.
2133 flag32, /**< atomic 32 bit flags */
2134 flag64, /**< 64 bit flags */
2135 atomic_flag64, /**< atomic 64 bit flags */
2136 flag_oncore, /**< special 64-bit flag for on-core barrier (hierarchical) */
2139
2141 bs_plain_barrier = 0, /* 0, All non-fork/join barriers (except reduction
2142 barriers if enabled) */
2143 bs_forkjoin_barrier, /* 1, All fork/join (parallel region) barriers */
2144#if KMP_FAST_REDUCTION_BARRIER
2145 bs_reduction_barrier, /* 2, All barriers that are used in reduction */
2146#endif // KMP_FAST_REDUCTION_BARRIER
2147 bs_last_barrier /* Just a placeholder to mark the end */
2149
2150// to work with reduction barriers just like with plain barriers
2151#if !KMP_FAST_REDUCTION_BARRIER
2152#define bs_reduction_barrier bs_plain_barrier
2153#endif // KMP_FAST_REDUCTION_BARRIER
2154
2155typedef enum kmp_bar_pat { /* Barrier communication patterns */
2157 0, /* Single level (degenerate) tree */
2158 bp_tree_bar =
2159 1, /* Balanced tree with branching factor 2^n */
2160 bp_hyper_bar = 2, /* Hypercube-embedded tree with min
2161 branching factor 2^n */
2162 bp_hierarchical_bar = 3, /* Machine hierarchy tree */
2163 bp_dist_bar = 4, /* Distributed barrier */
2164 bp_last_bar /* Placeholder to mark the end */
2166
2167#define KMP_BARRIER_ICV_PUSH 1
2168
2169/* Record for holding the values of the internal controls stack records */
2170typedef struct kmp_internal_control {
2171 int serial_nesting_level; /* corresponds to the value of the
2172 th_team_serialized field */
2173 kmp_int8 dynamic; /* internal control for dynamic adjustment of threads (per
2174 thread) */
2175 kmp_int8
2176 bt_set; /* internal control for whether blocktime is explicitly set */
2177 int blocktime; /* internal control for blocktime */
2178#if KMP_USE_MONITOR
2179 int bt_intervals; /* internal control for blocktime intervals */
2180#endif
2181 int nproc; /* internal control for #threads for next parallel region (per
2182 thread) */
2183 int thread_limit; /* internal control for thread-limit-var */
2184 int task_thread_limit; /* internal control for thread-limit-var of a task*/
2185 int max_active_levels; /* internal control for max_active_levels */
2187 sched; /* internal control for runtime schedule {sched,chunk} pair */
2188 kmp_proc_bind_t proc_bind; /* internal control for affinity */
2189 kmp_int32 default_device; /* internal control for default device */
2192
2193static inline void copy_icvs(kmp_internal_control_t *dst,
2195 *dst = *src;
2196}
2197
2198/* Thread barrier needs volatile barrier fields */
2200 // th_fixed_icvs is aligned by virtue of kmp_bstate being aligned (and all
2201 // uses of it). It is not explicitly aligned below, because we *don't* want
2202 // it to be padded -- instead, we fit b_go into the same cache line with
2203 // th_fixed_icvs, enabling NGO cache lines stores in the hierarchical barrier.
2204 kmp_internal_control_t th_fixed_icvs; // Initial ICVs for the thread
2205 // Tuck b_go into end of th_fixed_icvs cache line, so it can be stored with
2206 // same NGO store
2207 volatile kmp_uint64 b_go; // STATE => task should proceed (hierarchical)
2209 b_arrived; // STATE => task reached synch point.
2224#if USE_DEBUGGER
2225 // The following field is intended for the debugger solely. Only the worker
2226 // thread itself accesses this field: the worker increases it by 1 when it
2227 // arrives to a barrier.
2228 KMP_ALIGN_CACHE kmp_uint b_worker_arrived;
2229#endif /* USE_DEBUGGER */
2231
2233 double b_align; /* use worst case alignment */
2236};
2237
2239
2240/* Team barrier needs only non-volatile arrived counter */
2242 double b_align; /* use worst case alignment */
2243 char b_pad[CACHE_LINE];
2244 struct {
2245 kmp_uint64 b_arrived; /* STATE => task reached synch point. */
2246#if USE_DEBUGGER
2247 // The following two fields are indended for the debugger solely. Only
2248 // primary thread of the team accesses these fields: the first one is
2249 // increased by 1 when the primary thread arrives to a barrier, the second
2250 // one is increased by one when all the threads arrived.
2251 kmp_uint b_master_arrived;
2252 kmp_uint b_team_arrived;
2253#endif
2254 };
2255};
2256
2258
2259/* Padding for Linux* OS pthreads condition variables and mutexes used to signal
2260 threads when a condition changes. This is to workaround an NPTL bug where
2261 padding was added to pthread_cond_t which caused the initialization routine
2262 to write outside of the structure if compiled on pre-NPTL threads. */
2263#if KMP_OS_WINDOWS
2264typedef struct kmp_win32_mutex {
2265 /* The Lock */
2266 CRITICAL_SECTION cs;
2267} kmp_win32_mutex_t;
2268
2269typedef struct kmp_win32_cond {
2270 /* Count of the number of waiters. */
2271 int waiters_count_;
2272
2273 /* Serialize access to <waiters_count_> */
2274 kmp_win32_mutex_t waiters_count_lock_;
2275
2276 /* Number of threads to release via a <cond_broadcast> or a <cond_signal> */
2277 int release_count_;
2278
2279 /* Keeps track of the current "generation" so that we don't allow */
2280 /* one thread to steal all the "releases" from the broadcast. */
2281 int wait_generation_count_;
2282
2283 /* A manual-reset event that's used to block and release waiting threads. */
2284 HANDLE event_;
2285} kmp_win32_cond_t;
2286#endif
2287
2288#if KMP_OS_UNIX
2289
2290union KMP_ALIGN_CACHE kmp_cond_union {
2291 double c_align;
2292 char c_pad[CACHE_LINE];
2293 pthread_cond_t c_cond;
2294};
2295
2296typedef union kmp_cond_union kmp_cond_align_t;
2297
2298union KMP_ALIGN_CACHE kmp_mutex_union {
2299 double m_align;
2300 char m_pad[CACHE_LINE];
2301 pthread_mutex_t m_mutex;
2302};
2303
2304typedef union kmp_mutex_union kmp_mutex_align_t;
2305
2306#endif /* KMP_OS_UNIX */
2307
2308typedef struct kmp_desc_base {
2312 kmp_thread_t ds_thread;
2313 volatile int ds_tid;
2315#if KMP_OS_WINDOWS
2316 volatile int ds_alive;
2317 DWORD ds_thread_id;
2318/* ds_thread keeps thread handle on Windows* OS. It is enough for RTL purposes.
2319 However, debugger support (libomp_db) cannot work with handles, because they
2320 uncomparable. For example, debugger requests info about thread with handle h.
2321 h is valid within debugger process, and meaningless within debugee process.
2322 Even if h is duped by call to DuplicateHandle(), so the result h' is valid
2323 within debugee process, but it is a *new* handle which does *not* equal to
2324 any other handle in debugee... The only way to compare handles is convert
2325 them to system-wide ids. GetThreadId() function is available only in
2326 Longhorn and Server 2003. :-( In contrast, GetCurrentThreadId() is available
2327 on all Windows* OS flavours (including Windows* 95). Thus, we have to get
2328 thread id by call to GetCurrentThreadId() from within the thread and save it
2329 to let libomp_db identify threads. */
2330#endif /* KMP_OS_WINDOWS */
2332
2334 double ds_align; /* use worst case alignment */
2338
2339typedef struct kmp_local {
2340 volatile int this_construct; /* count of single's encountered by thread */
2342#if KMP_USE_BGET
2343 void *bget_data;
2344 void *bget_list;
2345#if !USE_CMP_XCHG_FOR_BGET
2346#ifdef USE_QUEUING_LOCK_FOR_BGET
2347 kmp_lock_t bget_lock; /* Lock for accessing bget free list */
2348#else
2349 kmp_bootstrap_lock_t bget_lock; // Lock for accessing bget free list. Must be
2350// bootstrap lock so we can use it at library
2351// shutdown.
2352#endif /* USE_LOCK_FOR_BGET */
2353#endif /* ! USE_CMP_XCHG_FOR_BGET */
2354#endif /* KMP_USE_BGET */
2355
2357 packed_reduction_method; /* stored by __kmpc_reduce*(), used by
2358 __kmpc_end_reduce*() */
2359
2361
2362#define KMP_CHECK_UPDATE(a, b) \
2363 if ((a) != (b)) \
2364 (a) = (b)
2365#define KMP_CHECK_UPDATE_SYNC(a, b) \
2366 if ((a) != (b)) \
2367 TCW_SYNC_PTR((a), (b))
2368
2369#define get__blocktime(xteam, xtid) \
2370 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime)
2371#define get__bt_set(xteam, xtid) \
2372 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set)
2373#if KMP_USE_MONITOR
2374#define get__bt_intervals(xteam, xtid) \
2375 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals)
2376#endif
2377
2378#define get__dynamic_2(xteam, xtid) \
2379 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic)
2380#define get__nproc_2(xteam, xtid) \
2381 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc)
2382#define get__sched_2(xteam, xtid) \
2383 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched)
2384
2385#define set__blocktime_team(xteam, xtid, xval) \
2386 (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime) = \
2387 (xval))
2388
2389#if KMP_USE_MONITOR
2390#define set__bt_intervals_team(xteam, xtid, xval) \
2391 (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals) = \
2392 (xval))
2393#endif
2394
2395#define set__bt_set_team(xteam, xtid, xval) \
2396 (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set) = (xval))
2397
2398#define set__dynamic(xthread, xval) \
2399 (((xthread)->th.th_current_task->td_icvs.dynamic) = (xval))
2400#define get__dynamic(xthread) \
2401 (((xthread)->th.th_current_task->td_icvs.dynamic) ? (FTN_TRUE) : (FTN_FALSE))
2402
2403#define set__nproc(xthread, xval) \
2404 (((xthread)->th.th_current_task->td_icvs.nproc) = (xval))
2405
2406#define set__thread_limit(xthread, xval) \
2407 (((xthread)->th.th_current_task->td_icvs.thread_limit) = (xval))
2408
2409#define set__max_active_levels(xthread, xval) \
2410 (((xthread)->th.th_current_task->td_icvs.max_active_levels) = (xval))
2411
2412#define get__max_active_levels(xthread) \
2413 ((xthread)->th.th_current_task->td_icvs.max_active_levels)
2414
2415#define set__sched(xthread, xval) \
2416 (((xthread)->th.th_current_task->td_icvs.sched) = (xval))
2417
2418#define set__proc_bind(xthread, xval) \
2419 (((xthread)->th.th_current_task->td_icvs.proc_bind) = (xval))
2420#define get__proc_bind(xthread) \
2421 ((xthread)->th.th_current_task->td_icvs.proc_bind)
2422
2423// OpenMP tasking data structures
2424
2425typedef enum kmp_tasking_mode {
2429 tskm_max = 2
2431
2432extern kmp_tasking_mode_t
2433 __kmp_tasking_mode; /* determines how/when to execute tasks */
2436extern kmp_int32 __kmp_default_device; // Set via OMP_DEFAULT_DEVICE if
2437// specified, defaults to 0 otherwise
2438// Set via OMP_MAX_TASK_PRIORITY if specified, defaults to 0 otherwise
2440// Set via KMP_TASKLOOP_MIN_TASKS if specified, defaults to 0 otherwise
2442
2443/* NOTE: kmp_taskdata_t and kmp_task_t structures allocated in single block with
2444 taskdata first */
2445#define KMP_TASK_TO_TASKDATA(task) (((kmp_taskdata_t *)task) - 1)
2446#define KMP_TASKDATA_TO_TASK(taskdata) (kmp_task_t *)(taskdata + 1)
2447
2448// The tt_found_tasks flag is a signal to all threads in the team that tasks
2449// were spawned and queued since the previous barrier release.
2450#define KMP_TASKING_ENABLED(task_team) \
2451 (TRUE == TCR_SYNC_4((task_team)->tt.tt_found_tasks))
2452/*!
2453@ingroup BASIC_TYPES
2454@{
2455*/
2456
2457/*!
2458 */
2460
2461typedef union kmp_cmplrdata {
2462 kmp_int32 priority; /**< priority specified by user for the task */
2464 destructors; /* pointer to function to invoke deconstructors of
2465 firstprivate C++ objects */
2466 /* future data */
2468
2469/* sizeof_kmp_task_t passed as arg to kmpc_omp_task call */
2470/*!
2471 */
2472typedef struct kmp_task { /* GEH: Shouldn't this be aligned somehow? */
2473 void *shareds; /**< pointer to block of pointers to shared vars */
2475 routine; /**< pointer to routine to call for executing task */
2476 kmp_int32 part_id; /**< part id for the task */
2478 data1; /* Two known optional additions: destructors and priority */
2479 kmp_cmplrdata_t data2; /* Process destructors first, priority second */
2480 /* future data */
2481 /* private vars */
2483
2484/*!
2485@}
2486*/
2487
2488typedef struct kmp_taskgroup {
2489 std::atomic<kmp_int32> count; // number of allocated and incomplete tasks
2490 std::atomic<kmp_int32>
2491 cancel_request; // request for cancellation of this taskgroup
2492 struct kmp_taskgroup *parent; // parent taskgroup
2493 // Block of data to perform task reduction
2494 void *reduce_data; // reduction related info
2495 kmp_int32 reduce_num_data; // number of data items to reduce
2496 uintptr_t *gomp_data; // gomp reduction data
2498
2499// forward declarations
2503
2504// macros for checking dep flag as an integer
2505#define KMP_DEP_IN 0x1
2506#define KMP_DEP_OUT 0x2
2507#define KMP_DEP_INOUT 0x3
2508#define KMP_DEP_MTX 0x4
2509#define KMP_DEP_SET 0x8
2510#define KMP_DEP_ALL 0x80
2511// Compiler sends us this info. Note: some test cases contain an explicit copy
2512// of this struct and should be in sync with any changes here.
2513typedef struct kmp_depend_info {
2515 size_t len;
2516 union {
2517 kmp_uint8 flag; // flag as an unsigned char
2518 struct { // flag as a set of 8 bits
2519#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2520 /* Same fields as in the #else branch, but in reverse order */
2521 unsigned all : 1;
2522 unsigned unused : 3;
2523 unsigned set : 1;
2524 unsigned mtx : 1;
2525 unsigned out : 1;
2526 unsigned in : 1;
2527#else
2528 unsigned in : 1;
2529 unsigned out : 1;
2530 unsigned mtx : 1;
2531 unsigned set : 1;
2532 unsigned unused : 3;
2533 unsigned all : 1;
2534#endif
2536 };
2538
2539// Internal structures to work with task dependencies:
2543};
2544
2545// Max number of mutexinoutset dependencies per node
2546#define MAX_MTX_DEPS 4
2547
2548typedef struct kmp_base_depnode {
2549 kmp_depnode_list_t *successors; /* used under lock */
2550 kmp_task_t *task; /* non-NULL if depnode is active, used under lock */
2551 kmp_lock_t *mtx_locks[MAX_MTX_DEPS]; /* lock mutexinoutset dependent tasks */
2552 kmp_int32 mtx_num_locks; /* number of locks in mtx_locks array */
2553 kmp_lock_t lock; /* guards shared fields: task, successors */
2554#if KMP_SUPPORT_GRAPH_OUTPUT
2555 kmp_uint32 id;
2556#endif
2557 std::atomic<kmp_int32> npredecessors;
2558 std::atomic<kmp_int32> nrefs;
2560
2562 double dn_align; /* use worst case alignment */
2565};
2566
2573 kmp_lock_t *mtx_lock; /* is referenced by depnodes w/mutexinoutset dep */
2575};
2576
2577typedef struct kmp_dephash {
2579 size_t size;
2585
2588 size_t len;
2589 struct {
2590 bool flag1 : 1;
2591 bool flag2 : 1;
2595
2596typedef enum kmp_event_type_t {
2600
2601typedef struct {
2604 union {
2606 } ed;
2607} kmp_event_t;
2608
2609#if OMPX_TASKGRAPH
2610// Initial number of allocated nodes while recording
2611#define INIT_MAPSIZE 50
2612
2613typedef struct kmp_taskgraph_flags { /*This needs to be exactly 32 bits */
2614 unsigned nowait : 1;
2615 unsigned re_record : 1;
2616 unsigned reserved : 30;
2617} kmp_taskgraph_flags_t;
2618
2619/// Represents a TDG node
2620typedef struct kmp_node_info {
2621 kmp_task_t *task; // Pointer to the actual task
2622 kmp_int32 *successors; // Array of the succesors ids
2623 kmp_int32 nsuccessors; // Number of succesors of the node
2624 std::atomic<kmp_int32>
2625 npredecessors_counter; // Number of predessors on the fly
2626 kmp_int32 npredecessors; // Total number of predecessors
2627 kmp_int32 successors_size; // Number of allocated succesors ids
2628 kmp_taskdata_t *parent_task; // Parent implicit task
2629} kmp_node_info_t;
2630
2631/// Represent a TDG's current status
2632typedef enum kmp_tdg_status {
2633 KMP_TDG_NONE = 0,
2634 KMP_TDG_RECORDING = 1,
2635 KMP_TDG_READY = 2
2636} kmp_tdg_status_t;
2637
2638/// Structure that contains a TDG
2639typedef struct kmp_tdg_info {
2640 kmp_int32 tdg_id; // Unique idenfifier of the TDG
2641 kmp_taskgraph_flags_t tdg_flags; // Flags related to a TDG
2642 kmp_int32 map_size; // Number of allocated TDG nodes
2643 kmp_int32 num_roots; // Number of roots tasks int the TDG
2644 kmp_int32 *root_tasks; // Array of tasks identifiers that are roots
2645 kmp_node_info_t *record_map; // Array of TDG nodes
2646 kmp_tdg_status_t tdg_status =
2647 KMP_TDG_NONE; // Status of the TDG (recording, ready...)
2648 std::atomic<kmp_int32> num_tasks; // Number of TDG nodes
2650 graph_lock; // Protect graph attributes when updated via taskloop_recur
2651 // Taskloop reduction related
2652 void *rec_taskred_data; // Data to pass to __kmpc_task_reduction_init or
2653 // __kmpc_taskred_init
2654 kmp_int32 rec_num_taskred;
2655} kmp_tdg_info_t;
2656
2657extern int __kmp_tdg_dot;
2658extern kmp_int32 __kmp_max_tdgs;
2659extern kmp_tdg_info_t **__kmp_global_tdgs;
2660extern kmp_int32 __kmp_curr_tdg_idx;
2661extern kmp_int32 __kmp_successors_size;
2662extern std::atomic<kmp_int32> __kmp_tdg_task_id;
2663extern kmp_int32 __kmp_num_tdg;
2664#endif
2665
2666#ifdef BUILD_TIED_TASK_STACK
2667
2668/* Tied Task stack definitions */
2669typedef struct kmp_stack_block {
2670 kmp_taskdata_t *sb_block[TASK_STACK_BLOCK_SIZE];
2671 struct kmp_stack_block *sb_next;
2672 struct kmp_stack_block *sb_prev;
2673} kmp_stack_block_t;
2674
2675typedef struct kmp_task_stack {
2676 kmp_stack_block_t ts_first_block; // first block of stack entries
2677 kmp_taskdata_t **ts_top; // pointer to the top of stack
2678 kmp_int32 ts_entries; // number of entries on the stack
2679} kmp_task_stack_t;
2680
2681#endif // BUILD_TIED_TASK_STACK
2682
2683typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
2684#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2685 /* Same fields as in the #else branch, but in reverse order */
2686#if OMPX_TASKGRAPH
2687 unsigned reserved31 : 5;
2688 unsigned onced : 1;
2689#else
2690 unsigned reserved31 : 6;
2691#endif
2692 unsigned target : 1;
2693 unsigned native : 1;
2694 unsigned freed : 1;
2695 unsigned complete : 1;
2696 unsigned executing : 1;
2697 unsigned started : 1;
2698 unsigned team_serial : 1;
2699 unsigned tasking_ser : 1;
2700 unsigned task_serial : 1;
2701 unsigned tasktype : 1;
2702 unsigned reserved : 8;
2703 unsigned hidden_helper : 1;
2704 unsigned detachable : 1;
2705 unsigned priority_specified : 1;
2706 unsigned proxy : 1;
2707 unsigned destructors_thunk : 1;
2708 unsigned merged_if0 : 1;
2709 unsigned final : 1;
2710 unsigned tiedness : 1;
2711#else
2712 /* Compiler flags */ /* Total compiler flags must be 16 bits */
2713 unsigned tiedness : 1; /* task is either tied (1) or untied (0) */
2714 unsigned final : 1; /* task is final(1) so execute immediately */
2715 unsigned merged_if0 : 1; /* no __kmpc_task_{begin/complete}_if0 calls in if0
2716 code path */
2717 unsigned destructors_thunk : 1; /* set if the compiler creates a thunk to
2718 invoke destructors from the runtime */
2719 unsigned proxy : 1; /* task is a proxy task (it will be executed outside the
2720 context of the RTL) */
2721 unsigned priority_specified : 1; /* set if the compiler provides priority
2722 setting for the task */
2723 unsigned detachable : 1; /* 1 == can detach */
2724 unsigned hidden_helper : 1; /* 1 == hidden helper task */
2725 unsigned reserved : 8; /* reserved for compiler use */
2726
2727 /* Library flags */ /* Total library flags must be 16 bits */
2728 unsigned tasktype : 1; /* task is either explicit(1) or implicit (0) */
2729 unsigned task_serial : 1; // task is executed immediately (1) or deferred (0)
2730 unsigned tasking_ser : 1; // all tasks in team are either executed immediately
2731 // (1) or may be deferred (0)
2732 unsigned team_serial : 1; // entire team is serial (1) [1 thread] or parallel
2733 // (0) [>= 2 threads]
2734 /* If either team_serial or tasking_ser is set, task team may be NULL */
2735 /* Task State Flags: */
2736 unsigned started : 1; /* 1==started, 0==not started */
2737 unsigned executing : 1; /* 1==executing, 0==not executing */
2738 unsigned complete : 1; /* 1==complete, 0==not complete */
2739 unsigned freed : 1; /* 1==freed, 0==allocated */
2740 unsigned native : 1; /* 1==gcc-compiled task, 0==intel */
2741 unsigned target : 1;
2742#if OMPX_TASKGRAPH
2743 unsigned onced : 1; /* 1==ran once already, 0==never ran, record & replay purposes */
2744 unsigned reserved31 : 5; /* reserved for library use */
2745#else
2746 unsigned reserved31 : 6; /* reserved for library use */
2747#endif
2748#endif
2750
2751typedef struct kmp_target_data {
2752 void *async_handle; // libomptarget async handle for task completion query
2754
2755struct kmp_taskdata { /* aligned during dynamic allocation */
2756 kmp_int32 td_task_id; /* id, assigned by debugger */
2758 kmp_team_t *td_team; /* team for this task */
2759 kmp_info_p *td_alloc_thread; /* thread that allocated data structures */
2760 /* Currently not used except for perhaps IDB */
2761 kmp_taskdata_t *td_parent; /* parent task */
2762 kmp_int32 td_level; /* task nesting level */
2763 std::atomic<kmp_int32> td_untied_count; // untied task active parts counter
2764 ident_t *td_ident; /* task identifier */
2765 // Taskwait data.
2768 kmp_int32 td_taskwait_thread; /* gtid + 1 of thread encountered taskwait */
2770 td_icvs; /* Internal control variables for the task */
2771 KMP_ALIGN_CACHE std::atomic<kmp_int32>
2772 td_allocated_child_tasks; /* Child tasks (+ current task) not yet
2773 deallocated */
2774 std::atomic<kmp_int32>
2775 td_incomplete_child_tasks; /* Child tasks not yet complete */
2777 *td_taskgroup; // Each task keeps pointer to its current taskgroup
2779 *td_dephash; // Dependencies for children tasks are tracked from here
2781 *td_depnode; // Pointer to graph node if this task has dependencies
2783 size_t td_size_alloc; // Size of task structure, including shareds etc.
2784#if defined(KMP_GOMP_COMPAT)
2785 // 4 or 8 byte integers for the loop bounds in GOMP_taskloop
2786 kmp_int32 td_size_loop_bounds;
2787#endif
2788 kmp_taskdata_t *td_last_tied; // keep tied task for task scheduling constraint
2789#if defined(KMP_GOMP_COMPAT)
2790 // GOMP sends in a copy function for copy constructors
2791 void (*td_copy_func)(void *, void *);
2792#endif
2794#if OMPT_SUPPORT
2795 ompt_task_info_t ompt_task_info;
2796#endif
2797#if OMPX_TASKGRAPH
2798 bool is_taskgraph = 0; // whether the task is within a TDG
2799 kmp_tdg_info_t *tdg; // used to associate task with a TDG
2800#endif
2802}; // struct kmp_taskdata
2803
2804// Make sure padding above worked
2805KMP_BUILD_ASSERT(sizeof(kmp_taskdata_t) % sizeof(void *) == 0);
2806
2807// Data for task team but per thread
2808typedef struct kmp_base_thread_data {
2809 kmp_info_p *td_thr; // Pointer back to thread info
2810 // Used only in __kmp_execute_tasks_template, maybe not avail until task is
2811 // queued?
2812 kmp_bootstrap_lock_t td_deque_lock; // Lock for accessing deque
2814 *td_deque; // Deque of tasks encountered by td_thr, dynamically allocated
2815 kmp_int32 td_deque_size; // Size of deck
2816 kmp_uint32 td_deque_head; // Head of deque (will wrap)
2817 kmp_uint32 td_deque_tail; // Tail of deque (will wrap)
2818 kmp_int32 td_deque_ntasks; // Number of tasks in deque
2819 // GEH: shouldn't this be volatile since used in while-spin?
2820 kmp_int32 td_deque_last_stolen; // Thread number of last successful steal
2821#ifdef BUILD_TIED_TASK_STACK
2822 kmp_task_stack_t td_susp_tied_tasks; // Stack of suspended tied tasks for task
2823// scheduling constraint
2824#endif // BUILD_TIED_TASK_STACK
2826
2827#define TASK_DEQUE_BITS 8 // Used solely to define INITIAL_TASK_DEQUE_SIZE
2828#define INITIAL_TASK_DEQUE_SIZE (1 << TASK_DEQUE_BITS)
2829
2830#define TASK_DEQUE_SIZE(td) ((td).td_deque_size)
2831#define TASK_DEQUE_MASK(td) ((td).td_deque_size - 1)
2832
2835 double td_align; /* use worst case alignment */
2838
2839typedef struct kmp_task_pri {
2844
2845// Data for task teams which are used when tasking is enabled for the team
2846typedef struct kmp_base_task_team {
2848 tt_threads_lock; /* Lock used to allocate per-thread part of task team */
2849 /* must be bootstrap lock since used at library shutdown*/
2850
2851 // TODO: check performance vs kmp_tas_lock_t
2852 kmp_bootstrap_lock_t tt_task_pri_lock; /* Lock to access priority tasks */
2854
2855 kmp_task_team_t *tt_next; /* For linking the task team free list */
2857 *tt_threads_data; /* Array of per-thread structures for task team */
2858 /* Data survives task team deallocation */
2859 kmp_int32 tt_found_tasks; /* Have we found tasks and queued them while
2860 executing this team? */
2861 /* TRUE means tt_threads_data is set up and initialized */
2862 kmp_int32 tt_nproc; /* #threads in team */
2863 kmp_int32 tt_max_threads; // # entries allocated for threads_data array
2864 kmp_int32 tt_found_proxy_tasks; // found proxy tasks since last barrier
2866 std::atomic<kmp_int32> tt_num_task_pri; // number of priority tasks enqueued
2867 // There is hidden helper thread encountered in this task team so that we must
2868 // wait when waiting on task team
2870
2872 std::atomic<kmp_int32> tt_unfinished_threads; /* #threads still active */
2873
2875 volatile kmp_uint32
2876 tt_active; /* is the team still actively executing tasks */
2878
2881 double tt_align; /* use worst case alignment */
2883};
2884
2885typedef struct kmp_task_team_list_t {
2889
2890#if (USE_FAST_MEMORY == 3) || (USE_FAST_MEMORY == 5)
2891// Free lists keep same-size free memory slots for fast memory allocation
2892// routines
2893typedef struct kmp_free_list {
2894 void *th_free_list_self; // Self-allocated tasks free list
2895 void *th_free_list_sync; // Self-allocated tasks stolen/returned by other
2896 // threads
2897 void *th_free_list_other; // Non-self free list (to be returned to owner's
2898 // sync list)
2899} kmp_free_list_t;
2900#endif
2901#if KMP_NESTED_HOT_TEAMS
2902// Hot teams array keeps hot teams and their sizes for given thread. Hot teams
2903// are not put in teams pool, and they don't put threads in threads pool.
2904typedef struct kmp_hot_team_ptr {
2905 kmp_team_p *hot_team; // pointer to hot_team of given nesting level
2906 kmp_int32 hot_team_nth; // number of threads allocated for the hot_team
2907} kmp_hot_team_ptr_t;
2908#endif
2909typedef struct kmp_teams_size {
2910 kmp_int32 nteams; // number of teams in a league
2911 kmp_int32 nth; // number of threads in each team of the league
2913
2914// This struct stores a thread that acts as a "root" for a contention
2915// group. Contention groups are rooted at kmp_root threads, but also at
2916// each primary thread of each team created in the teams construct.
2917// This struct therefore also stores a thread_limit associated with
2918// that contention group, and a counter to track the number of threads
2919// active in that contention group. Each thread has a list of these: CG
2920// root threads have an entry in their list in which cg_root refers to
2921// the thread itself, whereas other workers in the CG will have a
2922// single entry where cg_root is same as the entry containing their CG
2923// root. When a thread encounters a teams construct, it will add a new
2924// entry to the front of its list, because it now roots a new CG.
2925typedef struct kmp_cg_root {
2926 kmp_info_p *cg_root; // "root" thread for a contention group
2927 // The CG root's limit comes from OMP_THREAD_LIMIT for root threads, or
2928 // thread_limit clause for teams primary threads
2930 kmp_int32 cg_nthreads; // Count of active threads in CG rooted at cg_root
2931 struct kmp_cg_root *up; // pointer to higher level CG root in list
2933
2934// OpenMP thread data structures
2935
2937 /* Start with the readonly data which is cache aligned and padded. This is
2938 written before the thread starts working by the primary thread. Uber
2939 masters may update themselves later. Usage does not consider serialized
2940 regions. */
2942 kmp_team_p *th_team; /* team we belong to */
2943 kmp_root_p *th_root; /* pointer to root of task hierarchy */
2944 kmp_info_p *th_next_pool; /* next available thread in the pool */
2945 kmp_disp_t *th_dispatch; /* thread's dispatch data */
2946 int th_in_pool; /* in thread pool (32 bits for TCR/TCW) */
2947
2948 /* The following are cached from the team info structure */
2949 /* TODO use these in more places as determined to be needed via profiling */
2950 int th_team_nproc; /* number of threads in a team */
2951 kmp_info_p *th_team_master; /* the team's primary thread */
2952 int th_team_serialized; /* team is serialized */
2953 microtask_t th_teams_microtask; /* save entry address for teams construct */
2954 int th_teams_level; /* save initial level of teams construct */
2955/* it is 0 on device but may be any on host */
2956
2957/* The blocktime info is copied from the team struct to the thread struct */
2958/* at the start of a barrier, and the values stored in the team are used */
2959/* at points in the code where the team struct is no longer guaranteed */
2960/* to exist (from the POV of worker threads). */
2961#if KMP_USE_MONITOR
2962 int th_team_bt_intervals;
2963 int th_team_bt_set;
2964#else
2966#endif
2967
2968#if KMP_AFFINITY_SUPPORTED
2969 kmp_affin_mask_t *th_affin_mask; /* thread's current affinity mask */
2970 kmp_affinity_ids_t th_topology_ids; /* thread's current topology ids */
2971 kmp_affinity_attrs_t th_topology_attrs; /* thread's current topology attrs */
2972#endif
2974 /* The data set by the primary thread at reinit, then R/W by the worker */
2975 KMP_ALIGN_CACHE int
2976 th_set_nproc; /* if > 0, then only use this request for the next fork */
2978 bool th_nt_strict; // num_threads clause has strict modifier
2979 ident_t *th_nt_loc; // loc for strict modifier
2980 int th_nt_sev; // error severity for strict modifier
2981 const char *th_nt_msg; // error message for strict modifier
2983#if KMP_NESTED_HOT_TEAMS
2984 kmp_hot_team_ptr_t *th_hot_teams; /* array of hot teams */
2985#endif
2987 th_set_proc_bind; /* if != proc_bind_default, use request for next fork */
2989 th_teams_size; /* number of teams/threads in teams construct */
2990#if KMP_AFFINITY_SUPPORTED
2991 int th_current_place; /* place currently bound to */
2992 int th_new_place; /* place to bind to in par reg */
2993 int th_first_place; /* first place in partition */
2994 int th_last_place; /* last place in partition */
2995#endif
2996 int th_prev_level; /* previous level for affinity format */
2997 int th_prev_num_threads; /* previous num_threads for affinity format */
2998#if USE_ITT_BUILD
2999 kmp_uint64 th_bar_arrive_time; /* arrival to barrier timestamp */
3000 kmp_uint64 th_bar_min_time; /* minimum arrival time at the barrier */
3001 kmp_uint64 th_frame_time; /* frame timestamp */
3002#endif /* USE_ITT_BUILD */
3005
3006 /* Now the data only used by the worker (after initial allocation) */
3007 /* TODO the first serial team should actually be stored in the info_t
3008 structure. this will help reduce initial allocation overhead */
3010 *th_serial_team; /*serialized team held in reserve*/
3011
3012#if OMPT_SUPPORT
3013 ompt_thread_info_t ompt_thread_info;
3014#endif
3015
3016 /* The following are also read by the primary thread during reinit */
3018
3019 volatile kmp_uint32 th_spin_here; /* thread-local location for spinning */
3020 /* while awaiting queuing lock acquire */
3021
3022 volatile void *th_sleep_loc; // this points at a kmp_flag<T>
3023 flag_type th_sleep_loc_type; // enum type of flag stored in th_sleep_loc
3024
3026 unsigned th_x; // Random number generator data
3027 unsigned th_a; // Random number generator data
3028
3029 /* Tasking-related data for the thread */
3030 kmp_task_team_t *th_task_team; // Task team struct
3031 kmp_taskdata_t *th_current_task; // Innermost Task being executed
3032 kmp_uint8 th_task_state; // alternating 0/1 for task team identification
3033 kmp_uint32 th_reap_state; // Non-zero indicates thread is not
3034 // tasking, thus safe to reap
3035
3036 /* More stuff for keeping track of active/sleeping threads (this part is
3037 written by the worker thread) */
3038 kmp_uint8 th_active_in_pool; // included in count of #active threads in pool
3039 int th_active; // ! sleeping; 32 bits for TCR/TCW
3040 std::atomic<kmp_uint32> th_used_in_team; // Flag indicating use in team
3041 // 0 = not used in team; 1 = used in team;
3042 // 2 = transitioning to not used in team; 3 = transitioning to used in team
3043 struct cons_header *th_cons; // used for consistency check
3044#if KMP_USE_HIER_SCHED
3045 // used for hierarchical scheduling
3046 kmp_hier_private_bdata_t *th_hier_bar_data;
3047#endif
3048
3049 /* Add the syncronizing data which is cache aligned and padded. */
3051
3052 KMP_ALIGN_CACHE volatile kmp_int32
3053 th_next_waiting; /* gtid+1 of next thread on lock wait queue, 0 if none */
3054
3055#if (USE_FAST_MEMORY == 3) || (USE_FAST_MEMORY == 5)
3056#define NUM_LISTS 4
3057 kmp_free_list_t th_free_lists[NUM_LISTS]; // Free lists for fast memory
3058// allocation routines
3059#endif
3060
3061#if KMP_OS_WINDOWS
3062 kmp_win32_cond_t th_suspend_cv;
3063 kmp_win32_mutex_t th_suspend_mx;
3064 std::atomic<int> th_suspend_init;
3065#endif
3066#if KMP_OS_UNIX
3067 kmp_cond_align_t th_suspend_cv;
3068 kmp_mutex_align_t th_suspend_mx;
3069 std::atomic<int> th_suspend_init_count;
3070#endif
3071
3072#if USE_ITT_BUILD
3073 kmp_itt_mark_t th_itt_mark_single;
3074// alignment ???
3075#endif /* USE_ITT_BUILD */
3076#if KMP_STATS_ENABLED
3077 kmp_stats_list *th_stats;
3078#endif
3079#if KMP_OS_UNIX
3080 std::atomic<bool> th_blocking;
3081#endif
3082 kmp_cg_root_t *th_cg_roots; // list of cg_roots associated with this thread
3084
3086 double th_align; /* use worst case alignment */
3090
3091// OpenMP thread team data structures
3092
3093typedef struct kmp_base_data {
3096
3098 double dt_align; /* use worst case alignment */
3102
3104 double dt_align; /* use worst case alignment */
3108
3109typedef int (*launch_t)(int gtid);
3110
3111/* Minimum number of ARGV entries to malloc if necessary */
3112#define KMP_MIN_MALLOC_ARGV_ENTRIES 100
3113
3114// Set up how many argv pointers will fit in cache lines containing
3115// t_inline_argv. Historically, we have supported at least 96 bytes. Using a
3116// larger value for more space between the primary write/worker read section and
3117// read/write by all section seems to buy more performance on EPCC PARALLEL.
3118#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3119#define KMP_INLINE_ARGV_BYTES \
3120 (4 * CACHE_LINE - \
3121 ((3 * KMP_PTR_SKIP + 2 * sizeof(int) + 2 * sizeof(kmp_int8) + \
3122 sizeof(kmp_int16) + sizeof(kmp_uint32)) % \
3123 CACHE_LINE))
3124#else
3125#define KMP_INLINE_ARGV_BYTES \
3126 (2 * CACHE_LINE - ((3 * KMP_PTR_SKIP + 2 * sizeof(int)) % CACHE_LINE))
3127#endif
3128#define KMP_INLINE_ARGV_ENTRIES (int)(KMP_INLINE_ARGV_BYTES / KMP_PTR_SKIP)
3129
3131 // Synchronization Data
3132 // ---------------------------------------------------------------------------
3135 std::atomic<int> t_construct; // count of single directive encountered by team
3136 char pad[sizeof(kmp_lock_t)]; // padding to maintain performance on big iron
3137
3138 // [0] - parallel / [1] - worksharing task reduction data shared by taskgroups
3139 std::atomic<void *> t_tg_reduce_data[2]; // to support task modifier
3140 std::atomic<int> t_tg_fini_counter[2]; // sync end of task reductions
3141
3142 // Primary thread only
3143 // ---------------------------------------------------------------------------
3144 KMP_ALIGN_CACHE int t_master_tid; // tid of primary thread in parent team
3145 int t_master_this_cons; // "this_construct" single counter of primary thread
3146 // in parent team
3147 ident_t *t_ident; // if volatile, have to change too much other crud to
3148 // volatile too
3149 kmp_team_p *t_parent; // parent team
3150 kmp_team_p *t_next_pool; // next free team in the team pool
3151 kmp_disp_t *t_dispatch; // thread's dispatch data
3152 kmp_task_team_t *t_task_team[2]; // Task team struct; switch between 2
3153 kmp_proc_bind_t t_proc_bind; // bind type for par region
3154 int t_primary_task_state; // primary thread's task state saved
3155#if USE_ITT_BUILD
3156 kmp_uint64 t_region_time; // region begin timestamp
3157#endif /* USE_ITT_BUILD */
3158
3159 // Primary thread write, workers read
3160 // --------------------------------------------------------------------------
3163 int t_nproc; // number of threads in team
3165 launch_t t_invoke; // procedure to launch the microtask
3166
3167#if OMPT_SUPPORT
3168 ompt_team_info_t ompt_team_info;
3169 ompt_lw_taskteam_t *ompt_serialized_team_info;
3170#endif
3171
3172#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3173 kmp_int8 t_fp_control_saved;
3174 kmp_int8 t_pad2b;
3175 kmp_int16 t_x87_fpu_control_word; // FP control regs
3176 kmp_uint32 t_mxcsr;
3177#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
3178
3179 void *t_inline_argv[KMP_INLINE_ARGV_ENTRIES];
3180
3183 *t_implicit_task_taskdata; // Taskdata for the thread's implicit task
3184 int t_level; // nested parallel level
3185
3187 int t_max_nproc; // max threads this team can handle (dynamically expandable)
3188 int t_serialized; // levels deep of serialized teams
3189 dispatch_shared_info_t *t_disp_buffer; // buffers for dispatch system
3190 int t_id; // team's id, assigned by debugger.
3191 int t_active_level; // nested active parallel level
3192 kmp_r_sched_t t_sched; // run-time schedule for the team
3193#if KMP_AFFINITY_SUPPORTED
3194 int t_first_place; // first & last place in parent thread's partition.
3195 int t_last_place; // Restore these values to primary thread after par region.
3196#endif // KMP_AFFINITY_SUPPORTED
3198 int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via
3199 // omp_set_num_threads() call
3201
3202// Read/write by workers as well
3203#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
3204 // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf
3205 // regression of epcc 'parallel' and 'barrier' on fxe256lin01. This extra
3206 // padding serves to fix the performance of epcc 'parallel' and 'barrier' when
3207 // CACHE_LINE=64. TODO: investigate more and get rid if this padding.
3208 char dummy_padding[1024];
3209#endif
3210 // Internal control stack for additional nested teams.
3212 // for SERIALIZED teams nested 2 or more levels deep
3213 // typed flag to store request state of cancellation
3214 std::atomic<kmp_int32> t_cancel_request;
3215 int t_master_active; // save on fork, restore on join
3216 void *t_copypriv_data; // team specific pointer to copyprivate data array
3217#if KMP_OS_WINDOWS
3218 std::atomic<kmp_uint32> t_copyin_counter;
3219#endif
3220#if USE_ITT_BUILD
3221 void *t_stack_id; // team specific stack stitching id (for ittnotify)
3222#endif /* USE_ITT_BUILD */
3223 distributedBarrier *b; // Distributed barrier data associated with team
3226
3227// Assert that the list structure fits and aligns within
3228// the double task team pointer
3231 alignof(kmp_task_team_list_t));
3232
3235 double t_align; /* use worst case alignment */
3237};
3238
3240 double dt_align; /* use worst case alignment */
3244
3245typedef struct kmp_base_global {
3246 /* cache-aligned */
3248
3249 /* non cache-aligned */
3250 volatile int g_abort;
3251 volatile int g_done;
3252
3256
3259 double g_align; /* use worst case alignment */
3262
3263typedef struct kmp_base_root {
3264 // TODO: GEH - combine r_active with r_in_parallel then r_active ==
3265 // (r_in_parallel>= 0)
3266 // TODO: GEH - then replace r_active with t_active_levels if we can to reduce
3267 // the synch overhead or keeping r_active
3268 volatile int r_active; /* TRUE if some region in a nest has > 1 thread */
3269 // keeps a count of active parallel regions per root
3270 std::atomic<int> r_in_parallel;
3271 // GEH: This is misnamed, should be r_active_levels
3276 volatile int r_begin;
3277 int r_blocktime; /* blocktime for this root and descendants */
3278#if KMP_AFFINITY_SUPPORTED
3279 int r_affinity_assigned;
3280#endif // KMP_AFFINITY_SUPPORTED
3282
3285 double r_align; /* use worst case alignment */
3288
3291};
3292
3293// This list type exists to hold old __kmp_threads arrays so that
3294// old references to them may complete while reallocation takes place when
3295// expanding the array. The items in this list are kept alive until library
3296// shutdown.
3301
3302/* ------------------------------------------------------------------------ */
3303
3304extern int __kmp_settings;
3306#if USE_ITT_BUILD
3307extern int __kmp_forkjoin_frames;
3308extern int __kmp_forkjoin_frames_mode;
3309#endif
3311extern int __kmp_determ_red;
3312
3313#ifdef KMP_DEBUG
3314extern int kmp_a_debug;
3315extern int kmp_b_debug;
3316extern int kmp_c_debug;
3317extern int kmp_d_debug;
3318extern int kmp_e_debug;
3319extern int kmp_f_debug;
3320#endif /* KMP_DEBUG */
3321
3322/* For debug information logging using rotating buffer */
3323#define KMP_DEBUG_BUF_LINES_INIT 512
3324#define KMP_DEBUG_BUF_LINES_MIN 1
3325
3326#define KMP_DEBUG_BUF_CHARS_INIT 128
3327#define KMP_DEBUG_BUF_CHARS_MIN 2
3328
3329extern int
3330 __kmp_debug_buf; /* TRUE means use buffer, FALSE means print to stderr */
3331extern int __kmp_debug_buf_lines; /* How many lines of debug stored in buffer */
3332extern int
3333 __kmp_debug_buf_chars; /* How many characters allowed per line in buffer */
3334extern int __kmp_debug_buf_atomic; /* TRUE means use atomic update of buffer
3335 entry pointer */
3336
3337extern char *__kmp_debug_buffer; /* Debug buffer itself */
3338extern std::atomic<int> __kmp_debug_count; /* Counter for number of lines
3339 printed in buffer so far */
3340extern int __kmp_debug_buf_warn_chars; /* Keep track of char increase
3341 recommended in warnings */
3342/* end rotating debug buffer */
3343
3344#ifdef KMP_DEBUG
3345extern int __kmp_par_range; /* +1 => only go par for constructs in range */
3346
3347#define KMP_PAR_RANGE_ROUTINE_LEN 1024
3348extern char __kmp_par_range_routine[KMP_PAR_RANGE_ROUTINE_LEN];
3349#define KMP_PAR_RANGE_FILENAME_LEN 1024
3350extern char __kmp_par_range_filename[KMP_PAR_RANGE_FILENAME_LEN];
3351extern int __kmp_par_range_lb;
3352extern int __kmp_par_range_ub;
3353#endif
3354
3355/* For printing out dynamic storage map for threads and teams */
3356extern int
3357 __kmp_storage_map; /* True means print storage map for threads and teams */
3358extern int __kmp_storage_map_verbose; /* True means storage map includes
3359 placement info */
3361
3362#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3363extern kmp_cpuinfo_t __kmp_cpuinfo;
3364static inline bool __kmp_is_hybrid_cpu() { return __kmp_cpuinfo.flags.hybrid; }
3365#elif KMP_OS_DARWIN && KMP_ARCH_AARCH64
3366static inline bool __kmp_is_hybrid_cpu() { return true; }
3367#else
3368static inline bool __kmp_is_hybrid_cpu() { return false; }
3369#endif
3370
3371extern volatile int __kmp_init_serial;
3372extern volatile int __kmp_init_gtid;
3373extern volatile int __kmp_init_common;
3374extern volatile int __kmp_need_register_serial;
3375extern volatile int __kmp_init_middle;
3376extern volatile int __kmp_init_parallel;
3377#if KMP_USE_MONITOR
3378extern volatile int __kmp_init_monitor;
3379#endif
3380extern volatile int __kmp_init_user_locks;
3381extern volatile int __kmp_init_hidden_helper_threads;
3382extern int __kmp_init_counter;
3383extern int __kmp_root_counter;
3384extern int __kmp_version;
3385
3386/* list of address of allocated caches for commons */
3388
3389/* Barrier algorithm types and options */
3400extern char const *__kmp_barrier_type_name[bs_last_barrier];
3401extern char const *__kmp_barrier_pattern_name[bp_last_bar];
3402
3403/* Global Locks */
3404extern kmp_bootstrap_lock_t __kmp_initz_lock; /* control initialization */
3405extern kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
3408 __kmp_exit_lock; /* exit() is not always thread-safe */
3409#if KMP_USE_MONITOR
3411 __kmp_monitor_lock; /* control monitor thread creation */
3412#endif
3414 __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and
3415 __kmp_threads expansion to co-exist */
3416
3417extern kmp_lock_t __kmp_global_lock; /* control OS/global access */
3418extern kmp_queuing_lock_t __kmp_dispatch_lock; /* control dispatch access */
3419extern kmp_lock_t __kmp_debug_lock; /* control I/O access for KMP_DEBUG */
3420
3421extern enum library_type __kmp_library;
3422
3423extern enum sched_type __kmp_sched; /* default runtime scheduling */
3424extern enum sched_type __kmp_static; /* default static scheduling method */
3425extern enum sched_type __kmp_guided; /* default guided scheduling method */
3426extern enum sched_type __kmp_auto; /* default auto scheduling method */
3427extern int __kmp_chunk; /* default runtime chunk size */
3428extern int __kmp_force_monotonic; /* whether monotonic scheduling forced */
3429
3430extern size_t __kmp_stksize; /* stack size per thread */
3431#if KMP_USE_MONITOR
3432extern size_t __kmp_monitor_stksize; /* stack size for monitor thread */
3433#endif
3434extern size_t __kmp_stkoffset; /* stack offset per thread */
3435extern int __kmp_stkpadding; /* Should we pad root thread(s) stack */
3436
3437extern size_t
3438 __kmp_malloc_pool_incr; /* incremental size of pool for kmp_malloc() */
3439extern int __kmp_env_stksize; /* was KMP_STACKSIZE specified? */
3440extern int __kmp_env_blocktime; /* was KMP_BLOCKTIME specified? */
3441extern int __kmp_env_checks; /* was KMP_CHECKS specified? */
3442extern int __kmp_env_consistency_check; // was KMP_CONSISTENCY_CHECK specified?
3443extern int __kmp_generate_warnings; /* should we issue warnings? */
3444extern int __kmp_reserve_warn; /* have we issued reserve_threads warning? */
3445
3446#ifdef DEBUG_SUSPEND
3447extern int __kmp_suspend_count; /* count inside __kmp_suspend_template() */
3448#endif
3449
3455
3456/* ------------------------------------------------------------------------- */
3457extern int __kmp_allThreadsSpecified;
3458
3459extern size_t __kmp_align_alloc;
3460/* following data protected by initialization routines */
3461extern int __kmp_xproc; /* number of processors in the system */
3462extern int __kmp_avail_proc; /* number of processors available to the process */
3463extern size_t __kmp_sys_min_stksize; /* system-defined minimum stack size */
3464extern int __kmp_sys_max_nth; /* system-imposed maximum number of threads */
3465// maximum total number of concurrently-existing threads on device
3466extern int __kmp_max_nth;
3467// maximum total number of concurrently-existing threads in a contention group
3468extern int __kmp_cg_max_nth;
3469extern int __kmp_task_max_nth; // max threads used in a task
3470extern int __kmp_teams_max_nth; // max threads used in a teams construct
3471extern int __kmp_threads_capacity; /* capacity of the arrays __kmp_threads and
3472 __kmp_root */
3473extern int __kmp_dflt_team_nth; /* default number of threads in a parallel
3474 region a la OMP_NUM_THREADS */
3475extern int __kmp_dflt_team_nth_ub; /* upper bound on "" determined at serial
3476 initialization */
3477extern int __kmp_tp_capacity; /* capacity of __kmp_threads if threadprivate is
3478 used (fixed) */
3479extern int __kmp_tp_cached; /* whether threadprivate cache has been created
3480 (__kmpc_threadprivate_cached()) */
3481extern int __kmp_dflt_blocktime; /* number of microseconds to wait before
3482 blocking (env setting) */
3483extern char __kmp_blocktime_units; /* 'm' or 'u' to note units specified */
3484extern bool __kmp_wpolicy_passive; /* explicitly set passive wait policy */
3485
3486// Convert raw blocktime from ms to us if needed.
3487static inline void __kmp_aux_convert_blocktime(int *bt) {
3488 if (__kmp_blocktime_units == 'm') {
3489 if (*bt > INT_MAX / 1000) {
3490 *bt = INT_MAX / 1000;
3491 KMP_INFORM(MaxValueUsing, "kmp_set_blocktime(ms)", bt);
3492 }
3493 *bt = *bt * 1000;
3494 }
3495}
3496
3497#if KMP_USE_MONITOR
3498extern int
3499 __kmp_monitor_wakeups; /* number of times monitor wakes up per second */
3500extern int __kmp_bt_intervals; /* number of monitor timestamp intervals before
3501 blocking */
3502#endif
3503#ifdef KMP_ADJUST_BLOCKTIME
3504extern int __kmp_zero_bt; /* whether blocktime has been forced to zero */
3505#endif /* KMP_ADJUST_BLOCKTIME */
3506#ifdef KMP_DFLT_NTH_CORES
3507extern int __kmp_ncores; /* Total number of cores for threads placement */
3508#endif
3509/* Number of millisecs to delay on abort for Intel(R) VTune(TM) tools */
3510extern int __kmp_abort_delay;
3511
3513extern int __kmp_need_register_atfork; /* At initialization, call pthread_atfork
3514 to install fork handler */
3515extern int __kmp_gtid_mode; /* Method of getting gtid, values:
3516 0 - not set, will be set at runtime
3517 1 - using stack search
3518 2 - dynamic TLS (pthread_getspecific(Linux* OS/OS
3519 X*) or TlsGetValue(Windows* OS))
3520 3 - static TLS (__declspec(thread) __kmp_gtid),
3521 Linux* OS .so only. */
3522extern int
3523 __kmp_adjust_gtid_mode; /* If true, adjust method based on #threads */
3524#ifdef KMP_TDATA_GTID
3525extern KMP_THREAD_LOCAL int __kmp_gtid;
3526#endif
3527extern int __kmp_tls_gtid_min; /* #threads below which use sp search for gtid */
3528extern int __kmp_foreign_tp; // If true, separate TP var for each foreign thread
3529#if KMP_ARCH_X86 || KMP_ARCH_X86_64
3530extern int __kmp_inherit_fp_control; // copy fp creg(s) parent->workers at fork
3531extern kmp_int16 __kmp_init_x87_fpu_control_word; // init thread's FP ctrl reg
3532extern kmp_uint32 __kmp_init_mxcsr; /* init thread's mxscr */
3533#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
3534
3535// max_active_levels for nested parallelism enabled by default via
3536// OMP_MAX_ACTIVE_LEVELS, OMP_NESTED, OMP_NUM_THREADS, and OMP_PROC_BIND
3538// Indicates whether value of __kmp_dflt_max_active_levels was already
3539// explicitly set by OMP_MAX_ACTIVE_LEVELS or OMP_NESTED=false
3541extern int __kmp_dispatch_num_buffers; /* max possible dynamic loops in
3542 concurrent execution per team */
3543#if KMP_NESTED_HOT_TEAMS
3544extern int __kmp_hot_teams_mode;
3545extern int __kmp_hot_teams_max_level;
3546#endif
3547
3548#if KMP_OS_LINUX
3549extern enum clock_function_type __kmp_clock_function;
3550extern int __kmp_clock_function_param;
3551#endif /* KMP_OS_LINUX */
3552
3553#if KMP_MIC_SUPPORTED
3554extern enum mic_type __kmp_mic_type;
3555#endif
3556
3557#ifdef USE_LOAD_BALANCE
3558extern double __kmp_load_balance_interval; // load balance algorithm interval
3559#endif /* USE_LOAD_BALANCE */
3560
3561#if KMP_USE_ADAPTIVE_LOCKS
3562
3563// Parameters for the speculative lock backoff system.
3564struct kmp_adaptive_backoff_params_t {
3565 // Number of soft retries before it counts as a hard retry.
3566 kmp_uint32 max_soft_retries;
3567 // Badness is a bit mask : 0,1,3,7,15,... on each hard failure we move one to
3568 // the right
3569 kmp_uint32 max_badness;
3570};
3571
3572extern kmp_adaptive_backoff_params_t __kmp_adaptive_backoff_params;
3573
3574#if KMP_DEBUG_ADAPTIVE_LOCKS
3575extern const char *__kmp_speculative_statsfile;
3576#endif
3577
3578#endif // KMP_USE_ADAPTIVE_LOCKS
3579
3580extern int __kmp_display_env; /* TRUE or FALSE */
3581extern int __kmp_display_env_verbose; /* TRUE if OMP_DISPLAY_ENV=VERBOSE */
3582extern int __kmp_omp_cancellation; /* TRUE or FALSE */
3583extern int __kmp_nteams;
3584extern int __kmp_teams_thread_limit;
3585
3586/* ------------------------------------------------------------------------- */
3587
3588/* the following are protected by the fork/join lock */
3589/* write: lock read: anytime */
3590extern kmp_info_t **__kmp_threads; /* Descriptors for the threads */
3591/* Holds old arrays of __kmp_threads until library shutdown */
3593/* read/write: lock */
3594extern volatile kmp_team_t *__kmp_team_pool;
3595extern volatile kmp_info_t *__kmp_thread_pool;
3597
3598// total num threads reachable from some root thread including all root threads
3599extern volatile int __kmp_nth;
3600/* total number of threads reachable from some root thread including all root
3601 threads, and those in the thread pool */
3602extern volatile int __kmp_all_nth;
3603extern std::atomic<int> __kmp_thread_pool_active_nth;
3604
3605extern kmp_root_t **__kmp_root; /* root of thread hierarchy */
3606/* end data protected by fork/join lock */
3607/* ------------------------------------------------------------------------- */
3608
3609#define __kmp_get_gtid() __kmp_get_global_thread_id()
3610#define __kmp_entry_gtid() __kmp_get_global_thread_id_reg()
3611#define __kmp_get_tid() (__kmp_tid_from_gtid(__kmp_get_gtid()))
3612#define __kmp_get_team() (__kmp_threads[(__kmp_get_gtid())]->th.th_team)
3613#define __kmp_get_thread() (__kmp_thread_from_gtid(__kmp_get_gtid()))
3614
3615// AT: Which way is correct?
3616// AT: 1. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team -> t.t_nproc;
3617// AT: 2. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team_nproc;
3618#define __kmp_get_team_num_threads(gtid) \
3619 (__kmp_threads[(gtid)]->th.th_team->t.t_nproc)
3620
3621static inline bool KMP_UBER_GTID(int gtid) {
3624 return (gtid >= 0 && __kmp_root[gtid] && __kmp_threads[gtid] &&
3625 __kmp_threads[gtid] == __kmp_root[gtid]->r.r_uber_thread);
3626}
3627
3628static inline int __kmp_tid_from_gtid(int gtid) {
3629 KMP_DEBUG_ASSERT(gtid >= 0);
3630 return __kmp_threads[gtid]->th.th_info.ds.ds_tid;
3631}
3632
3633static inline int __kmp_gtid_from_tid(int tid, const kmp_team_t *team) {
3634 KMP_DEBUG_ASSERT(tid >= 0 && team);
3635 return team->t.t_threads[tid]->th.th_info.ds.ds_gtid;
3636}
3637
3638static inline int __kmp_gtid_from_thread(const kmp_info_t *thr) {
3639 KMP_DEBUG_ASSERT(thr);
3640 return thr->th.th_info.ds.ds_gtid;
3641}
3642
3643static inline kmp_info_t *__kmp_thread_from_gtid(int gtid) {
3644 KMP_DEBUG_ASSERT(gtid >= 0);
3645 return __kmp_threads[gtid];
3646}
3647
3648static inline kmp_team_t *__kmp_team_from_gtid(int gtid) {
3649 KMP_DEBUG_ASSERT(gtid >= 0);
3650 return __kmp_threads[gtid]->th.th_team;
3651}
3652
3653static inline void __kmp_assert_valid_gtid(kmp_int32 gtid) {
3654 if (UNLIKELY(gtid < 0 || gtid >= __kmp_threads_capacity))
3655 KMP_FATAL(ThreadIdentInvalid);
3656}
3657
3658#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
3659extern int __kmp_user_level_mwait; // TRUE or FALSE; from KMP_USER_LEVEL_MWAIT
3660extern int __kmp_umwait_enabled; // Runtime check if user-level mwait enabled
3661extern int __kmp_mwait_enabled; // Runtime check if ring3 mwait is enabled
3662extern int __kmp_mwait_hints; // Hints to pass in to mwait
3663#endif
3664
3665#if KMP_HAVE_UMWAIT
3666extern int __kmp_waitpkg_enabled; // Runtime check if waitpkg exists
3667extern int __kmp_tpause_state; // 0 (default), 1=C0.1, 2=C0.2; from KMP_TPAUSE
3668extern int __kmp_tpause_hint; // 1=C0.1 (default), 0=C0.2; from KMP_TPAUSE
3669extern int __kmp_tpause_enabled; // 0 (default), 1 (KMP_TPAUSE is non-zero)
3670#endif
3671
3672/* ------------------------------------------------------------------------- */
3673
3674extern kmp_global_t __kmp_global; /* global status */
3675
3677// For Debugging Support Library
3678extern std::atomic<kmp_int32> __kmp_team_counter;
3679// For Debugging Support Library
3680extern std::atomic<kmp_int32> __kmp_task_counter;
3681
3682#if USE_DEBUGGER
3683#define _KMP_GEN_ID(counter) \
3684 (__kmp_debugging ? KMP_ATOMIC_INC(&counter) + 1 : ~0)
3685#else
3686#define _KMP_GEN_ID(counter) (~0)
3687#endif /* USE_DEBUGGER */
3688
3689#define KMP_GEN_TASK_ID() _KMP_GEN_ID(__kmp_task_counter)
3690#define KMP_GEN_TEAM_ID() _KMP_GEN_ID(__kmp_team_counter)
3691
3692/* ------------------------------------------------------------------------ */
3693
3694extern void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2,
3695 size_t size, char const *format, ...);
3696
3697extern void __kmp_serial_initialize(void);
3698extern void __kmp_middle_initialize(void);
3699extern void __kmp_parallel_initialize(void);
3700
3701extern void __kmp_internal_begin(void);
3702extern void __kmp_internal_end_library(int gtid);
3703extern void __kmp_internal_end_thread(int gtid);
3704extern void __kmp_internal_end_atexit(void);
3705extern void __kmp_internal_end_dtor(void);
3706extern void __kmp_internal_end_dest(void *);
3707
3708extern int __kmp_register_root(int initial_thread);
3709extern void __kmp_unregister_root(int gtid);
3710extern void __kmp_unregister_library(void); // called by __kmp_internal_end()
3711
3712extern int __kmp_ignore_mppbeg(void);
3713extern int __kmp_ignore_mppend(void);
3714
3715extern int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws);
3716extern void __kmp_exit_single(int gtid);
3717
3718extern void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref);
3719extern void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref);
3720
3721#ifdef USE_LOAD_BALANCE
3722extern int __kmp_get_load_balance(int);
3723#endif
3724
3725extern int __kmp_get_global_thread_id(void);
3726extern int __kmp_get_global_thread_id_reg(void);
3727extern void __kmp_exit_thread(int exit_status);
3728extern void __kmp_abort(char const *format, ...);
3729extern void __kmp_abort_thread(void);
3730KMP_NORETURN extern void __kmp_abort_process(void);
3731extern void __kmp_warn(char const *format, ...);
3732
3733extern void __kmp_set_num_threads(int new_nth, int gtid);
3734
3735extern bool __kmp_detect_shm();
3736extern bool __kmp_detect_tmp();
3737
3738// Returns current thread (pointer to kmp_info_t). Current thread *must* be
3739// registered.
3741 int gtid = __kmp_entry_gtid();
3742
3743 return __kmp_threads[gtid];
3744}
3745
3746extern void __kmp_set_max_active_levels(int gtid, int new_max_active_levels);
3747extern int __kmp_get_max_active_levels(int gtid);
3748extern int __kmp_get_ancestor_thread_num(int gtid, int level);
3749extern int __kmp_get_team_size(int gtid, int level);
3750extern void __kmp_set_schedule(int gtid, kmp_sched_t new_sched, int chunk);
3751extern void __kmp_get_schedule(int gtid, kmp_sched_t *sched, int *chunk);
3752
3753extern unsigned short __kmp_get_random(kmp_info_t *thread);
3754extern void __kmp_init_random(kmp_info_t *thread);
3755
3757extern void __kmp_adjust_num_threads(int new_nproc);
3758extern void __kmp_check_stksize(size_t *val);
3759
3760extern void *___kmp_allocate(size_t size KMP_SRC_LOC_DECL);
3761extern void *___kmp_page_allocate(size_t size KMP_SRC_LOC_DECL);
3762extern void ___kmp_free(void *ptr KMP_SRC_LOC_DECL);
3763#define __kmp_allocate(size) ___kmp_allocate((size)KMP_SRC_LOC_CURR)
3764#define __kmp_page_allocate(size) ___kmp_page_allocate((size)KMP_SRC_LOC_CURR)
3765#define __kmp_free(ptr) ___kmp_free((ptr)KMP_SRC_LOC_CURR)
3766
3767#if USE_FAST_MEMORY
3768extern void *___kmp_fast_allocate(kmp_info_t *this_thr,
3769 size_t size KMP_SRC_LOC_DECL);
3770extern void ___kmp_fast_free(kmp_info_t *this_thr, void *ptr KMP_SRC_LOC_DECL);
3771extern void __kmp_free_fast_memory(kmp_info_t *this_thr);
3772extern void __kmp_initialize_fast_memory(kmp_info_t *this_thr);
3773#define __kmp_fast_allocate(this_thr, size) \
3774 ___kmp_fast_allocate((this_thr), (size)KMP_SRC_LOC_CURR)
3775#define __kmp_fast_free(this_thr, ptr) \
3776 ___kmp_fast_free((this_thr), (ptr)KMP_SRC_LOC_CURR)
3777#endif
3778
3779extern void *___kmp_thread_malloc(kmp_info_t *th, size_t size KMP_SRC_LOC_DECL);
3780extern void *___kmp_thread_calloc(kmp_info_t *th, size_t nelem,
3781 size_t elsize KMP_SRC_LOC_DECL);
3782extern void *___kmp_thread_realloc(kmp_info_t *th, void *ptr,
3783 size_t size KMP_SRC_LOC_DECL);
3784extern void ___kmp_thread_free(kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL);
3785#define __kmp_thread_malloc(th, size) \
3786 ___kmp_thread_malloc((th), (size)KMP_SRC_LOC_CURR)
3787#define __kmp_thread_calloc(th, nelem, elsize) \
3788 ___kmp_thread_calloc((th), (nelem), (elsize)KMP_SRC_LOC_CURR)
3789#define __kmp_thread_realloc(th, ptr, size) \
3790 ___kmp_thread_realloc((th), (ptr), (size)KMP_SRC_LOC_CURR)
3791#define __kmp_thread_free(th, ptr) \
3792 ___kmp_thread_free((th), (ptr)KMP_SRC_LOC_CURR)
3793
3794extern void __kmp_push_num_threads(ident_t *loc, int gtid, int num_threads);
3795extern void __kmp_push_num_threads_list(ident_t *loc, int gtid,
3796 kmp_uint32 list_length,
3797 int *num_threads_list);
3798extern void __kmp_set_strict_num_threads(ident_t *loc, int gtid, int sev,
3799 const char *msg);
3800
3801extern void __kmp_push_proc_bind(ident_t *loc, int gtid,
3802 kmp_proc_bind_t proc_bind);
3803extern void __kmp_push_num_teams(ident_t *loc, int gtid, int num_teams,
3804 int num_threads);
3805extern void __kmp_push_num_teams_51(ident_t *loc, int gtid, int num_teams_lb,
3806 int num_teams_ub, int num_threads);
3807
3808extern void __kmp_yield();
3809
3810extern void __kmpc_dispatch_init_4(ident_t *loc, kmp_int32 gtid,
3811 enum sched_type schedule, kmp_int32 lb,
3812 kmp_int32 ub, kmp_int32 st, kmp_int32 chunk);
3813extern void __kmpc_dispatch_init_4u(ident_t *loc, kmp_int32 gtid,
3814 enum sched_type schedule, kmp_uint32 lb,
3815 kmp_uint32 ub, kmp_int32 st,
3816 kmp_int32 chunk);
3817extern void __kmpc_dispatch_init_8(ident_t *loc, kmp_int32 gtid,
3818 enum sched_type schedule, kmp_int64 lb,
3819 kmp_int64 ub, kmp_int64 st, kmp_int64 chunk);
3820extern void __kmpc_dispatch_init_8u(ident_t *loc, kmp_int32 gtid,
3821 enum sched_type schedule, kmp_uint64 lb,
3822 kmp_uint64 ub, kmp_int64 st,
3823 kmp_int64 chunk);
3824
3826 kmp_int32 *p_last, kmp_int32 *p_lb,
3827 kmp_int32 *p_ub, kmp_int32 *p_st);
3829 kmp_int32 *p_last, kmp_uint32 *p_lb,
3830 kmp_uint32 *p_ub, kmp_int32 *p_st);
3832 kmp_int32 *p_last, kmp_int64 *p_lb,
3833 kmp_int64 *p_ub, kmp_int64 *p_st);
3835 kmp_int32 *p_last, kmp_uint64 *p_lb,
3836 kmp_uint64 *p_ub, kmp_int64 *p_st);
3837
3838extern void __kmpc_dispatch_fini_4(ident_t *loc, kmp_int32 gtid);
3839extern void __kmpc_dispatch_fini_8(ident_t *loc, kmp_int32 gtid);
3840extern void __kmpc_dispatch_fini_4u(ident_t *loc, kmp_int32 gtid);
3841extern void __kmpc_dispatch_fini_8u(ident_t *loc, kmp_int32 gtid);
3842
3843extern void __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 gtid);
3844
3845#ifdef KMP_GOMP_COMPAT
3846
3847extern void __kmp_aux_dispatch_init_4(ident_t *loc, kmp_int32 gtid,
3848 enum sched_type schedule, kmp_int32 lb,
3849 kmp_int32 ub, kmp_int32 st,
3850 kmp_int32 chunk, int push_ws);
3851extern void __kmp_aux_dispatch_init_4u(ident_t *loc, kmp_int32 gtid,
3852 enum sched_type schedule, kmp_uint32 lb,
3853 kmp_uint32 ub, kmp_int32 st,
3854 kmp_int32 chunk, int push_ws);
3855extern void __kmp_aux_dispatch_init_8(ident_t *loc, kmp_int32 gtid,
3856 enum sched_type schedule, kmp_int64 lb,
3857 kmp_int64 ub, kmp_int64 st,
3858 kmp_int64 chunk, int push_ws);
3859extern void __kmp_aux_dispatch_init_8u(ident_t *loc, kmp_int32 gtid,
3860 enum sched_type schedule, kmp_uint64 lb,
3861 kmp_uint64 ub, kmp_int64 st,
3862 kmp_int64 chunk, int push_ws);
3863extern void __kmp_aux_dispatch_fini_chunk_4(ident_t *loc, kmp_int32 gtid);
3864extern void __kmp_aux_dispatch_fini_chunk_8(ident_t *loc, kmp_int32 gtid);
3865extern void __kmp_aux_dispatch_fini_chunk_4u(ident_t *loc, kmp_int32 gtid);
3866extern void __kmp_aux_dispatch_fini_chunk_8u(ident_t *loc, kmp_int32 gtid);
3867
3868#endif /* KMP_GOMP_COMPAT */
3869
3875extern kmp_uint32 __kmp_wait_4(kmp_uint32 volatile *spinner, kmp_uint32 checker,
3877 void *obj);
3878extern void __kmp_wait_4_ptr(void *spinner, kmp_uint32 checker,
3879 kmp_uint32 (*pred)(void *, kmp_uint32), void *obj);
3880
3882 int final_spin
3883#if USE_ITT_BUILD
3884 ,
3885 void *itt_sync_obj
3886#endif
3887);
3888extern void __kmp_release_64(kmp_flag_64<> *flag);
3889
3890extern void __kmp_infinite_loop(void);
3891
3892extern void __kmp_cleanup(void);
3893
3894#if KMP_HANDLE_SIGNALS
3895extern int __kmp_handle_signals;
3896extern void __kmp_install_signals(int parallel_init);
3897extern void __kmp_remove_signals(void);
3898#endif
3899
3900extern void __kmp_clear_system_time(void);
3901extern void __kmp_read_system_time(double *delta);
3902
3903extern void __kmp_check_stack_overlap(kmp_info_t *thr);
3904
3905extern void __kmp_expand_host_name(char *buffer, size_t size);
3906extern void __kmp_expand_file_name(char *result, size_t rlen, char *pattern);
3907
3908#if KMP_ARCH_X86 || KMP_ARCH_X86_64 || (KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM))
3909extern void
3910__kmp_initialize_system_tick(void); /* Initialize timer tick value */
3911#endif
3912
3913extern void
3914__kmp_runtime_initialize(void); /* machine specific initialization */
3915extern void __kmp_runtime_destroy(void);
3916
3917#if KMP_AFFINITY_SUPPORTED
3918extern char *__kmp_affinity_print_mask(char *buf, int buf_len,
3919 kmp_affin_mask_t *mask);
3920extern kmp_str_buf_t *__kmp_affinity_str_buf_mask(kmp_str_buf_t *buf,
3921 kmp_affin_mask_t *mask);
3922extern void __kmp_affinity_initialize(kmp_affinity_t &affinity);
3923extern void __kmp_affinity_uninitialize(void);
3924extern void __kmp_affinity_set_init_mask(
3925 int gtid, int isa_root); /* set affinity according to KMP_AFFINITY */
3926void __kmp_affinity_bind_init_mask(int gtid);
3927extern void __kmp_affinity_bind_place(int gtid);
3928extern void __kmp_affinity_determine_capable(const char *env_var);
3929extern int __kmp_aux_set_affinity(void **mask);
3930extern int __kmp_aux_get_affinity(void **mask);
3931extern int __kmp_aux_get_affinity_max_proc();
3932extern int __kmp_aux_set_affinity_mask_proc(int proc, void **mask);
3933extern int __kmp_aux_unset_affinity_mask_proc(int proc, void **mask);
3934extern int __kmp_aux_get_affinity_mask_proc(int proc, void **mask);
3935extern void __kmp_balanced_affinity(kmp_info_t *th, int team_size);
3936#if KMP_WEIGHTED_ITERATIONS_SUPPORTED
3937extern int __kmp_get_first_osid_with_ecore(void);
3938#endif
3939#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
3940 KMP_OS_AIX
3941extern int kmp_set_thread_affinity_mask_initial(void);
3942#endif
3943static inline void __kmp_assign_root_init_mask() {
3944 int gtid = __kmp_entry_gtid();
3945 kmp_root_t *r = __kmp_threads[gtid]->th.th_root;
3946 if (r->r.r_uber_thread == __kmp_threads[gtid] && !r->r.r_affinity_assigned) {
3947 __kmp_affinity_set_init_mask(gtid, /*isa_root=*/TRUE);
3948 __kmp_affinity_bind_init_mask(gtid);
3949 r->r.r_affinity_assigned = TRUE;
3950 }
3951}
3952static inline void __kmp_reset_root_init_mask(int gtid) {
3953 if (!KMP_AFFINITY_CAPABLE())
3954 return;
3955 kmp_info_t *th = __kmp_threads[gtid];
3956 kmp_root_t *r = th->th.th_root;
3957 if (r->r.r_uber_thread == th && r->r.r_affinity_assigned) {
3958 __kmp_set_system_affinity(__kmp_affin_origMask, FALSE);
3959 KMP_CPU_COPY(th->th.th_affin_mask, __kmp_affin_origMask);
3960 r->r.r_affinity_assigned = FALSE;
3961 }
3962}
3963#else /* KMP_AFFINITY_SUPPORTED */
3964#define __kmp_assign_root_init_mask() /* Nothing */
3965static inline void __kmp_reset_root_init_mask(int gtid) {}
3966#endif /* KMP_AFFINITY_SUPPORTED */
3967// No need for KMP_AFFINITY_SUPPORTED guard as only one field in the
3968// format string is for affinity, so platforms that do not support
3969// affinity can still use the other fields, e.g., %n for num_threads
3970extern size_t __kmp_aux_capture_affinity(int gtid, const char *format,
3971 kmp_str_buf_t *buffer);
3972extern void __kmp_aux_display_affinity(int gtid, const char *format);
3973
3974extern void __kmp_cleanup_hierarchy();
3975extern void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar);
3976
3977#if KMP_USE_FUTEX
3978
3979extern int __kmp_futex_determine_capable(void);
3980
3981#endif // KMP_USE_FUTEX
3982
3983extern void __kmp_gtid_set_specific(int gtid);
3984extern int __kmp_gtid_get_specific(void);
3985
3986extern double __kmp_read_cpu_time(void);
3987
3988extern int __kmp_read_system_info(struct kmp_sys_info *info);
3989
3990#if KMP_USE_MONITOR
3991extern void __kmp_create_monitor(kmp_info_t *th);
3992#endif
3993
3994extern void *__kmp_launch_thread(kmp_info_t *thr);
3995
3996extern void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size);
3997
3998#if KMP_OS_WINDOWS
3999extern int __kmp_still_running(kmp_info_t *th);
4000extern int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val);
4001extern void __kmp_free_handle(kmp_thread_t tHandle);
4002#endif
4003
4004#if KMP_USE_MONITOR
4005extern void __kmp_reap_monitor(kmp_info_t *th);
4006#endif
4007extern void __kmp_reap_worker(kmp_info_t *th);
4008extern void __kmp_terminate_thread(int gtid);
4009
4010extern int __kmp_try_suspend_mx(kmp_info_t *th);
4011extern void __kmp_lock_suspend_mx(kmp_info_t *th);
4012extern void __kmp_unlock_suspend_mx(kmp_info_t *th);
4013
4014extern void __kmp_elapsed(double *);
4015extern void __kmp_elapsed_tick(double *);
4016
4017extern void __kmp_enable(int old_state);
4018extern void __kmp_disable(int *old_state);
4019
4020extern void __kmp_thread_sleep(int millis);
4021
4022extern void __kmp_common_initialize(void);
4023extern void __kmp_common_destroy(void);
4024extern void __kmp_common_destroy_gtid(int gtid);
4025
4026#if KMP_OS_UNIX
4027extern void __kmp_register_atfork(void);
4028#endif
4029extern void __kmp_suspend_initialize(void);
4032
4034 int tid);
4035extern kmp_team_t *
4036__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
4037#if OMPT_SUPPORT
4038 ompt_data_t ompt_parallel_data,
4039#endif
4040 kmp_proc_bind_t proc_bind, kmp_internal_control_t *new_icvs,
4041 int argc USE_NESTED_HOT_ARG(kmp_info_t *thr));
4042extern void __kmp_free_thread(kmp_info_t *);
4046
4047/* ------------------------------------------------------------------------ */
4048
4051
4052KMP_EXPORT void *kmpc_malloc(size_t size);
4053KMP_EXPORT void *kmpc_aligned_malloc(size_t size, size_t alignment);
4054KMP_EXPORT void *kmpc_calloc(size_t nelem, size_t elsize);
4055KMP_EXPORT void *kmpc_realloc(void *ptr, size_t size);
4056KMP_EXPORT void kmpc_free(void *ptr);
4057
4058/* declarations for internal use */
4059
4060extern int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,
4061 size_t reduce_size, void *reduce_data,
4062 void (*reduce)(void *, void *));
4063extern void __kmp_end_split_barrier(enum barrier_type bt, int gtid);
4064extern int __kmp_barrier_gomp_cancel(int gtid);
4065
4066/*!
4067 * Tell the fork call which compiler generated the fork call, and therefore how
4068 * to deal with the call.
4069 */
4071 fork_context_gnu, /**< Called from GNU generated code, so must not invoke the
4072 microtask internally. */
4073 fork_context_intel, /**< Called from Intel generated code. */
4076extern int __kmp_fork_call(ident_t *loc, int gtid,
4077 enum fork_context_e fork_context, kmp_int32 argc,
4079 kmp_va_list ap);
4080
4081extern void __kmp_join_call(ident_t *loc, int gtid
4082#if OMPT_SUPPORT
4083 ,
4084 enum fork_context_e fork_context
4085#endif
4086 ,
4087 int exit_teams = 0);
4088
4089extern void __kmp_serialized_parallel(ident_t *id, kmp_int32 gtid);
4090extern void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team);
4091extern void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team);
4092extern int __kmp_invoke_task_func(int gtid);
4093extern void __kmp_run_before_invoked_task(int gtid, int tid,
4094 kmp_info_t *this_thr,
4095 kmp_team_t *team);
4096extern void __kmp_run_after_invoked_task(int gtid, int tid,
4097 kmp_info_t *this_thr,
4098 kmp_team_t *team);
4099
4100// should never have been exported
4102extern int __kmp_invoke_teams_master(int gtid);
4103extern void __kmp_teams_master(int gtid);
4104extern int __kmp_aux_get_team_num();
4105extern int __kmp_aux_get_num_teams();
4106extern void __kmp_save_internal_controls(kmp_info_t *thread);
4107extern void __kmp_user_set_library(enum library_type arg);
4108extern void __kmp_aux_set_library(enum library_type arg);
4109extern void __kmp_aux_set_stacksize(size_t arg);
4110extern void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid);
4111extern void __kmp_aux_set_defaults(char const *str, size_t len);
4112
4113/* Functions called from __kmp_aux_env_initialize() in kmp_settings.cpp */
4114void kmpc_set_blocktime(int arg);
4115void ompc_set_nested(int flag);
4116void ompc_set_dynamic(int flag);
4117void ompc_set_num_threads(int arg);
4118
4119extern void __kmp_push_current_task_to_thread(kmp_info_t *this_thr,
4120 kmp_team_t *team, int tid);
4121extern void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr);
4122extern kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
4123 kmp_tasking_flags_t *flags,
4124 size_t sizeof_kmp_task_t,
4125 size_t sizeof_shareds,
4127extern void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
4128 kmp_team_t *team, int tid,
4129 int set_curr_task);
4130extern void __kmp_finish_implicit_task(kmp_info_t *this_thr);
4131extern void __kmp_free_implicit_task(kmp_info_t *this_thr);
4132
4134 int gtid,
4135 kmp_task_t *task);
4137
4138extern void __kmp_free_task_team(kmp_info_t *thread,
4139 kmp_task_team_t *task_team);
4140extern void __kmp_reap_task_teams(void);
4141extern void __kmp_push_task_team_node(kmp_info_t *thread, kmp_team_t *team);
4142extern void __kmp_pop_task_team_node(kmp_info_t *thread, kmp_team_t *team);
4143extern void __kmp_wait_to_unref_task_teams(void);
4144extern void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team);
4145extern void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team);
4146extern void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team
4147#if USE_ITT_BUILD
4148 ,
4149 void *itt_sync_obj
4150#endif /* USE_ITT_BUILD */
4151 ,
4152 int wait = 1);
4153extern void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread,
4154 int gtid);
4155#if KMP_DEBUG
4156#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr) \
4157 KMP_DEBUG_ASSERT( \
4158 __kmp_tasking_mode != tskm_task_teams || team->t.t_nproc == 1 || \
4159 thr->th.th_task_team == team->t.t_task_team[thr->th.th_task_state])
4160#else
4161#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr) /* Nothing */
4162#endif
4163
4164extern int __kmp_is_address_mapped(void *addr);
4166
4167#if KMP_OS_UNIX
4168extern int __kmp_read_from_file(char const *path, char const *format, ...);
4169#endif
4170
4171/* ------------------------------------------------------------------------ */
4172//
4173// Assembly routines that have no compiler intrinsic replacement
4174//
4175
4176extern int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int npr, int argc,
4177 void *argv[]
4178#if OMPT_SUPPORT
4179 ,
4180 void **exit_frame_ptr
4181#endif
4182);
4183
4184/* ------------------------------------------------------------------------ */
4185
4188
4190 kmpc_ctor_vec ctor,
4191 kmpc_cctor_vec cctor,
4192 kmpc_dtor_vec dtor,
4193 size_t vector_length);
4195 kmpc_ctor ctor, kmpc_cctor cctor,
4196 kmpc_dtor dtor);
4198 void *data, size_t size);
4199
4204
4207 kmpc_micro microtask, ...);
4210 void *args);
4211
4214
4216KMP_EXPORT void __kmpc_barrier(ident_t *, kmp_int32 global_tid);
4218KMP_EXPORT void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
4220 kmp_int32 filter);
4221KMP_EXPORT void __kmpc_end_masked(ident_t *, kmp_int32 global_tid);
4222KMP_EXPORT void __kmpc_ordered(ident_t *, kmp_int32 global_tid);
4224KMP_EXPORT void __kmpc_critical(ident_t *, kmp_int32 global_tid,
4229 kmp_critical_name *, uint32_t hint);
4230
4233
4235 kmp_int32 global_tid);
4236
4238KMP_EXPORT void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
4239
4242 kmp_int32 numberOfSections);
4244
4246 kmp_int32 schedtype, kmp_int32 *plastiter,
4247 kmp_int *plower, kmp_int *pupper,
4248 kmp_int *pstride, kmp_int incr,
4249 kmp_int chunk);
4250
4252
4254 size_t cpy_size, void *cpy_data,
4255 void (*cpy_func)(void *, void *),
4256 kmp_int32 didit);
4257
4259 void *cpy_data);
4260
4261extern void KMPC_SET_NUM_THREADS(int arg);
4262extern void KMPC_SET_DYNAMIC(int flag);
4263extern void KMPC_SET_NESTED(int flag);
4264
4265/* OMP 3.0 tasking interface routines */
4267 kmp_task_t *new_task);
4269 kmp_int32 flags,
4270 size_t sizeof_kmp_task_t,
4271 size_t sizeof_shareds,
4274 ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
4275 size_t sizeof_shareds, kmp_routine_entry_t task_entry, kmp_int64 device_id);
4277 kmp_task_t *task);
4279 kmp_task_t *task);
4281 kmp_task_t *new_task);
4284 int end_part);
4285
4286#if TASK_UNUSED
4287void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task);
4288void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid,
4289 kmp_task_t *task);
4290#endif // TASK_UNUSED
4291
4292/* ------------------------------------------------------------------------ */
4293
4294KMP_EXPORT void __kmpc_taskgroup(ident_t *loc, int gtid);
4296
4298 ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 ndeps,
4299 kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
4300 kmp_depend_info_t *noalias_dep_list);
4301
4303
4305
4307 kmp_int32 ndeps,
4308 kmp_depend_info_t *dep_list,
4309 kmp_int32 ndeps_noalias,
4310 kmp_depend_info_t *noalias_dep_list);
4311/* __kmpc_omp_taskwait_deps_51 : Function for OpenMP 5.1 nowait clause.
4312 * Placeholder for taskwait with nowait clause.*/
4314 kmp_int32 ndeps,
4315 kmp_depend_info_t *dep_list,
4316 kmp_int32 ndeps_noalias,
4317 kmp_depend_info_t *noalias_dep_list,
4318 kmp_int32 has_no_wait);
4319
4320extern kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
4321 bool serialize_immediate);
4322
4324 kmp_int32 cncl_kind);
4326 kmp_int32 cncl_kind);
4328KMP_EXPORT int __kmp_get_cancellation_status(int cancel_kind);
4329
4333 kmp_int32 if_val, kmp_uint64 *lb,
4334 kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup,
4335 kmp_int32 sched, kmp_uint64 grainsize,
4336 void *task_dup);
4338 kmp_task_t *task, kmp_int32 if_val,
4339 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4340 kmp_int32 nogroup, kmp_int32 sched,
4341 kmp_uint64 grainsize, kmp_int32 modifier,
4342 void *task_dup);
4343KMP_EXPORT void *__kmpc_task_reduction_init(int gtid, int num_data, void *data);
4344KMP_EXPORT void *__kmpc_taskred_init(int gtid, int num_data, void *data);
4345KMP_EXPORT void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void *d);
4347 int is_ws, int num,
4348 void *data);
4349KMP_EXPORT void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws,
4350 int num, void *data);
4352 int is_ws);
4354 ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 naffins,
4355 kmp_task_affinity_info_t *affin_list);
4356KMP_EXPORT void __kmp_set_num_teams(int num_teams);
4360
4361/* Interface target task integration */
4364
4365/* Lock interface routines (fast versions with gtid passed in) */
4367 void **user_lock);
4369 void **user_lock);
4371 void **user_lock);
4373 void **user_lock);
4374KMP_EXPORT void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock);
4376 void **user_lock);
4378 void **user_lock);
4380 void **user_lock);
4381KMP_EXPORT int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock);
4383 void **user_lock);
4384
4386 void **user_lock, uintptr_t hint);
4388 void **user_lock,
4389 uintptr_t hint);
4390
4391#if OMPX_TASKGRAPH
4392// Taskgraph's Record & Replay mechanism
4393// __kmp_tdg_is_recording: check whether a given TDG is recording
4394// status: the tdg's current status
4395static inline bool __kmp_tdg_is_recording(kmp_tdg_status_t status) {
4396 return status == KMP_TDG_RECORDING;
4397}
4398
4399KMP_EXPORT kmp_int32 __kmpc_start_record_task(ident_t *loc, kmp_int32 gtid,
4400 kmp_int32 input_flags,
4401 kmp_int32 tdg_id);
4402KMP_EXPORT void __kmpc_end_record_task(ident_t *loc, kmp_int32 gtid,
4403 kmp_int32 input_flags, kmp_int32 tdg_id);
4404#endif
4405/* Interface to fast scalable reduce methods routines */
4406
4408 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
4409 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
4414 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
4415 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
4419
4420/* Internal fast reduction routines */
4421
4423 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
4424 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
4426
4427// this function is for testing set/get/determine reduce method
4429
4432
4433// C++ port
4434// missing 'extern "C"' declarations
4435
4439 kmp_int32 num_threads);
4441 kmp_int32 global_tid,
4442 kmp_int32 num_threads,
4443 int severity,
4444 const char *message);
4445
4447 kmp_uint32 list_length,
4448 kmp_int32 *num_threads_list);
4450 ident_t *loc, kmp_int32 global_tid, kmp_uint32 list_length,
4451 kmp_int32 *num_threads_list, int severity, const char *message);
4452
4454 int proc_bind);
4456 kmp_int32 num_teams,
4457 kmp_int32 num_threads);
4459 kmp_int32 thread_limit);
4460/* Function for OpenMP 5.1 num_teams clause */
4462 kmp_int32 num_teams_lb,
4463 kmp_int32 num_teams_ub,
4464 kmp_int32 num_threads);
4466 kmpc_micro microtask, ...);
4467struct kmp_dim { // loop bounds info casted to kmp_int64
4468 kmp_int64 lo; // lower
4469 kmp_int64 up; // upper
4470 kmp_int64 st; // stride
4471};
4473 kmp_int32 num_dims,
4474 const struct kmp_dim *dims);
4476 const kmp_int64 *vec);
4478 const kmp_int64 *vec);
4480
4482 void *data, size_t size,
4483 void ***cache);
4484
4485// The routines below are not exported.
4486// Consider making them 'static' in corresponding source files.
4487void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr,
4488 void *data_addr, size_t pc_size);
4489struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr,
4490 void *data_addr,
4491 size_t pc_size);
4492void __kmp_threadprivate_resize_cache(int newCapacity);
4494
4495// ompc_, kmpc_ entries moved from omp.h.
4496#if KMP_OS_WINDOWS
4497#define KMPC_CONVENTION __cdecl
4498#else
4499#define KMPC_CONVENTION
4500#endif
4501
4502#ifndef __OMP_H
4503typedef enum omp_sched_t {
4507 omp_sched_auto = 4
4510#endif
4511
4522
4528void KMP_EXPAND_NAME(ompc_set_affinity_format)(char const *format);
4529size_t KMP_EXPAND_NAME(ompc_get_affinity_format)(char *buffer, size_t size);
4530void KMP_EXPAND_NAME(ompc_display_affinity)(char const *format);
4531size_t KMP_EXPAND_NAME(ompc_capture_affinity)(char *buffer, size_t buf_size,
4532 char const *format);
4533
4537 tgt_mandatory = 2
4540// Set via OMP_TARGET_OFFLOAD if specified, defaults to tgt_default otherwise
4542extern int __kmpc_get_target_offload();
4543
4544// Constants used in libomptarget
4545#define KMP_DEVICE_DEFAULT -1 // This is libomptarget's default device.
4546#define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices".
4547
4548// OMP Pause Resource
4549
4550// The following enum is used both to set the status in __kmp_pause_status, and
4551// as the internal equivalent of the externally-visible omp_pause_resource_t.
4553 kmp_not_paused = 0, // status is not paused, or, requesting resume
4554 kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause
4555 kmp_hard_paused = 2, // status is hard-paused, or, requesting hard pause
4556 kmp_stop_tool_paused = 3 // requesting stop_tool pause
4558
4559// This stores the pause state of the runtime
4563// Soft resume sets __kmp_pause_status, and wakes up all threads.
4564extern void __kmp_resume_if_soft_paused();
4565// Hard resume simply resets the status to not paused. Library will appear to
4566// be uninitialized after hard pause. Let OMP constructs trigger required
4567// initializations.
4568static inline void __kmp_resume_if_hard_paused() {
4571 }
4572}
4573
4574extern void __kmp_omp_display_env(int verbose);
4575
4576// 1: it is initializing hidden helper team
4577extern volatile int __kmp_init_hidden_helper;
4578// 1: the hidden helper team is done
4579extern volatile int __kmp_hidden_helper_team_done;
4580// 1: enable hidden helper task
4582// Main thread of hidden helper team
4584// Descriptors for the hidden helper threads
4586// Number of hidden helper threads
4588// Number of hidden helper tasks that have not been executed yet
4589extern std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks;
4590
4591extern void __kmp_hidden_helper_initialize();
4602
4603// Check whether a given thread is a hidden helper thread
4604#define KMP_HIDDEN_HELPER_THREAD(gtid) \
4605 ((gtid) >= 1 && (gtid) <= __kmp_hidden_helper_threads_num)
4606
4607#define KMP_HIDDEN_HELPER_WORKER_THREAD(gtid) \
4608 ((gtid) > 1 && (gtid) <= __kmp_hidden_helper_threads_num)
4609
4610#define KMP_HIDDEN_HELPER_MAIN_THREAD(gtid) \
4611 ((gtid) == 1 && (gtid) <= __kmp_hidden_helper_threads_num)
4612
4613#define KMP_HIDDEN_HELPER_TEAM(team) \
4614 (team->t.t_threads[0] == __kmp_hidden_helper_main_thread)
4615
4616// Map a gtid to a hidden helper thread. The first hidden helper thread, a.k.a
4617// main thread, is skipped.
4618#define KMP_GTID_TO_SHADOW_GTID(gtid) \
4619 ((gtid) % (__kmp_hidden_helper_threads_num - 1) + 2)
4620
4621// Return the adjusted gtid value by subtracting from gtid the number
4622// of hidden helper threads. This adjusted value is the gtid the thread would
4623// have received if there were no hidden helper threads.
4624static inline int __kmp_adjust_gtid_for_hidden_helpers(int gtid) {
4625 int adjusted_gtid = gtid;
4626 if (__kmp_hidden_helper_threads_num > 0 && gtid > 0 &&
4627 gtid - __kmp_hidden_helper_threads_num >= 0) {
4628 adjusted_gtid -= __kmp_hidden_helper_threads_num;
4629 }
4630 return adjusted_gtid;
4631}
4632
4633// Support for error directive
4634typedef enum kmp_severity_t {
4636 severity_fatal = 2
4638extern void __kmpc_error(ident_t *loc, int severity, const char *message);
4639
4640// Support for scope directive
4641KMP_EXPORT void __kmpc_scope(ident_t *loc, kmp_int32 gtid, void *reserved);
4642KMP_EXPORT void __kmpc_end_scope(ident_t *loc, kmp_int32 gtid, void *reserved);
4643
4644#ifdef __cplusplus
4645}
4646#endif
4647
4648template <bool C, bool S>
4649extern void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag);
4650template <bool C, bool S>
4651extern void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag);
4652template <bool C, bool S>
4653extern void __kmp_atomic_suspend_64(int th_gtid,
4655extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag);
4656#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
4657template <bool C, bool S>
4658extern void __kmp_mwait_32(int th_gtid, kmp_flag_32<C, S> *flag);
4659template <bool C, bool S>
4660extern void __kmp_mwait_64(int th_gtid, kmp_flag_64<C, S> *flag);
4661template <bool C, bool S>
4662extern void __kmp_atomic_mwait_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag);
4663extern void __kmp_mwait_oncore(int th_gtid, kmp_flag_oncore *flag);
4664#endif
4665template <bool C, bool S>
4666extern void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag);
4667template <bool C, bool S>
4668extern void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag);
4669template <bool C, bool S>
4670extern void __kmp_atomic_resume_64(int target_gtid,
4672extern void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag);
4673
4674template <bool C, bool S>
4676 kmp_flag_32<C, S> *flag, int final_spin,
4677 int *thread_finished,
4678#if USE_ITT_BUILD
4679 void *itt_sync_obj,
4680#endif /* USE_ITT_BUILD */
4681 kmp_int32 is_constrained);
4682template <bool C, bool S>
4684 kmp_flag_64<C, S> *flag, int final_spin,
4685 int *thread_finished,
4686#if USE_ITT_BUILD
4687 void *itt_sync_obj,
4688#endif /* USE_ITT_BUILD */
4689 kmp_int32 is_constrained);
4690template <bool C, bool S>
4693 int final_spin, int *thread_finished,
4694#if USE_ITT_BUILD
4695 void *itt_sync_obj,
4696#endif /* USE_ITT_BUILD */
4697 kmp_int32 is_constrained);
4699 kmp_flag_oncore *flag, int final_spin,
4700 int *thread_finished,
4701#if USE_ITT_BUILD
4702 void *itt_sync_obj,
4703#endif /* USE_ITT_BUILD */
4704 kmp_int32 is_constrained);
4705
4706extern int __kmp_nesting_mode;
4708extern int *__kmp_nesting_nth_level;
4709extern void __kmp_init_nesting_mode();
4710extern void __kmp_set_nesting_mode_threads();
4711
4712/// This class safely opens and closes a C-style FILE* object using RAII
4713/// semantics. There are also methods which allow using stdout or stderr as
4714/// the underlying FILE* object. With the implicit conversion operator to
4715/// FILE*, an object with this type can be used in any function which takes
4716/// a FILE* object e.g., fprintf().
4717/// No close method is needed at use sites.
4719 FILE *f;
4720
4721 void close() {
4722 if (f && f != stdout && f != stderr) {
4723 fclose(f);
4724 f = nullptr;
4725 }
4726 }
4727
4728public:
4729 kmp_safe_raii_file_t() : f(nullptr) {}
4730 kmp_safe_raii_file_t(const char *filename, const char *mode,
4731 const char *env_var = nullptr)
4732 : f(nullptr) {
4733 open(filename, mode, env_var);
4734 }
4738
4739 /// Open filename using mode. This is automatically closed in the destructor.
4740 /// The env_var parameter indicates the environment variable the filename
4741 /// came from if != nullptr.
4742 void open(const char *filename, const char *mode,
4743 const char *env_var = nullptr) {
4744 KMP_ASSERT(!f);
4745 f = fopen(filename, mode);
4746 if (!f) {
4747 int code = errno;
4748 if (env_var) {
4749 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4750 KMP_HNT(CheckEnvVar, env_var, filename), __kmp_msg_null);
4751 } else {
4752 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4754 }
4755 }
4756 }
4757 /// Instead of erroring out, return non-zero when
4758 /// unsuccessful fopen() for any reason
4759 int try_open(const char *filename, const char *mode) {
4760 KMP_ASSERT(!f);
4761 f = fopen(filename, mode);
4762 if (!f)
4763 return errno;
4764 return 0;
4765 }
4766 /// Set the FILE* object to stdout and output there
4767 /// No open call should happen before this call.
4768 void set_stdout() {
4769 KMP_ASSERT(!f);
4770 f = stdout;
4771 }
4772 /// Set the FILE* object to stderr and output there
4773 /// No open call should happen before this call.
4774 void set_stderr() {
4775 KMP_ASSERT(!f);
4776 f = stderr;
4777 }
4778 operator bool() { return bool(f); }
4779 operator FILE *() { return f; }
4780};
4781
4782template <typename SourceType, typename TargetType,
4783 bool isSourceSmaller = (sizeof(SourceType) < sizeof(TargetType)),
4784 bool isSourceEqual = (sizeof(SourceType) == sizeof(TargetType)),
4785 bool isSourceSigned = std::is_signed<SourceType>::value,
4786 bool isTargetSigned = std::is_signed<TargetType>::value>
4787struct kmp_convert {};
4788
4789// Both types are signed; Source smaller
4790template <typename SourceType, typename TargetType>
4791struct kmp_convert<SourceType, TargetType, true, false, true, true> {
4792 static TargetType to(SourceType src) { return (TargetType)src; }
4793};
4794// Source equal
4795template <typename SourceType, typename TargetType>
4796struct kmp_convert<SourceType, TargetType, false, true, true, true> {
4797 static TargetType to(SourceType src) { return src; }
4798};
4799// Source bigger
4800template <typename SourceType, typename TargetType>
4801struct kmp_convert<SourceType, TargetType, false, false, true, true> {
4802 static TargetType to(SourceType src) {
4803 KMP_ASSERT(src <= static_cast<SourceType>(
4804 (std::numeric_limits<TargetType>::max)()));
4805 KMP_ASSERT(src >= static_cast<SourceType>(
4806 (std::numeric_limits<TargetType>::min)()));
4807 return (TargetType)src;
4808 }
4809};
4810
4811// Source signed, Target unsigned
4812// Source smaller
4813template <typename SourceType, typename TargetType>
4814struct kmp_convert<SourceType, TargetType, true, false, true, false> {
4815 static TargetType to(SourceType src) {
4816 KMP_ASSERT(src >= 0);
4817 return (TargetType)src;
4818 }
4819};
4820// Source equal
4821template <typename SourceType, typename TargetType>
4822struct kmp_convert<SourceType, TargetType, false, true, true, false> {
4823 static TargetType to(SourceType src) {
4824 KMP_ASSERT(src >= 0);
4825 return (TargetType)src;
4826 }
4827};
4828// Source bigger
4829template <typename SourceType, typename TargetType>
4830struct kmp_convert<SourceType, TargetType, false, false, true, false> {
4831 static TargetType to(SourceType src) {
4832 KMP_ASSERT(src >= 0);
4833 KMP_ASSERT(src <= static_cast<SourceType>(
4834 (std::numeric_limits<TargetType>::max)()));
4835 return (TargetType)src;
4836 }
4837};
4838
4839// Source unsigned, Target signed
4840// Source smaller
4841template <typename SourceType, typename TargetType>
4842struct kmp_convert<SourceType, TargetType, true, false, false, true> {
4843 static TargetType to(SourceType src) { return (TargetType)src; }
4844};
4845// Source equal
4846template <typename SourceType, typename TargetType>
4847struct kmp_convert<SourceType, TargetType, false, true, false, true> {
4848 static TargetType to(SourceType src) {
4849 KMP_ASSERT(src <= static_cast<SourceType>(
4850 (std::numeric_limits<TargetType>::max)()));
4851 return (TargetType)src;
4852 }
4853};
4854// Source bigger
4855template <typename SourceType, typename TargetType>
4856struct kmp_convert<SourceType, TargetType, false, false, false, true> {
4857 static TargetType to(SourceType src) {
4858 KMP_ASSERT(src <= static_cast<SourceType>(
4859 (std::numeric_limits<TargetType>::max)()));
4860 return (TargetType)src;
4861 }
4862};
4863
4864// Source unsigned, Target unsigned
4865// Source smaller
4866template <typename SourceType, typename TargetType>
4867struct kmp_convert<SourceType, TargetType, true, false, false, false> {
4868 static TargetType to(SourceType src) { return (TargetType)src; }
4869};
4870// Source equal
4871template <typename SourceType, typename TargetType>
4872struct kmp_convert<SourceType, TargetType, false, true, false, false> {
4873 static TargetType to(SourceType src) { return src; }
4874};
4875// Source bigger
4876template <typename SourceType, typename TargetType>
4877struct kmp_convert<SourceType, TargetType, false, false, false, false> {
4878 static TargetType to(SourceType src) {
4879 KMP_ASSERT(src <= static_cast<SourceType>(
4880 (std::numeric_limits<TargetType>::max)()));
4881 return (TargetType)src;
4882 }
4883};
4884
4885template <typename T1, typename T2>
4886static inline void __kmp_type_convert(T1 src, T2 *dest) {
4887 *dest = kmp_convert<T1, T2>::to(src);
4888}
4889
4890#endif /* KMP_H */
char buf[BUFFER_SIZE]
void * target(void *task)
uint8_t kmp_uint8
char bool
int task_entry(kmp_int32 gtid, kmp_task_t *task)
int result[2]
This class safely opens and closes a C-style FILE* object using RAII semantics.
Definition: kmp.h:4718
void set_stdout()
Set the FILE* object to stdout and output there No open call should happen before this call.
Definition: kmp.h:4768
void set_stderr()
Set the FILE* object to stderr and output there No open call should happen before this call.
Definition: kmp.h:4774
kmp_safe_raii_file_t(const kmp_safe_raii_file_t &other)=delete
int try_open(const char *filename, const char *mode)
Instead of erroring out, return non-zero when unsuccessful fopen() for any reason.
Definition: kmp.h:4759
kmp_safe_raii_file_t & operator=(const kmp_safe_raii_file_t &other)=delete
kmp_safe_raii_file_t(const char *filename, const char *mode, const char *env_var=nullptr)
Definition: kmp.h:4730
void open(const char *filename, const char *mode, const char *env_var=nullptr)
Open filename using mode.
Definition: kmp.h:4742
intptr_t kmp_intptr_t
Definition: common.h:12
int32_t kmp_int32
Definition: common.h:9
kmp_int32(*)(kmp_int32, void *) kmp_routine_entry_t
Definition: common.h:11
int64_t kmp_int64
Definition: common.h:10
struct kmp_task kmp_task_t
struct ident ident_t
The ident structure that describes a source location.
union kmp_cmplrdata kmp_cmplrdata_t
@ KMP_IDENT_BARRIER_IMPL_FOR
Definition: kmp.h:220
@ KMP_IDENT_KMPC
Use c-style ident structure.
Definition: kmp.h:209
@ KMP_IDENT_ATOMIC_HINT_CONTENDED
Definition: kmp.h:238
@ KMP_IDENT_BARRIER_IMPL_MASK
Definition: kmp.h:219
@ KMP_IDENT_BARRIER_IMPL_SECTIONS
Definition: kmp.h:221
@ KMP_IDENT_IMB
Use trampoline for internal microtasks.
Definition: kmp.h:207
@ KMP_IDENT_BARRIER_IMPL_WORKSHARE
Definition: kmp.h:224
@ KMP_IDENT_WORK_LOOP
To mark a static loop in OMPT callbacks.
Definition: kmp.h:227
@ KMP_IDENT_BARRIER_IMPL
To Mark implicit barriers.
Definition: kmp.h:218
@ KMP_IDENT_ATOMIC_HINT_UNCONTENDED
Definition: kmp.h:237
@ KMP_IDENT_WORK_SECTIONS
To mark a sections directive in OMPT callbacks.
Definition: kmp.h:229
@ KMP_IDENT_AUTOPAR
Entry point generated by auto-parallelization.
Definition: kmp.h:212
@ KMP_IDENT_ATOMIC_HINT_SPECULATIVE
Definition: kmp.h:240
@ KMP_IDENT_BARRIER_IMPL_SINGLE
Definition: kmp.h:223
@ KMP_IDENT_ATOMIC_HINT_MASK
Atomic hint; bottom four bits as omp_sync_hint_t.
Definition: kmp.h:236
@ KMP_IDENT_WORK_DISTRIBUTE
To mark a distribute construct in OMPT callbacks.
Definition: kmp.h:231
@ KMP_IDENT_OPENMP_SPEC_VERSION_MASK
Definition: kmp.h:241
@ KMP_IDENT_BARRIER_EXPL
To mark a 'barrier' directive in user code.
Definition: kmp.h:216
@ KMP_IDENT_ATOMIC_HINT_NONSPECULATIVE
Definition: kmp.h:239
@ KMP_IDENT_ATOMIC_REDUCE
Compiler generates atomic reduction option for kmpc_reduce*.
Definition: kmp.h:214
KMP_EXPORT kmp_int32 __kmpc_ok_to_fork(ident_t *)
KMP_EXPORT void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
KMP_EXPORT void __kmpc_fork_call_if(ident_t *loc, kmp_int32 nargs, kmpc_micro microtask, kmp_int32 cond, void *args)
KMP_EXPORT void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
KMP_EXPORT void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit)
KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_push_num_threads_list(ident_t *loc, kmp_int32 global_tid, kmp_uint32 list_length, kmp_int32 *num_threads_list)
KMP_EXPORT void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
KMP_EXPORT void __kmpc_fork_call(ident_t *, kmp_int32 nargs, kmpc_micro microtask,...)
KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams_lb, kmp_int32 num_teams_ub, kmp_int32 num_threads)
KMP_EXPORT void __kmpc_begin(ident_t *, kmp_int32 flags)
KMP_EXPORT void __kmpc_end(ident_t *)
KMP_EXPORT void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
KMP_EXPORT void __kmpc_end_barrier_master(ident_t *, kmp_int32 global_tid)
KMP_EXPORT kmp_int32 __kmpc_barrier_master_nowait(ident_t *, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
KMP_EXPORT kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
KMP_EXPORT void __kmpc_barrier(ident_t *, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_flush(ident_t *)
KMP_EXPORT kmp_int32 __kmpc_barrier_master(ident_t *, kmp_int32 global_tid)
KMP_EXPORT kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
KMP_EXPORT void * __kmpc_task_reduction_get_th_data(int gtid, void *tg, void *d)
KMP_EXPORT void * __kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
KMP_EXPORT void * __kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
KMP_EXPORT bool __kmpc_omp_has_task_team(kmp_int32 gtid)
KMP_EXPORT void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask)
KMP_EXPORT void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws)
KMP_EXPORT kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list)
KMP_EXPORT kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 naffins, kmp_task_affinity_info_t *affin_list)
KMP_EXPORT void * __kmpc_task_reduction_init(int gtid, int num_data, void *data)
KMP_EXPORT void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask)
KMP_EXPORT void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list)
KMP_EXPORT void * __kmpc_taskred_init(int gtid, int num_data, void *data)
KMP_EXPORT void ** __kmpc_omp_get_target_async_handle_ptr(kmp_int32 gtid)
void(* kmpc_dtor)(void *)
Pointer to the destructor function.
Definition: kmp.h:1776
void *(* kmpc_cctor)(void *, void *)
Pointer to an alternate constructor.
Definition: kmp.h:1783
KMP_EXPORT void __kmpc_threadprivate_register(ident_t *, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor)
KMP_EXPORT void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void *(* kmpc_cctor_vec)(void *, void *, size_t)
Array constructor.
Definition: kmp.h:1805
void *(* kmpc_ctor)(void *)
Pointer to the constructor function.
Definition: kmp.h:1770
KMP_EXPORT void * __kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid, void *cpy_data)
void *(* kmpc_ctor_vec)(void *, size_t)
Array constructor.
Definition: kmp.h:1793
KMP_EXPORT void * __kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid, void *data, size_t size, void ***cache)
void(* kmpc_dtor_vec)(void *, size_t)
Pointer to the array destructor function.
Definition: kmp.h:1799
KMP_EXPORT void __kmpc_threadprivate_register_vec(ident_t *, void *data, kmpc_ctor_vec ctor, kmpc_cctor_vec cctor, kmpc_dtor_vec dtor, size_t vector_length)
KMP_EXPORT kmp_int32 __kmpc_global_num_threads(ident_t *)
KMP_EXPORT kmp_int32 __kmpc_global_thread_num(ident_t *)
KMP_EXPORT kmp_int32 __kmpc_in_parallel(ident_t *loc)
KMP_EXPORT kmp_int32 __kmpc_bound_thread_num(ident_t *)
KMP_EXPORT kmp_int32 __kmpc_bound_num_threads(ident_t *)
KMP_EXPORT void __kmpc_end_ordered(ident_t *, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_end_critical(ident_t *, kmp_int32 global_tid, kmp_critical_name *)
KMP_EXPORT void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
sched_type
Describes the loop schedule to be used for a parallel for loop.
Definition: kmp.h:370
KMP_EXPORT void __kmpc_end_masked(ident_t *, kmp_int32 global_tid)
KMP_EXPORT kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid)
KMP_EXPORT kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid)
KMP_EXPORT kmp_int32 __kmpc_next_section(ident_t *loc, kmp_int32 global_tid, kmp_int32 numberOfSections)
KMP_EXPORT void __kmpc_end_master(ident_t *, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_end_sections(ident_t *loc, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_end_single(ident_t *, kmp_int32 global_tid)
KMP_EXPORT kmp_int32 __kmpc_sections_init(ident_t *loc, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_ordered(ident_t *, kmp_int32 global_tid)
KMP_EXPORT kmp_int32 __kmpc_masked(ident_t *, kmp_int32 global_tid, kmp_int32 filter)
void __kmpc_dispatch_init_4(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk)
KMP_EXPORT void __kmpc_critical(ident_t *, kmp_int32 global_tid, kmp_critical_name *)
@ kmp_nm_static_balanced
Definition: kmp.h:428
@ kmp_nm_guided_chunked
guided unspecialized
Definition: kmp.h:421
@ kmp_nm_static_chunked
Definition: kmp.h:417
@ kmp_sch_runtime_simd
runtime with chunk adjustment
Definition: kmp.h:392
@ kmp_nm_ord_auto
auto
Definition: kmp.h:440
@ kmp_nm_ord_dynamic_chunked
Definition: kmp.h:437
@ kmp_sch_auto
auto
Definition: kmp.h:377
@ kmp_nm_auto
auto
Definition: kmp.h:423
@ kmp_distribute_static_chunked
distribute static chunked
Definition: kmp.h:408
@ kmp_sch_static
static unspecialized
Definition: kmp.h:373
@ kmp_sch_guided_simd
guided with chunk adjustment
Definition: kmp.h:391
@ kmp_sch_runtime
Definition: kmp.h:376
@ kmp_ord_dynamic_chunked
Definition: kmp.h:400
@ kmp_sch_modifier_monotonic
Set if the monotonic schedule modifier was present.
Definition: kmp.h:458
@ kmp_nm_static_steal
Definition: kmp.h:432
@ kmp_sch_default
default scheduling algorithm
Definition: kmp.h:478
@ kmp_ord_runtime
Definition: kmp.h:402
@ kmp_sch_modifier_nonmonotonic
Set if the nonmonotonic schedule modifier was present.
Definition: kmp.h:460
@ kmp_nm_static_greedy
Definition: kmp.h:427
@ kmp_nm_ord_static
ordered static unspecialized
Definition: kmp.h:436
@ kmp_distribute_static
distribute static unspecialized
Definition: kmp.h:409
@ kmp_ord_guided_chunked
Definition: kmp.h:401
@ kmp_nm_trapezoidal
Definition: kmp.h:424
@ kmp_sch_guided_chunked
guided unspecialized
Definition: kmp.h:375
@ kmp_sch_dynamic_chunked
Definition: kmp.h:374
@ kmp_sch_guided_analytical_chunked
Definition: kmp.h:385
@ kmp_sch_static_balanced
Definition: kmp.h:382
@ kmp_sch_static_greedy
Definition: kmp.h:381
@ kmp_nm_static
static unspecialized
Definition: kmp.h:419
@ kmp_sch_lower
lower bound for unordered values
Definition: kmp.h:371
@ kmp_sch_static_chunked
Definition: kmp.h:372
@ kmp_nm_ord_trapezoidal
Definition: kmp.h:441
@ kmp_sch_trapezoidal
Definition: kmp.h:378
@ kmp_nm_guided_analytical_chunked
Definition: kmp.h:431
@ kmp_ord_trapezoidal
Definition: kmp.h:404
@ kmp_nm_upper
upper bound for nomerge values
Definition: kmp.h:442
@ kmp_ord_lower
lower bound for ordered values, must be power of 2
Definition: kmp.h:397
@ kmp_ord_static
ordered static unspecialized
Definition: kmp.h:399
@ kmp_sch_guided_iterative_chunked
Definition: kmp.h:384
@ kmp_nm_ord_runtime
Definition: kmp.h:439
@ kmp_sch_static_balanced_chunked
Definition: kmp.h:390
@ kmp_sch_upper
upper bound for unordered values
Definition: kmp.h:395
@ kmp_nm_runtime
Definition: kmp.h:422
@ kmp_ord_upper
upper bound for ordered values
Definition: kmp.h:405
@ kmp_nm_lower
lower bound for nomerge values
Definition: kmp.h:415
@ kmp_nm_guided_iterative_chunked
Definition: kmp.h:430
@ kmp_ord_auto
ordered auto
Definition: kmp.h:403
@ kmp_nm_dynamic_chunked
Definition: kmp.h:420
@ kmp_sch_static_steal
Definition: kmp.h:387
@ kmp_nm_ord_static_chunked
Definition: kmp.h:435
@ kmp_ord_static_chunked
Definition: kmp.h:398
@ kmp_nm_ord_guided_chunked
Definition: kmp.h:438
__itt_string_handle * name
Definition: ittnotify.h:3305
void
Definition: ittnotify.h:3324
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void * data
void const char const char int ITT_FORMAT __itt_group_sync s
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int mask
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp end
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int initialized
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp begin
void * addr
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d __itt_event ITT_FORMAT __itt_group_mark d void const wchar_t const wchar_t int ITT_FORMAT __itt_group_sync __itt_group_fsync x void const wchar_t int const wchar_t int int ITT_FORMAT __itt_group_sync __itt_group_fsync x void ITT_FORMAT __itt_group_sync __itt_group_fsync p void ITT_FORMAT __itt_group_sync __itt_group_fsync p void size_t ITT_FORMAT lu no args __itt_obj_prop_t __itt_obj_state_t ITT_FORMAT d const char ITT_FORMAT s const char ITT_FORMAT s __itt_frame ITT_FORMAT p __itt_counter ITT_FORMAT p __itt_counter unsigned long long ITT_FORMAT lu __itt_counter unsigned long long ITT_FORMAT lu __itt_counter __itt_clock_domain unsigned long long void ITT_FORMAT p const wchar_t ITT_FORMAT S __itt_mark_type const wchar_t ITT_FORMAT S __itt_mark_type const char ITT_FORMAT s __itt_mark_type ITT_FORMAT d __itt_caller ITT_FORMAT p __itt_caller ITT_FORMAT p no args const __itt_domain __itt_clock_domain unsigned long long __itt_id ITT_FORMAT lu const __itt_domain __itt_clock_domain unsigned long long __itt_id __itt_id void ITT_FORMAT p const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_clock_domain unsigned long long __itt_id __itt_string_handle __itt_scope ITT_FORMAT d const __itt_domain __itt_scope __itt_string_handle const char size_t ITT_FORMAT lu const __itt_domain __itt_clock_domain unsigned long long __itt_relation __itt_id ITT_FORMAT lu __itt_track_group __itt_string_handle __itt_track_group_type ITT_FORMAT d __itt_track ITT_FORMAT p void int const int int const char int ITT_FORMAT d void void const char * path
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t count
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d __itt_event event
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id parent
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long value
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t mode
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t size
void const char const char int ITT_FORMAT __itt_group_sync p
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void ITT_FORMAT p const __itt_domain __itt_id __itt_string_handle const wchar_t size_t ITT_FORMAT lu const __itt_domain __itt_id __itt_relation __itt_id ITT_FORMAT p const wchar_t int ITT_FORMAT __itt_group_mark d int
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type type
void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team)
struct kmp_disp kmp_disp_t
int __kmp_memkind_available
Definition: kmp_global.cpp:303
omp_memspace_handle_t const omp_default_mem_space
Definition: kmp_global.cpp:331
KMP_EXPORT void __kmpc_destroy_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void __kmp_finish_implicit_task(kmp_info_t *this_thr)
void * omp_memspace_handle_t
Definition: kmp.h:1082
volatile kmp_team_t * __kmp_team_pool
Definition: kmp_global.cpp:459
KMP_EXPORT kmp_int32 __kmpc_cancel_barrier(ident_t *loc_ref, kmp_int32 gtid)
Definition: kmp_cancel.cpp:243
int __kmp_pause_resource(kmp_pause_status_t level)
void * omp_allocator_handle_t
Definition: kmp.h:1097
void __kmp_warn(char const *format,...)
void ___kmp_thread_free(kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL)
Definition: kmp_alloc.cpp:1213
void KMPC_SET_DYNAMIC(int flag)
kmp_bar_pat_e __kmp_barrier_release_pat_dflt
Definition: kmp_global.cpp:87
struct kmp_dephash kmp_dephash_t
kmp_info_t * __kmp_hidden_helper_main_thread
int __kmp_generate_warnings
Definition: kmp_global.cpp:120
volatile int __kmp_init_user_locks
Definition: kmp_global.cpp:57
int __kmp_debug_buf_lines
Definition: kmp_global.cpp:385
omp_allocator_handle_t __kmpc_init_allocator(int gtid, omp_memspace_handle_t, int ntraits, omp_alloctrait_t traits[])
Definition: kmp_alloc.cpp:1379
int __kmp_cg_max_nth
Definition: kmp_global.cpp:127
int __kmp_abort_delay
Definition: kmp_global.cpp:174
kmp_proc_bind_t __kmp_teams_proc_bind
Definition: kmp_global.cpp:292
KMP_EXPORT void KMPC_CONVENTION kmpc_set_library(int)
kmp_uint32 __kmp_ge_4(kmp_uint32 value, kmp_uint32 checker)
void ompc_set_dynamic(int flag)
bool __kmp_detect_shm()
int __kmp_version
Definition: kmp_global.cpp:64
int __kmp_hws_requested
kmp_bootstrap_lock_t __kmp_initz_lock
void __kmp_aux_set_defaults(char const *str, size_t len)
int __kmp_display_env_verbose
Definition: kmp_global.cpp:214
omp_allocator_handle_t const omp_cgroup_mem_alloc
Definition: kmp_global.cpp:315
kmp_global_t __kmp_global
Definition: kmp_global.cpp:467
void __kmp_set_schedule(int gtid, kmp_sched_t new_sched, int chunk)
omp_alloctrait_key_t
Definition: kmp.h:1048
@ omp_atk_fallback
Definition: kmp.h:1053
@ omp_atk_pinned
Definition: kmp.h:1055
@ omp_atk_access
Definition: kmp.h:1051
@ omp_atk_alignment
Definition: kmp.h:1050
@ omp_atk_pool_size
Definition: kmp.h:1052
@ omp_atk_fb_data
Definition: kmp.h:1054
@ omp_atk_partition
Definition: kmp.h:1056
@ omp_atk_sync_hint
Definition: kmp.h:1049
void __kmp_init_target_mem()
Definition: kmp_alloc.cpp:1359
#define MAX_MTX_DEPS
Definition: kmp.h:2546
KMP_EXPORT void * kmpc_malloc(size_t size)
Definition: kmp_alloc.cpp:1096
cons_type
Definition: kmp.h:1670
@ ct_psingle
Definition: kmp.h:1676
@ ct_parallel
Definition: kmp.h:1672
@ ct_ordered_in_pdo
Definition: kmp.h:1679
@ ct_none
Definition: kmp.h:1671
@ ct_reduce
Definition: kmp.h:1681
@ ct_pdo_ordered
Definition: kmp.h:1674
@ ct_critical
Definition: kmp.h:1677
@ ct_master
Definition: kmp.h:1680
@ ct_barrier
Definition: kmp.h:1682
@ ct_masked
Definition: kmp.h:1683
@ ct_pdo
Definition: kmp.h:1673
@ ct_ordered_in_parallel
Definition: kmp.h:1678
@ ct_psections
Definition: kmp.h:1675
void __kmp_hidden_helper_worker_thread_signal()
KMP_EXPORT void __kmpc_push_num_threads_list_strict(ident_t *loc, kmp_int32 global_tid, kmp_uint32 list_length, kmp_int32 *num_threads_list, int severity, const char *message)
void __kmp_teams_master(int gtid)
void __kmp_elapsed_tick(double *)
void __kmp_common_destroy(void)
void __kmp_common_initialize(void)
#define KMP_HASH_TABLE_SIZE
Definition: kmp.h:1858
size_t __kmp_align_alloc
Definition: kmp_global.cpp:118
void __kmp_release_64(kmp_flag_64<> *flag)
void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1538
kmp_pause_status_t __kmp_pause_status
Definition: kmp_global.cpp:558
struct kmp_teams_size kmp_teams_size_t
kmp_lock_t __kmp_debug_lock
int __kmp_teams_max_nth
Definition: kmp_global.cpp:129
enum kmp_target_offload_kind kmp_target_offload_kind_t
Definition: kmp.h:4539
void __kmp_read_system_time(double *delta)
KMP_NORETURN void __kmp_abort_process(void)
void __kmp_free_thread(kmp_info_t *)
KMP_EXPORT kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task)
void __kmp_enable(int old_state)
kmp_bootstrap_lock_t __kmp_tp_cached_lock
void __kmp_check_stack_overlap(kmp_info_t *thr)
struct kmp_base_root kmp_base_root_t
void * __kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1503
void __kmp_infinite_loop(void)
kmp_info_t * __kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, int tid)
KMP_EXPORT void __kmpc_init_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
kmp_team_t * __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc, kmp_proc_bind_t proc_bind, kmp_internal_control_t *new_icvs, int argc USE_NESTED_HOT_ARG(kmp_info_t *thr))
void __kmp_reap_task_teams(void)
omp_memspace_handle_t const llvm_omp_target_host_mem_space
Definition: kmp_global.cpp:341
kmp_int32 __kmp_use_yield
Definition: kmp_global.cpp:433
kmp_pause_status_t
Definition: kmp.h:4552
@ kmp_hard_paused
Definition: kmp.h:4555
@ kmp_stop_tool_paused
Definition: kmp.h:4556
@ kmp_soft_paused
Definition: kmp.h:4554
@ kmp_not_paused
Definition: kmp.h:4553
kmp_target_offload_kind
Definition: kmp.h:4534
@ tgt_mandatory
Definition: kmp.h:4537
@ tgt_disabled
Definition: kmp.h:4535
@ tgt_default
Definition: kmp.h:4536
char const * __kmp_barrier_type_name[bs_last_barrier]
Definition: kmp_global.cpp:108
char const * __kmp_barrier_pattern_name[bp_last_bar]
Definition: kmp_global.cpp:114
int __kmp_dflt_team_nth_ub
Definition: kmp_global.cpp:132
void __kmp_hidden_helper_threads_initz_wait()
void __kmp_pop_task_team_node(kmp_info_t *thread, kmp_team_t *team)
int __kmp_aux_get_num_teams()
KMP_EXPORT int __kmpc_test_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
struct dispatch_shared_info dispatch_shared_info_t
struct kmp_taskgroup kmp_taskgroup_t
struct kmp_hws_item kmp_hws_item_t
void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr, kmp_team_t *team)
void __kmp_wait_to_unref_task_teams(void)
struct KMP_ALIGN_CACHE kmp_base_info kmp_base_info_t
void __kmp_wait_4_ptr(void *spinner, kmp_uint32 checker, kmp_uint32(*pred)(void *, kmp_uint32), void *obj)
void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team)
int __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size, void *reduce_data, void(*reduce)(void *, void *))
struct KMP_ALIGN_CACHE dispatch_private_info dispatch_private_info_t
int __kmp_get_max_active_levels(int gtid)
KMP_EXPORT void * kmpc_aligned_malloc(size_t size, size_t alignment)
Definition: kmp_alloc.cpp:1109
void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1824
void __kmp_aux_set_library(enum library_type arg)
void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size, char const *format,...)
void __kmp_yield()
void __kmp_push_num_teams_51(ident_t *loc, int gtid, int num_teams_lb, int num_teams_ub, int num_threads)
#define __kmp_assign_root_init_mask()
Definition: kmp.h:3964
int __kmp_dflt_max_active_levels
Definition: kmp_global.cpp:136
int __kmp_xproc
Definition: kmp_global.cpp:122
int __kmp_debug_buf
Definition: kmp_global.cpp:383
KMP_EXPORT void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32 num_dims, const struct kmp_dim *dims)
void __kmp_unlock_suspend_mx(kmp_info_t *th)
kmp_bar_pat_e __kmp_barrier_gather_pat_dflt
Definition: kmp_global.cpp:85
unsigned short __kmp_get_random(kmp_info_t *thread)
void * __kmpc_calloc(int gtid, size_t nmemb, size_t sz, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1519
static kmp_team_t * __kmp_team_from_gtid(int gtid)
Definition: kmp.h:3648
int __kmp_register_root(int initial_thread)
int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin, int *thread_finished, kmp_int32 is_constrained)
omp_memspace_handle_t const omp_low_lat_mem_space
Definition: kmp_global.cpp:339
void __kmp_do_initialize_hidden_helper_threads()
int __kmp_storage_map_verbose_specified
Definition: kmp_global.cpp:413
struct kmp_local kmp_local_t
omp_allocator_handle_t __kmpc_get_default_allocator(int gtid)
Definition: kmp_alloc.cpp:1489
kmp_bar_pat_e __kmp_barrier_gather_pattern[bs_last_barrier]
Definition: kmp_global.cpp:92
void __kmp_thread_sleep(int millis)
KMP_EXPORT kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part)
kmp_tasking_mode_t __kmp_tasking_mode
Definition: kmp_global.cpp:299
char * __kmp_affinity_format
Definition: kmp_global.cpp:295
void __kmp_abort_thread(void)
int __kmp_dflt_blocktime
Definition: kmp_global.cpp:158
volatile kmp_info_t * __kmp_thread_pool
Definition: kmp_global.cpp:458
void __kmp_internal_end_atexit(void)
kmp_hws_item_t __kmp_hws_die
KMP_EXPORT void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar)
volatile int __kmp_init_gtid
Definition: kmp_global.cpp:45
omp_allocator_handle_t __kmp_def_allocator
Definition: kmp_global.cpp:329
kmp_hws_item_t __kmp_hws_node
union KMP_ALIGN_CACHE kmp_sleep_team kmp_sleep_team_t
kmp_bootstrap_lock_t __kmp_task_team_lock
void * __kmp_alloc(int gtid, size_t align, size_t sz, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1546
int __kmp_omp_cancellation
Definition: kmp_global.cpp:215
void __kmp_suspend_64(int th_gtid, kmp_flag_64< C, S > *flag)
static void __kmp_resume_if_hard_paused()
Definition: kmp.h:4568
size_t __kmp_stksize
Definition: kmp_global.cpp:69
kmp_hws_item_t __kmp_hws_tile
int __kmp_env_checks
Definition: kmp_global.cpp:422
void * __kmp_calloc(int gtid, size_t align, size_t nmemb, size_t sz, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1758
kmp_nested_proc_bind_t __kmp_nested_proc_bind
Definition: kmp_global.cpp:291
void __kmp_free_implicit_task(kmp_info_t *this_thr)
KMP_EXPORT void __kmpc_destroy_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void KMP_EXPAND_NAME() ompc_set_affinity_format(char const *format)
void __kmp_hidden_helper_main_thread_release()
void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr, void *data_addr, size_t pc_size)
fork_context_e
Tell the fork call which compiler generated the fork call, and therefore how to deal with the call.
Definition: kmp.h:4070
@ fork_context_gnu
Called from GNU generated code, so must not invoke the microtask internally.
Definition: kmp.h:4071
@ fork_context_intel
Called from Intel generated code.
Definition: kmp.h:4073
@ fork_context_last
Definition: kmp.h:4074
void __kmp_exit_single(int gtid)
struct KMP_ALIGN_CACHE dispatch_private_info32 dispatch_private_info32_t
void __kmp_suspend_initialize(void)
int __kmp_get_team_size(int gtid, int level)
kmp_nested_nthreads_t __kmp_nested_nth
Definition: kmp_global.cpp:200
int __kmp_max_nth
Definition: kmp_global.cpp:126
KMP_EXPORT void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind)
omp_allocator_handle_t const omp_default_mem_alloc
Definition: kmp_global.cpp:305
int __kmp_chunk
Definition: kmp_global.cpp:172
kmp_uint32 __kmp_eq_4(kmp_uint32 value, kmp_uint32 checker)
omp_alloctrait_value_t
Definition: kmp.h:1059
@ omp_atv_sequential
Definition: kmp.h:1065
@ omp_atv_cgroup
Definition: kmp.h:1070
@ omp_atv_all
Definition: kmp.h:1067
@ omp_atv_interleaved
Definition: kmp.h:1078
@ omp_atv_false
Definition: kmp.h:1060
@ omp_atv_environment
Definition: kmp.h:1075
@ omp_atv_serialized
Definition: kmp.h:1064
@ omp_atv_blocked
Definition: kmp.h:1077
@ omp_atv_thread
Definition: kmp.h:1068
@ omp_atv_default_mem_fb
Definition: kmp.h:1071
@ omp_atv_allocator_fb
Definition: kmp.h:1074
@ omp_atv_pteam
Definition: kmp.h:1069
@ omp_atv_contended
Definition: kmp.h:1062
@ omp_atv_null_fb
Definition: kmp.h:1072
@ omp_atv_nearest
Definition: kmp.h:1076
@ omp_atv_uncontended
Definition: kmp.h:1063
@ omp_atv_true
Definition: kmp.h:1061
@ omp_atv_private
Definition: kmp.h:1066
@ omp_atv_abort_fb
Definition: kmp.h:1073
kmp_cached_addr_t * __kmp_threadpriv_cache_list
Definition: kmp_global.cpp:60
flag_type
Definition: kmp.h:2132
@ flag_unset
Definition: kmp.h:2137
@ atomic_flag64
atomic 64 bit flags
Definition: kmp.h:2135
@ flag64
64 bit flags
Definition: kmp.h:2134
@ flag_oncore
special 64-bit flag for on-core barrier (hierarchical)
Definition: kmp.h:2136
@ flag32
atomic 32 bit flags
Definition: kmp.h:2133
void __kmp_internal_end_dtor(void)
kmp_uint64 __kmp_now_nsec()
KMP_EXPORT int __kmp_get_cancellation_status(int cancel_kind)
Definition: kmp_cancel.cpp:309
void KMP_EXPAND_NAME() ompc_display_affinity(char const *format)
volatile int __kmp_all_nth
Definition: kmp_global.cpp:457
void __kmp_check_stksize(size_t *val)
kmp_target_offload_kind_t __kmp_target_offload
Definition: kmp_global.cpp:555
int __kmp_debug_buf_chars
Definition: kmp_global.cpp:387
int __kmpc_get_target_offload()
void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref)
KMP_EXPORT void __kmpc_critical_with_hint(ident_t *, kmp_int32 global_tid, kmp_critical_name *, uint32_t hint)
int __kmp_get_global_thread_id_reg(void)
void __kmp_push_num_threads(ident_t *loc, int gtid, int num_threads)
KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize(int)
#define SCHEDULE_HAS_MONOTONIC(s)
Definition: kmp.h:466
int __kmpc_dispatch_next_4(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_int32 *p_lb, kmp_int32 *p_ub, kmp_int32 *p_st)
int __kmp_is_address_mapped(void *addr)
kmp_lock_t __kmp_global_lock
kmp_severity_t
Definition: kmp.h:4634
@ severity_warning
Definition: kmp.h:4635
@ severity_fatal
Definition: kmp.h:4636
int __kmp_barrier_gomp_cancel(int gtid)
double __kmp_read_cpu_time(void)
void __kmpc_destroy_allocator(int gtid, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1478
union KMP_ALIGN_CACHE kmp_root kmp_root_t
int __kmp_adjust_gtid_mode
Definition: kmp_global.cpp:183
int __kmp_env_blocktime
Definition: kmp_global.cpp:421
#define __kmp_entry_gtid()
Definition: kmp.h:3610
kmp_old_threads_list_t * __kmp_old_threads_list
Definition: kmp_global.cpp:452
void __kmp_internal_end_library(int gtid)
void __kmp_suspend_32(int th_gtid, kmp_flag_32< C, S > *flag)
struct kmp_internal_control kmp_internal_control_t
void __kmp_hidden_helper_worker_thread_wait()
void __kmpc_dispatch_fini_4(ident_t *loc, kmp_int32 gtid)
volatile int __kmp_init_common
Definition: kmp_global.cpp:46
kmp_info_t __kmp_monitor
void __kmp_set_max_active_levels(int gtid, int new_max_active_levels)
void __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 gtid)
See __kmpc_dispatch_deinit.
enum sched_type __kmp_auto
Definition: kmp_global.cpp:150
void __kmp_init_random(kmp_info_t *thread)
static int __kmp_tid_from_gtid(int gtid)
Definition: kmp.h:3628
static bool KMP_UBER_GTID(int gtid)
Definition: kmp.h:3621
int __kmp_display_env
Definition: kmp_global.cpp:213
kmp_int32 __kmp_use_yield_exp_set
Definition: kmp_global.cpp:437
kmp_event_t * __kmpc_task_allow_completion_event(ident_t *loc_ref, int gtid, kmp_task_t *task)
int __kmp_tp_cached
Definition: kmp_global.cpp:134
void __kmp_internal_end_thread(int gtid)
struct kmp_sys_info kmp_sys_info_t
KMP_EXPORT void __kmp_set_num_teams(int num_teams)
void __kmp_disable(int *old_state)
omp_allocator_handle_t const omp_large_cap_mem_alloc
Definition: kmp_global.cpp:307
volatile int __kmp_init_hidden_helper
Definition: kmp_global.cpp:50
void __kmp_push_num_threads_list(ident_t *loc, int gtid, kmp_uint32 list_length, int *num_threads_list)
struct kmp_depend_info kmp_depend_info_t
void __kmp_user_set_library(enum library_type arg)
const char * __kmp_hw_get_catalog_string(kmp_hw_t type, bool plural=false)
KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
void __kmpc_set_default_allocator(int gtid, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1483
omp_allocator_handle_t const omp_low_lat_mem_alloc
Definition: kmp_global.cpp:313
kmp_event_type_t
Definition: kmp.h:2596
@ KMP_EVENT_UNINITIALIZED
Definition: kmp.h:2597
@ KMP_EVENT_ALLOW_COMPLETION
Definition: kmp.h:2598
void __kmp_elapsed(double *)
KMP_EXPORT void KMPC_CONVENTION kmpc_set_disp_num_buffers(int)
int __kmp_gtid_get_specific(void)
int __kmp_aux_get_team_num()
struct KMP_ALIGN_CACHE dispatch_private_info64 dispatch_private_info64_t
KMP_EXPORT void __kmpc_taskloop(ident_t *loc, kmp_int32 gtid, kmp_task_t *task, kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup, kmp_int32 sched, kmp_uint64 grainsize, void *task_dup)
volatile int __kmp_init_middle
Definition: kmp_global.cpp:48
void __kmp_hidden_helper_threads_deinitz_wait()
omp_allocator_handle_t const omp_high_bw_mem_alloc
Definition: kmp_global.cpp:311
void __kmp_set_num_threads(int new_nth, int gtid)
std::atomic< kmp_int32 > __kmp_task_counter
Definition: kmp_global.cpp:67
void __kmpc_error(ident_t *loc, int severity, const char *message)
static kmp_sched_t __kmp_sched_without_mods(kmp_sched_t kind)
Definition: kmp.h:501
kmp_cancel_kind_t
Definition: kmp.h:1006
@ cancel_parallel
Definition: kmp.h:1008
@ cancel_taskgroup
Definition: kmp.h:1011
@ cancel_sections
Definition: kmp.h:1010
@ cancel_loop
Definition: kmp.h:1009
@ cancel_noreq
Definition: kmp.h:1007
KMP_EXPORT kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task)
KMP_EXPORT kmp_task_t * __kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, kmp_routine_entry_t task_entry)
KMP_EXPORT void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid)
int __kmp_reserve_warn
Definition: kmp_global.cpp:121
kmp_r_sched_t __kmp_get_schedule_global(void)
int __kmp_storage_map_verbose
Definition: kmp_global.cpp:411
int __kmp_allThreadsSpecified
Definition: kmp_global.cpp:117
enum sched_type __kmp_static
Definition: kmp_global.cpp:146
int __kmp_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64< C, S > *flag, int final_spin, int *thread_finished, kmp_int32 is_constrained)
void kmpc_set_blocktime(int arg)
KMP_EXPORT void __kmpc_taskloop_5(ident_t *loc, kmp_int32 gtid, kmp_task_t *task, kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup, kmp_int32 sched, kmp_uint64 grainsize, kmp_int32 modifier, void *task_dup)
KMP_EXPORT void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task)
PACKED_REDUCTION_METHOD_T __kmp_determine_reduction_method(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
volatile int __kmp_nth
Definition: kmp_global.cpp:456
enum kmp_tasking_mode kmp_tasking_mode_t
void * __kmp_realloc(int gtid, void *ptr, size_t sz, omp_allocator_handle_t al, omp_allocator_handle_t free_al)
Definition: kmp_alloc.cpp:1787
void __kmp_end_split_barrier(enum barrier_type bt, int gtid)
int PACKED_REDUCTION_METHOD_T
Definition: kmp.h:605
std::atomic< int > __kmp_thread_pool_active_nth
Definition: kmp_global.cpp:462
void __kmp_hidden_helper_threads_initz_routine()
KMP_EXPORT void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid)
const char * __kmp_hw_get_keyword(kmp_hw_t type, bool plural=false)
union KMP_ALIGN_CACHE kmp_thread_data kmp_thread_data_t
kmp_uint32 __kmp_lt_4(kmp_uint32 value, kmp_uint32 checker)
KMP_EXPORT void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, const kmp_int64 *vec)
int __kmp_affinity_num_places
Definition: kmp_global.cpp:293
int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws)
int __kmp_duplicate_library_ok
Definition: kmp_global.cpp:363
void * ___kmp_thread_malloc(kmp_info_t *th, size_t size KMP_SRC_LOC_DECL)
Definition: kmp_alloc.cpp:1185
struct kmp_base_data kmp_base_data_t
struct kmp_base_thread_data kmp_base_thread_data_t
volatile int __kmp_need_register_serial
Definition: kmp_global.cpp:47
#define KMP_PAD(type, sz)
Definition: kmp.h:1031
#define KMP_MAX_FIELDS
Definition: kmp.h:1305
void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team)
kmp_bootstrap_lock_t __kmp_forkjoin_lock
KMP_EXPORT kmp_uint64 __kmpc_get_taskid()
omp_memspace_handle_t const omp_const_mem_space
Definition: kmp_global.cpp:335
struct kmp_cg_root kmp_cg_root_t
kmp_uint32 __kmp_barrier_release_branch_bits[bs_last_barrier]
Definition: kmp_global.cpp:91
KMP_EXPORT int KMPC_CONVENTION kmpc_get_affinity_mask_proc(int, kmp_affinity_mask_t *)
static kmp_info_t * __kmp_entry_thread()
Definition: kmp.h:3740
KMP_EXPORT kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid)
void __kmp_init_memkind()
Definition: kmp_alloc.cpp:1272
struct kmp_task_affinity_info kmp_task_affinity_info_t
int __kmp_get_ancestor_thread_num(int gtid, int level)
void __kmp_hidden_helper_main_thread_wait()
void * __kmp_launch_thread(kmp_info_t *thr)
void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr, kmp_team_t *team, int tid, int set_curr_task)
kmp_int32 __kmp_default_device
Definition: kmp_global.cpp:297
void __kmp_omp_display_env(int verbose)
void __kmp_cleanup_threadprivate_caches()
void __kmp_middle_initialize(void)
static void copy_icvs(kmp_internal_control_t *dst, kmp_internal_control_t *src)
Definition: kmp.h:2193
KMP_EXPORT void __kmpc_end_taskgroup(ident_t *loc, int gtid)
kmp_bootstrap_lock_t __kmp_exit_lock
KMP_EXPORT void __kmpc_omp_taskwait_deps_51(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list, kmp_int32 has_no_wait)
omp_memspace_handle_t const omp_large_cap_mem_space
Definition: kmp_global.cpp:333
int __kmp_force_monotonic
Definition: kmp_global.cpp:173
kmp_info_t ** __kmp_threads
Definition: kmp_global.cpp:450
void __kmp_abort(char const *format,...)
void __kmp_hidden_helper_initz_release()
enum sched_type __kmp_sched
Definition: kmp_global.cpp:144
uintptr_t omp_uintptr_t
Definition: kmp.h:1046
void __kmp_suspend_uninitialize_thread(kmp_info_t *th)
void * ___kmp_page_allocate(size_t size KMP_SRC_LOC_DECL)
Definition: kmp_alloc.cpp:2010
#define KMPC_CONVENTION
Definition: kmp.h:4499
struct kmp_cached_addr kmp_cached_addr_t
int __kmp_enable_task_throttling
Definition: kmp_global.cpp:356
void __kmp_unregister_root(int gtid)
void __kmp_finalize_bget(kmp_info_t *th)
static void __kmp_reset_root_init_mask(int gtid)
Definition: kmp.h:3965
kmp_uint32 __kmp_barrier_gather_bb_dflt
Definition: kmp_global.cpp:80
KMP_EXPORT void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task)
kmp_uint32 __kmp_barrier_release_bb_dflt
Definition: kmp_global.cpp:82
struct dispatch_shared_info32 dispatch_shared_info32_t
int __kmp_task_stealing_constraint
Definition: kmp_global.cpp:355
int __kmp_need_register_atfork
Definition: kmp_global.cpp:416
struct private_common * kmp_threadprivate_insert(int gtid, void *pc_addr, void *data_addr, size_t pc_size)
void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref)
struct kmp_target_data kmp_target_data_t
int __kmp_dispatch_num_buffers
Definition: kmp_global.cpp:135
KMP_EXPORT void __kmpc_unset_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
#define SCHEDULE_WITHOUT_MODIFIERS(s)
Definition: kmp.h:463
kmp_uint32 __kmp_yield_init
Definition: kmp_global.cpp:439
KMP_EXPORT void __kmp_set_teams_thread_limit(int limit)
void __kmp_internal_end_dest(void *)
int __kmp_nesting_mode
Definition: kmp_global.cpp:561
void * __kmpc_realloc(int gtid, void *ptr, size_t sz, omp_allocator_handle_t al, omp_allocator_handle_t free_al)
Definition: kmp_alloc.cpp:1528
void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag)
bool __kmp_dflt_max_active_levels_set
Definition: kmp_global.cpp:137
void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr, kmp_team_t *team)
size_t KMP_EXPAND_NAME() ompc_get_affinity_format(char *buffer, size_t size)
KMP_EXPORT void KMPC_CONVENTION kmpc_set_defaults(char const *)
void __kmp_lock_suspend_mx(kmp_info_t *th)
struct dispatch_shared_info64 dispatch_shared_info64_t
omp_memspace_handle_t const llvm_omp_target_shared_mem_space
Definition: kmp_global.cpp:343
bool __kmp_detect_tmp()
char * __kmp_debug_buffer
Definition: kmp_global.cpp:392
omp_memspace_handle_t const omp_high_bw_mem_space
Definition: kmp_global.cpp:337
void __kmp_parallel_initialize(void)
void __kmp_terminate_thread(int gtid)
int __kmp_nesting_mode_nlevels
Definition: kmp_global.cpp:562
void __kmp_set_nesting_mode_threads()
void __kmp_unregister_library(void)
int __kmp_nteams
Definition: kmp_global.cpp:216
int __kmp_storage_map
Definition: kmp_global.cpp:409
int __kmpc_dispatch_next_4u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_uint32 *p_lb, kmp_uint32 *p_ub, kmp_int32 *p_st)
See __kmpc_dispatch_next_4.
int(* launch_t)(int gtid)
Definition: kmp.h:3109
int __kmp_ignore_mppbeg(void)
void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size)
int * __kmp_nesting_nth_level
Definition: kmp_global.cpp:563
KMP_EXPORT void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, const kmp_int64 *vec)
omp_allocator_handle_t const omp_const_mem_alloc
Definition: kmp_global.cpp:309
KMP_EXPORT void __kmpc_set_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
volatile int __kmp_init_parallel
Definition: kmp_global.cpp:49
int __kmp_init_counter
Definition: kmp_global.cpp:62
KMP_EXPORT void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads, int severity, const char *message)
omp_allocator_handle_t const omp_pteam_mem_alloc
Definition: kmp_global.cpp:317
int __kmp_sys_max_nth
Definition: kmp_global.cpp:125
kmp_queuing_lock_t __kmp_dispatch_lock
Definition: kmp_global.cpp:516
KMP_EXPORT int KMPC_CONVENTION kmpc_set_affinity_mask_proc(int, kmp_affinity_mask_t *)
omp_allocator_handle_t const llvm_omp_target_host_mem_alloc
Definition: kmp_global.cpp:321
kmp_root_t ** __kmp_root
Definition: kmp_global.cpp:451
int __kmp_need_register_atfork_specified
Definition: kmp_global.cpp:418
omp_allocator_handle_t const kmp_max_mem_alloc
Definition: kmp_global.cpp:327
kmp_int32 __kmp_enable_hidden_helper
kmp_state_timer
Definition: kmp.h:325
@ ts_last_state
Definition: kmp.h:330
@ ts_pause
Definition: kmp.h:328
@ ts_start
Definition: kmp.h:327
@ ts_stop
Definition: kmp.h:326
struct kmp_desc_base kmp_desc_base_t
enum kmp_sched kmp_sched_t
void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team)
void __kmp_aux_set_stacksize(size_t arg)
static const size_t KMP_AFFINITY_FORMAT_SIZE
Definition: kmp.h:988
#define TRUE
Definition: kmp.h:1333
enum library_type __kmp_library
Definition: kmp_global.cpp:143
#define FALSE
Definition: kmp.h:1332
int __kmp_tp_capacity
Definition: kmp_global.cpp:133
void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid)
void __kmp_push_num_teams(ident_t *loc, int gtid, int num_teams, int num_threads)
struct kmp_tasking_flags kmp_tasking_flags_t
omp_memspace_handle_t kmp_memspace_t
Definition: kmp.h:1118
static bool __kmp_is_hybrid_cpu()
Definition: kmp.h:3368
void __kmp_clear_system_time(void)
KMP_EXPORT kmp_uint64 __kmpc_get_parent_taskid()
int __kmp_settings
Definition: kmp_global.cpp:362
struct KMP_ALIGN_CACHE kmp_base_team kmp_base_team_t
size_t __kmp_aux_capture_affinity(int gtid, const char *format, kmp_str_buf_t *buffer)
kmp_tasking_mode
Definition: kmp.h:2425
@ tskm_extra_barrier
Definition: kmp.h:2427
@ tskm_max
Definition: kmp.h:2429
@ tskm_task_teams
Definition: kmp.h:2428
@ tskm_immediate_exec
Definition: kmp.h:2426
KMP_EXPORT int __kmp_get_max_teams(void)
void KMPC_SET_NESTED(int flag)
void(* kmpc_micro_bound)(kmp_int32 *bound_tid, kmp_int32 *bound_nth,...)
Definition: kmp.h:1753
void __kmp_get_schedule(int gtid, kmp_sched_t *sched, int *chunk)
kmp_team_t * __kmp_reap_team(kmp_team_t *)
kmp_key_t __kmp_gtid_threadprivate_key
Definition: kmp_global.cpp:19
KMP_EXPORT void * __kmpc_threadprivate(ident_t *, kmp_int32 global_tid, void *data, size_t size)
struct kmp_task_pri kmp_task_pri_t
kmp_hws_item_t __kmp_hws_socket
KMP_EXPORT void KMPC_CONVENTION ompc_set_schedule(omp_sched_t, int)
int __kmp_fork_call(ident_t *loc, int gtid, enum fork_context_e fork_context, kmp_int32 argc, microtask_t microtask, launch_t invoker, kmp_va_list ap)
kmp_info_t * __kmp_thread_pool_insert_pt
KMP_EXPORT kmp_int32 __kmpc_cancel(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind)
Definition: kmp_cancel.cpp:29
KMP_EXPORT void * kmpc_calloc(size_t nelem, size_t elsize)
Definition: kmp_alloc.cpp:1131
#define UNLIKELY(x)
Definition: kmp.h:159
int __kmpc_dispatch_next_8u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_uint64 *p_lb, kmp_uint64 *p_ub, kmp_int64 *p_st)
See __kmpc_dispatch_next_4.
int __kmp_get_global_thread_id(void)
#define USE_NESTED_HOT_ARG(x)
Definition: kmp.h:174
int __kmp_env_consistency_check
Definition: kmp_global.cpp:423
#define bs_reduction_barrier
Definition: kmp.h:2152
void __kmp_runtime_destroy(void)
kmp_uint64 __kmp_pause_init
Definition: kmp_global.cpp:441
#define KMP_MAX_ORDERED
Definition: kmp.h:1303
kmp_uint64 __kmp_taskloop_min_tasks
Definition: kmp_global.cpp:301
KMP_EXPORT int KMPC_CONVENTION ompc_get_ancestor_thread_num(int)
union KMP_ALIGN_CACHE kmp_desc kmp_desc_t
char const * __kmp_barrier_branch_bit_env_name[bs_last_barrier]
Definition: kmp_global.cpp:94
kmp_hws_item_t __kmp_hws_proc
omp_sched_t
Definition: kmp.h:4503
@ omp_sched_dynamic
Definition: kmp.h:4505
@ omp_sched_auto
Definition: kmp.h:4507
@ omp_sched_guided
Definition: kmp.h:4506
@ omp_sched_static
Definition: kmp.h:4504
void __kmp_aux_display_affinity(int gtid, const char *format)
static void __kmp_sched_apply_mods_intkind(kmp_sched_t kind, enum sched_type *internal_kind)
Definition: kmp.h:492
void __kmp_fulfill_event(kmp_event_t *event)
KMP_EXPORT void __kmpc_taskgroup(ident_t *loc, int gtid)
int __kmp_read_system_info(struct kmp_sys_info *info)
void * ___kmp_thread_realloc(kmp_info_t *th, void *ptr, size_t size KMP_SRC_LOC_DECL)
Definition: kmp_alloc.cpp:1204
kmp_uint32 __kmp_neq_4(kmp_uint32 value, kmp_uint32 checker)
volatile int __kmp_hidden_helper_team_done
Definition: kmp_global.cpp:52
KMP_EXPORT kmp_depnode_list_t * __kmpc_task_get_successors(kmp_task_t *task)
void __kmp_push_proc_bind(ident_t *loc, int gtid, kmp_proc_bind_t proc_bind)
static void __kmp_sched_apply_mods_stdkind(kmp_sched_t *kind, enum sched_type internal_kind)
Definition: kmp.h:483
int __kmp_determ_red
Definition: kmp_global.cpp:370
struct kmp_base_depnode kmp_base_depnode_t
void __kmp_init_nesting_mode()
void __kmp_free_team(kmp_root_t *, kmp_team_t *USE_NESTED_HOT_ARG(kmp_info_t *))
std::atomic< kmp_int32 > __kmp_unexecuted_hidden_helper_tasks
KMP_EXPORT int KMPC_CONVENTION kmpc_unset_affinity_mask_proc(int, kmp_affinity_mask_t *)
KMP_EXPORT void __kmpc_end_scope(ident_t *loc, kmp_int32 gtid, void *reserved)
KMP_EXPORT int __kmpc_invoke_task_func(int gtid)
#define KMP_GTID_MIN
Definition: kmp.h:1040
void * __kmpc_aligned_alloc(int gtid, size_t align, size_t sz, omp_allocator_handle_t al)
Definition: kmp_alloc.cpp:1510
size_t __kmp_sys_min_stksize
Definition: kmp_global.cpp:124
char __kmp_blocktime_units
Definition: kmp_global.cpp:159
kmp_sched
Definition: kmp.h:347
@ kmp_sched_upper
Definition: kmp.h:360
@ kmp_sched_lower
Definition: kmp.h:348
@ kmp_sched_trapezoidal
Definition: kmp.h:356
@ kmp_sched_upper_std
Definition: kmp.h:354
@ kmp_sched_dynamic
Definition: kmp.h:351
@ kmp_sched_auto
Definition: kmp.h:353
@ kmp_sched_guided
Definition: kmp.h:352
@ kmp_sched_lower_ext
Definition: kmp.h:355
@ kmp_sched_default
Definition: kmp.h:361
@ kmp_sched_monotonic
Definition: kmp.h:362
@ kmp_sched_static
Definition: kmp.h:350
void * ___kmp_allocate(size_t size KMP_SRC_LOC_DECL)
Definition: kmp_alloc.cpp:1997
KMP_EXPORT void KMPC_CONVENTION ompc_set_max_active_levels(int)
struct kmp_sched_flags kmp_sched_flags_t
kmp_hws_item_t __kmp_hws_core
union KMP_ALIGN_CACHE kmp_ordered_team kmp_ordered_team_t
int __kmp_invoke_task_func(int gtid)
struct kmp_base_global kmp_base_global_t
void ompc_set_nested(int flag)
int __kmp_hws_abs_flag
void __kmp_set_strict_num_threads(ident_t *loc, int gtid, int sev, const char *msg)
kmp_uint32 __kmp_barrier_gather_branch_bits[bs_last_barrier]
Definition: kmp_global.cpp:90
KMP_EXPORT void __kmpc_scope(ident_t *loc, kmp_int32 gtid, void *reserved)
size_t __kmp_stkoffset
Definition: kmp_global.cpp:73
KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize_s(size_t)
int __kmp_task_max_nth
Definition: kmp_global.cpp:128
size_t __kmp_malloc_pool_incr
Definition: kmp_global.cpp:76
static int __kmp_adjust_gtid_for_hidden_helpers(int gtid)
Definition: kmp.h:4624
kmp_task_t * __kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, kmp_tasking_flags_t *flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, kmp_routine_entry_t task_entry)
void __kmp_adjust_num_threads(int new_nproc)
void __kmpc_dispatch_fini_8(ident_t *loc, kmp_int32 gtid)
See __kmpc_dispatch_fini_4.
KMP_EXPORT void kmpc_free(void *ptr)
Definition: kmp_alloc.cpp:1172
int __kmp_threads_capacity
Definition: kmp_global.cpp:130
KMP_EXPORT int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
kmp_info_t ** __kmp_hidden_helper_threads
kmp_uint32 __kmp_le_4(kmp_uint32 value, kmp_uint32 checker)
void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team, int tid)
int __kmp_debug_buf_warn_chars
Definition: kmp_global.cpp:395
int __kmp_foreign_tp
Definition: kmp_global.cpp:189
static int __kmp_gtid_from_tid(int tid, const kmp_team_t *team)
Definition: kmp.h:3633
KMP_EXPORT void KMPC_FOR_STATIC_INIT(ident_t *loc, kmp_int32 global_tid, kmp_int32 schedtype, kmp_int32 *plastiter, kmp_int *plower, kmp_int *pupper, kmp_int *pstride, kmp_int incr, kmp_int chunk)
void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag)
bool __kmp_wpolicy_passive
Definition: kmp_global.cpp:160
void __kmp_save_internal_controls(kmp_info_t *thread)
size_t KMP_EXPAND_NAME() ompc_capture_affinity(char *buffer, size_t buf_size, char const *format)
void __kmp_push_task_team_node(kmp_info_t *thread, kmp_team_t *team)
void __kmp_threadprivate_resize_cache(int newCapacity)
union kmp_r_sched kmp_r_sched_t
void __kmp_runtime_initialize(void)
barrier_type
Definition: kmp.h:2140
@ bs_plain_barrier
Definition: kmp.h:2141
@ bs_last_barrier
Definition: kmp.h:2147
@ bs_forkjoin_barrier
Definition: kmp.h:2143
int __kmp_invoke_teams_master(int gtid)
void __kmp_hidden_helper_initialize()
volatile int __kmp_init_hidden_helper_threads
Definition: kmp_global.cpp:51
void KMPC_SET_NUM_THREADS(int arg)
KMP_EXPORT kmp_int32 __kmpc_cancellationpoint(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind)
Definition: kmp_cancel.cpp:135
void __kmp_common_destroy_gtid(int gtid)
int __kmp_try_suspend_mx(kmp_info_t *th)
static void __kmp_aux_convert_blocktime(int *bt)
Definition: kmp.h:3487
int __kmpc_dispatch_next_8(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_int64 *p_lb, kmp_int64 *p_ub, kmp_int64 *p_st)
See __kmpc_dispatch_next_4.
int __kmp_display_affinity
Definition: kmp_global.cpp:294
enum sched_type __kmp_guided
Definition: kmp_global.cpp:148
void __kmp_resume_64(int target_gtid, kmp_flag_64< C, S > *flag)
void __kmp_resume_32(int target_gtid, kmp_flag_32< C, S > *flag)
KMP_EXPORT int __kmp_get_teams_thread_limit(void)
#define KMP_INLINE_ARGV_ENTRIES
Definition: kmp.h:3128
int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32< C, S > *flag, int final_spin, int *thread_finished, kmp_int32 is_constrained)
const char * __kmp_hw_get_core_type_string(kmp_hw_core_type_t type)
void __kmpc_dispatch_fini_8u(ident_t *loc, kmp_int32 gtid)
See __kmpc_dispatch_fini_4.
PACKED_REDUCTION_METHOD_T __kmp_force_reduction_method
Definition: kmp_global.cpp:368
void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64< C, S > *flag)
KMP_EXPORT int KMPC_CONVENTION ompc_get_team_size(int)
void * kmp_affinity_mask_t
Definition: kmp.h:4509
void __kmp_serial_initialize(void)
int __kmp_avail_proc
Definition: kmp_global.cpp:123
omp_allocator_handle_t const omp_thread_mem_alloc
Definition: kmp_global.cpp:319
kmp_uint32 __kmp_wait_4(kmp_uint32 volatile *spinner, kmp_uint32 checker, kmp_uint32(*pred)(kmp_uint32, kmp_uint32), void *obj)
void __kmp_resume_if_soft_paused()
KMP_EXPORT void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
kmp_int32 __kmp_max_task_priority
Definition: kmp_global.cpp:300
void __kmp_initialize_bget(kmp_info_t *th)
static void __kmp_assert_valid_gtid(kmp_int32 gtid)
Definition: kmp.h:3653
int __kmp_teams_thread_limit
Definition: kmp_global.cpp:217
int __kmp_stkpadding
Definition: kmp_global.cpp:74
KMP_EXPORT void * kmpc_realloc(void *ptr, size_t size)
Definition: kmp_alloc.cpp:1142
void __kmp_cleanup_hierarchy()
KMP_EXPORT kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, kmp_routine_entry_t task_entry, kmp_int64 device_id)
int __kmp_dflt_team_nth
Definition: kmp_global.cpp:131
KMP_EXPORT void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
void __kmp_serialized_parallel(ident_t *id, kmp_int32 gtid)
void ompc_set_num_threads(int arg)
kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task, bool serialize_immediate)
struct kmp_base_task_team kmp_base_task_team_t
void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr)
void __kmp_gtid_set_specific(int gtid)
char const * __kmp_barrier_pattern_env_name[bs_last_barrier]
Definition: kmp_global.cpp:101
void __kmp_internal_begin(void)
std::atomic< int > __kmp_debug_count
Definition: kmp_global.cpp:393
void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team)
int __kmp_atomic_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid, kmp_atomic_flag_64< C, S > *flag, int final_spin, int *thread_finished, kmp_int32 is_constrained)
kmp_proc_bind_t
Definition: kmp.h:967
@ proc_bind_false
Definition: kmp.h:968
@ proc_bind_true
Definition: kmp.h:969
@ proc_bind_close
Definition: kmp.h:971
@ proc_bind_intel
Definition: kmp.h:973
@ proc_bind_primary
Definition: kmp.h:970
@ proc_bind_spread
Definition: kmp.h:972
@ proc_bind_default
Definition: kmp.h:974
kmp_hw_t
Definition: kmp.h:628
@ KMP_HW_UNKNOWN
Definition: kmp.h:629
@ KMP_HW_NUMA
Definition: kmp.h:632
@ KMP_HW_SOCKET
Definition: kmp.h:630
@ KMP_HW_CORE
Definition: kmp.h:640
@ KMP_HW_L2
Definition: kmp.h:638
@ KMP_HW_PROC_GROUP
Definition: kmp.h:631
@ KMP_HW_L1
Definition: kmp.h:639
@ KMP_HW_L3
Definition: kmp.h:635
@ KMP_HW_MODULE
Definition: kmp.h:637
@ KMP_HW_DIE
Definition: kmp.h:633
@ KMP_HW_TILE
Definition: kmp.h:636
@ KMP_HW_THREAD
Definition: kmp.h:641
@ KMP_HW_LAST
Definition: kmp.h:642
@ KMP_HW_LLC
Definition: kmp.h:634
static kmp_info_t * __kmp_thread_from_gtid(int gtid)
Definition: kmp.h:3643
void __kmp_expand_file_name(char *result, size_t rlen, char *pattern)
void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64< C, S > *flag)
int __kmp_root_counter
Definition: kmp_global.cpp:63
void * ___kmp_thread_calloc(kmp_info_t *th, size_t nelem, size_t elsize KMP_SRC_LOC_DECL)
Definition: kmp_alloc.cpp:1194
static int __kmp_gtid_from_thread(const kmp_info_t *thr)
Definition: kmp.h:3638
int __kmp_gtid_mode
Definition: kmp_global.cpp:182
omp_allocator_handle_t const llvm_omp_target_shared_mem_alloc
Definition: kmp_global.cpp:323
kmp_hw_core_type_t
Definition: kmp.h:645
@ KMP_HW_MAX_NUM_CORE_TYPES
Definition: kmp.h:652
@ KMP_HW_CORE_TYPE_UNKNOWN
Definition: kmp.h:646
void __kmpc_dispatch_init_4u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk)
See __kmpc_dispatch_init_4.
void __kmp_suspend_initialize_thread(kmp_info_t *th)
library_type
Definition: kmp.h:517
@ library_none
Definition: kmp.h:518
@ library_turnaround
Definition: kmp.h:520
@ library_throughput
Definition: kmp.h:521
@ library_serial
Definition: kmp.h:519
volatile int __kmp_init_serial
Definition: kmp_global.cpp:44
_reduction_method
Definition: kmp.h:554
@ empty_reduce_block
Definition: kmp.h:559
@ critical_reduce_block
Definition: kmp.h:556
@ tree_reduce_block
Definition: kmp.h:558
@ reduction_method_not_defined
Definition: kmp.h:555
@ atomic_reduce_block
Definition: kmp.h:557
void __kmpc_dispatch_init_8u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk)
See __kmpc_dispatch_init_4.
void __kmp_exit_thread(int exit_status)
KMP_EXPORT kmp_base_depnode_t * __kmpc_task_get_depnode(kmp_task_t *task)
int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int npr, int argc, void *argv[])
kmp_int32 __kmp_hidden_helper_threads_num
void __kmpc_dispatch_fini_4u(ident_t *loc, kmp_int32 gtid)
See __kmpc_dispatch_fini_4.
static void __kmp_type_convert(T1 src, T2 *dest)
Definition: kmp.h:4886
void __kmp_join_call(ident_t *loc, int gtid, int exit_teams=0)
enum kmp_bar_pat kmp_bar_pat_e
void __kmp_fini_memkind()
Definition: kmp_alloc.cpp:1334
KMP_EXPORT kmp_int32 __kmp_get_reduce_method(void)
omp_memspace_handle_t const llvm_omp_target_device_mem_space
Definition: kmp_global.cpp:345
int __kmp_ignore_mppend(void)
void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64<> *flag, int final_spin)
int __kmp_debug_buf_atomic
Definition: kmp_global.cpp:389
struct KMP_ALIGN_CACHE kmp_bstate kmp_bstate_t
KMP_EXPORT void __kmpc_unset_lock(ident_t *loc, kmp_int32 gtid, void **user_lock)
kmp_bar_pat_e __kmp_barrier_release_pattern[bs_last_barrier]
Definition: kmp_global.cpp:93
std::atomic< kmp_int32 > __kmp_team_counter
Definition: kmp_global.cpp:66
void __kmp_reap_worker(kmp_info_t *th)
int __kmp_env_stksize
Definition: kmp_global.cpp:420
kmp_bar_pat
Definition: kmp.h:2155
@ bp_dist_bar
Definition: kmp.h:2163
@ bp_linear_bar
Definition: kmp.h:2156
@ bp_tree_bar
Definition: kmp.h:2158
@ bp_last_bar
Definition: kmp.h:2164
@ bp_hierarchical_bar
Definition: kmp.h:2162
@ bp_hyper_bar
Definition: kmp.h:2160
dynamic_mode
Definition: kmp.h:333
@ dynamic_max
Definition: kmp.h:340
@ dynamic_thread_limit
Definition: kmp.h:339
@ dynamic_default
Definition: kmp.h:334
@ dynamic_random
Definition: kmp.h:338
void __kmp_cleanup(void)
void __kmpc_dispatch_init_8(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk)
See __kmpc_dispatch_init_4.
void __kmp_hidden_helper_threads_deinitz_release()
void __kmp_expand_host_name(char *buffer, size_t size)
int __kmpc_pause_resource(kmp_pause_status_t level)
union KMP_ALIGN_CACHE kmp_info kmp_info_t
enum sched_type __kmp_sch_map[]
Definition: kmp_global.cpp:235
int __kmp_tls_gtid_min
Definition: kmp_global.cpp:188
void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team, int wait=1)
kmp_uint64 __kmp_hardware_timestamp(void)
void ___kmp_free(void *ptr KMP_SRC_LOC_DECL)
Definition: kmp_alloc.cpp:2023
union KMP_ALIGN_CACHE kmp_time_global kmp_time_global_t
omp_allocator_handle_t const llvm_omp_target_device_mem_alloc
Definition: kmp_global.cpp:325
union KMP_ALIGN_CACHE kmp_global kmp_global_t
omp_allocator_handle_t const omp_null_allocator
Definition: kmp_global.cpp:304
kmp_uint32 __kmp_yield_next
Definition: kmp_global.cpp:440
void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid)
static int __kmp_ncores
static int alignment
Definition: kmp_alloc.cpp:1500
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 kmp_int16
KMP_ARCH_X86 KMP_ARCH_X86 KMP_ARCH_X86 kmp_int8
Definition: kmp_atomic.cpp:985
#define KMP_ASSERT(cond)
Definition: kmp_debug.h:59
#define KMP_BUILD_ASSERT(expr)
Definition: kmp_debug.h:26
#define KMP_DEBUG_ASSERT(cond)
Definition: kmp_debug.h:61
unsigned long long kmp_uint64
kmp_event_type_t
static volatile kmp_i18n_cat_status_t status
Definition: kmp_i18n.cpp:48
kmp_msg_t __kmp_msg_null
Definition: kmp_i18n.cpp:36
void __kmp_fatal(kmp_msg_t message,...)
Definition: kmp_i18n.cpp:864
#define KMP_INFORM(...)
Definition: kmp_i18n.h:142
#define KMP_MSG(...)
Definition: kmp_i18n.h:121
#define KMP_FATAL(...)
Definition: kmp_i18n.h:146
#define KMP_HNT(...)
Definition: kmp_i18n.h:122
#define KMP_ERR
Definition: kmp_i18n.h:125
kmp_ticket_lock_t kmp_lock_t
Definition: kmp_lock.h:555
int kmp_int32
Definition: kmp_num_teams.c:15
#define KMP_EXPORT
Definition: kmp_os.h:294
void(* microtask_t)(int *gtid, int *npr,...)
Definition: kmp_os.h:1183
long kmp_intptr_t
Definition: kmp_os.h:204
#define INTERNODE_CACHE_LINE
Definition: kmp_os.h:335
#define KMP_ALIGN_CACHE
Definition: kmp_os.h:453
#define KMP_ALIGN(bytes)
Definition: kmp_os.h:392
#define KMP_NORETURN
Definition: kmp_os.h:384
#define KMP_THREAD_LOCAL
Definition: kmp_os.h:393
#define CACHE_LINE
Definition: kmp_os.h:339
#define KMP_ATTRIBUTE_TARGET_WAITPKG
Definition: kmp_os.h:369
kmp_int32 kmp_int
Definition: kmp_os.h:214
va_list kmp_va_list
Definition: kmp_os.h:227
#define KMP_EXPAND_NAME(api_name)
Definition: kmp_os.h:444
#define args
#define i
Definition: kmp_stub.cpp:87
struct ID id
int counter
Definition: kmp_taskloop.c:13
#define KMP_SRC_LOC_DECL
__attribute__((noinline))
void microtask(int *global_tid, int *bound_tid)
int32_t kmp_int32
void * kmpc_micro
omp_lock_t lck
Definition: omp_lock.c:7
static id loc
static int checker
int num_roots
volatile int flag
struct private_common * data[KMP_HASH_TABLE_SIZE]
Definition: kmp.h:1865
ident_t const * ident
Definition: kmp.h:1689
kmp_user_lock_p name
Definition: kmp.h:1693
enum cons_type type
Definition: kmp.h:1690
int prev
Definition: kmp.h:1691
int stack_size
Definition: kmp.h:1698
int stack_top
Definition: kmp.h:1698
int p_top
Definition: kmp.h:1697
int w_top
Definition: kmp.h:1697
int s_top
Definition: kmp.h:1697
struct cons_data * stack_data
Definition: kmp.h:1699
kmp_int32 parm1
Definition: kmp.h:1982
kmp_int32 parm3
Definition: kmp.h:1984
kmp_int32 parm4
Definition: kmp.h:1985
kmp_uint32 ordered_lower
Definition: kmp.h:1989
kmp_int32 count
Definition: kmp.h:1987
kmp_int32 parm2
Definition: kmp.h:1983
kmp_uint32 ordered_upper
Definition: kmp.h:1990
kmp_int64 count
Definition: kmp.h:2008
kmp_uint64 ordered_upper
Definition: kmp.h:2011
kmp_uint64 ordered_lower
Definition: kmp.h:2010
kmp_int64 parm2
Definition: kmp.h:2004
kmp_int64 parm4
Definition: kmp.h:2006
kmp_int64 parm1
Definition: kmp.h:2003
kmp_int64 parm3
Definition: kmp.h:2005
struct dispatch_private_info * next
Definition: kmp.h:2028
kmp_sched_flags_t flags
Definition: kmp.h:2024
enum sched_type schedule
Definition: kmp.h:2023
kmp_int32 ordered_bumped
Definition: kmp.h:2026
enum cons_type pushed_ws
Definition: kmp.h:2034
kmp_int32 type_size
Definition: kmp.h:2029
std::atomic< kmp_uint32 > steal_flag
Definition: kmp.h:2025
kmp_int32 ordered_dummy[KMP_MAX_ORDERED - 1]
Definition: kmp.h:2044
volatile kmp_uint32 ordered_iteration
Definition: kmp.h:2042
volatile kmp_uint32 iteration
Definition: kmp.h:2040
volatile kmp_int32 num_done
Definition: kmp.h:2041
volatile kmp_int64 num_done
Definition: kmp.h:2051
volatile kmp_uint64 iteration
Definition: kmp.h:2050
kmp_int64 ordered_dummy[KMP_MAX_ORDERED - 3]
Definition: kmp.h:2054
volatile kmp_uint64 ordered_iteration
Definition: kmp.h:2052
volatile kmp_uint32 buffer_index
Definition: kmp.h:2062
kmp_int32 doacross_num_done
Definition: kmp.h:2065
union dispatch_shared_info::shared_info u
volatile kmp_int32 doacross_buf_idx
Definition: kmp.h:2063
volatile kmp_uint32 * doacross_flags
Definition: kmp.h:2064
kmp_int32 data
Definition: kmp.h:3290
The ident structure that describes a source location.
Definition: kmp.h:247
kmp_int32 get_openmp_version()
Definition: kmp.h:262
char const * psource
String describing the source location.
Definition: kmp.h:257
kmp_int32 reserved_1
might be used in Fortran; see above
Definition: kmp.h:248
kmp_int32 reserved_2
not really used in Fortran any more; see above
Definition: kmp.h:251
kmp_int32 reserved_3
source[4] in Fortran, do not use for C++
Definition: kmp.h:256
kmp_int32 flags
also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC identifies this union member
Definition: kmp.h:249
size_t alignment
Definition: kmp.h:1123
kmp_uint64 pool_size
Definition: kmp.h:1126
kmp_allocator_t * fb_data
Definition: kmp.h:1125
kmp_uint64 pool_used
Definition: kmp.h:1127
omp_alloctrait_value_t fb
Definition: kmp.h:1124
void ** memkind
Definition: kmp.h:1122
omp_memspace_handle_t memspace
Definition: kmp.h:1121
bool pinned
Definition: kmp.h:1128
volatile kmp_uint32 t_value
Definition: kmp.h:3094
std::atomic< kmp_int32 > npredecessors
Definition: kmp.h:2557
std::atomic< kmp_int32 > nrefs
Definition: kmp.h:2558
kmp_lock_t * mtx_locks[MAX_MTX_DEPS]
Definition: kmp.h:2551
kmp_lock_t lock
Definition: kmp.h:2553
kmp_task_t * task
Definition: kmp.h:2550
kmp_int32 mtx_num_locks
Definition: kmp.h:2552
kmp_depnode_list_t * successors
Definition: kmp.h:2549
volatile int g_abort
Definition: kmp.h:3250
enum dynamic_mode g_dynamic_mode
Definition: kmp.h:3254
kmp_time_global_t g_time
Definition: kmp.h:3247
volatile int g_done
Definition: kmp.h:3251
int g_dynamic
Definition: kmp.h:3253
kmp_disp_t * th_dispatch
Definition: kmp.h:2945
KMP_ALIGN_CACHE int th_set_nproc
Definition: kmp.h:2976
int th_team_serialized
Definition: kmp.h:2952
kmp_cg_root_t * th_cg_roots
Definition: kmp.h:3082
kmp_taskdata_t * th_current_task
Definition: kmp.h:3031
KMP_ALIGN_CACHE kmp_team_p * th_serial_team
Definition: kmp.h:3010
const char * th_nt_msg
Definition: kmp.h:2981
kmp_task_team_t * th_task_team
Definition: kmp.h:3030
kmp_local_t th_local
Definition: kmp.h:3003
ident_t * th_nt_loc
Definition: kmp.h:2979
kmp_info_p * th_next_pool
Definition: kmp.h:2944
int th_nt_sev
Definition: kmp.h:2980
kmp_uint64 th_team_bt_intervals
Definition: kmp.h:2965
kmp_uint32 th_reap_state
Definition: kmp.h:3033
int * th_set_nested_nth
Definition: kmp.h:2977
microtask_t th_teams_microtask
Definition: kmp.h:2953
KMP_ALIGN_CACHE volatile kmp_int32 th_next_waiting
Definition: kmp.h:3053
struct cons_header * th_cons
Definition: kmp.h:3043
int th_prev_level
Definition: kmp.h:2996
struct private_common * th_pri_head
Definition: kmp.h:3004
omp_allocator_handle_t th_def_allocator
Definition: kmp.h:2973
int th_teams_level
Definition: kmp.h:2954
kmp_uint8 th_active_in_pool
Definition: kmp.h:3038
int th_set_nested_nth_sz
Definition: kmp.h:2982
kmp_desc_t th_info
Definition: kmp.h:2941
kmp_team_p * th_team
Definition: kmp.h:2942
std::atomic< kmp_uint32 > th_used_in_team
Definition: kmp.h:3040
struct common_table * th_pri_common
Definition: kmp.h:3017
unsigned th_a
Definition: kmp.h:3027
kmp_teams_size_t th_teams_size
Definition: kmp.h:2989
kmp_root_p * th_root
Definition: kmp.h:2943
volatile void * th_sleep_loc
Definition: kmp.h:3022
volatile kmp_uint32 th_spin_here
Definition: kmp.h:3019
int th_in_pool
Definition: kmp.h:2946
int th_prev_num_threads
Definition: kmp.h:2997
int th_team_nproc
Definition: kmp.h:2950
bool th_nt_strict
Definition: kmp.h:2978
unsigned th_x
Definition: kmp.h:3026
flag_type th_sleep_loc_type
Definition: kmp.h:3023
kmp_proc_bind_t th_set_proc_bind
Definition: kmp.h:2987
ident_t * th_ident
Definition: kmp.h:3025
kmp_uint8 th_task_state
Definition: kmp.h:3032
int th_active
Definition: kmp.h:3039
kmp_info_p * th_team_master
Definition: kmp.h:2951
kmp_team_t * r_hot_team
Definition: kmp.h:3273
volatile int r_active
Definition: kmp.h:3268
volatile int r_begin
Definition: kmp.h:3276
kmp_info_t * r_uber_thread
Definition: kmp.h:3274
kmp_team_t * r_root_team
Definition: kmp.h:3272
int r_blocktime
Definition: kmp.h:3277
kmp_lock_t r_begin_lock
Definition: kmp.h:3275
std::atomic< int > r_in_parallel
Definition: kmp.h:3270
kmp_int32 tt_found_proxy_tasks
Definition: kmp.h:2864
KMP_ALIGN_CACHE std::atomic< kmp_int32 > tt_unfinished_threads
Definition: kmp.h:2872
kmp_int32 tt_max_threads
Definition: kmp.h:2863
kmp_int32 tt_nproc
Definition: kmp.h:2862
kmp_bootstrap_lock_t tt_task_pri_lock
Definition: kmp.h:2852
std::atomic< kmp_int32 > tt_num_task_pri
Definition: kmp.h:2866
kmp_bootstrap_lock_t tt_threads_lock
Definition: kmp.h:2848
kmp_int32 tt_untied_task_encountered
Definition: kmp.h:2865
kmp_task_pri_t * tt_task_pri_list
Definition: kmp.h:2853
kmp_int32 tt_hidden_helper_task_encountered
Definition: kmp.h:2869
kmp_thread_data_t * tt_threads_data
Definition: kmp.h:2857
KMP_ALIGN_CACHE volatile kmp_uint32 tt_active
Definition: kmp.h:2876
kmp_task_team_t * tt_next
Definition: kmp.h:2855
kmp_int32 tt_found_tasks
Definition: kmp.h:2859
omp_allocator_handle_t t_def_allocator
Definition: kmp.h:3200
int t_size_changed
Definition: kmp.h:3198
int t_id
Definition: kmp.h:3190
kmp_nested_nthreads_t * t_nested_nth
Definition: kmp.h:3224
int t_master_active
Definition: kmp.h:3215
launch_t t_invoke
Definition: kmp.h:3165
kmp_proc_bind_t t_proc_bind
Definition: kmp.h:3153
int t_level
Definition: kmp.h:3184
KMP_ALIGN_CACHE void ** t_argv
Definition: kmp.h:3161
int t_master_this_cons
Definition: kmp.h:3145
kmp_taskdata_t * t_implicit_task_taskdata
Definition: kmp.h:3183
std::atomic< kmp_int32 > t_cancel_request
Definition: kmp.h:3214
KMP_ALIGN_CACHE kmp_info_t ** t_threads
Definition: kmp.h:3181
dispatch_shared_info_t * t_disp_buffer
Definition: kmp.h:3189
KMP_ALIGN_CACHE kmp_internal_control_t * t_control_stack_top
Definition: kmp.h:3211
ident_t * t_ident
Definition: kmp.h:3147
kmp_team_p * t_next_pool
Definition: kmp.h:3150
kmp_team_p * t_parent
Definition: kmp.h:3149
KMP_ALIGN_CACHE int t_max_argc
Definition: kmp.h:3186
std::atomic< int > t_construct
Definition: kmp.h:3135
void * t_copypriv_data
Definition: kmp.h:3216
int t_nproc
Definition: kmp.h:3163
int t_primary_task_state
Definition: kmp.h:3154
int t_max_nproc
Definition: kmp.h:3187
int t_argc
Definition: kmp.h:3162
int t_display_affinity
Definition: kmp.h:3197
microtask_t t_pkfn
Definition: kmp.h:3164
kmp_disp_t * t_dispatch
Definition: kmp.h:3151
KMP_ALIGN_CACHE kmp_ordered_team_t t_ordered
Definition: kmp.h:3133
distributedBarrier * b
Definition: kmp.h:3223
kmp_r_sched_t t_sched
Definition: kmp.h:3192
int t_serialized
Definition: kmp.h:3188
KMP_ALIGN_CACHE int t_master_tid
Definition: kmp.h:3144
int t_active_level
Definition: kmp.h:3191
kmp_uint32 td_deque_tail
Definition: kmp.h:2817
kmp_uint32 td_deque_head
Definition: kmp.h:2816
kmp_int32 td_deque_size
Definition: kmp.h:2815
kmp_int32 td_deque_ntasks
Definition: kmp.h:2818
kmp_taskdata_t ** td_deque
Definition: kmp.h:2814
kmp_int32 td_deque_last_stolen
Definition: kmp.h:2820
kmp_info_p * td_thr
Definition: kmp.h:2809
kmp_bootstrap_lock_t td_deque_lock
Definition: kmp.h:2812
kmp_uint8 wait_flag
Definition: kmp.h:2222
kmp_uint32 * skip_per_level
Definition: kmp.h:2210
kmp_int32 old_tid
Definition: kmp.h:2213
kmp_uint8 leaf_kids
Definition: kmp.h:2220
kmp_uint32 nproc
Definition: kmp.h:2218
volatile kmp_uint64 b_go
Definition: kmp.h:2207
kmp_uint32 my_level
Definition: kmp.h:2211
kmp_uint8 offset
Definition: kmp.h:2221
KMP_ALIGN_CACHE volatile kmp_uint64 b_arrived
Definition: kmp.h:2209
kmp_uint8 use_oncore_barrier
Definition: kmp.h:2223
kmp_team_t * team
Definition: kmp.h:2216
struct kmp_bstate * parent_bar
Definition: kmp.h:2215
kmp_uint64 leaf_state
Definition: kmp.h:2217
kmp_internal_control_t th_fixed_icvs
Definition: kmp.h:2204
kmp_uint32 depth
Definition: kmp.h:2214
kmp_uint8 base_leaf_kids
Definition: kmp.h:2219
kmp_int32 parent_tid
Definition: kmp.h:2212
void *** compiler_cache
Definition: kmp.h:1815
struct kmp_cached_addr * next
Definition: kmp.h:1817
void * data
Definition: kmp.h:1816
void ** addr
Definition: kmp.h:1814
kmp_info_p * cg_root
Definition: kmp.h:2926
kmp_int32 cg_nthreads
Definition: kmp.h:2930
kmp_int32 cg_thread_limit
Definition: kmp.h:2929
struct kmp_cg_root * up
Definition: kmp.h:2931
unsigned mtx
Definition: kmp.h:2530
struct kmp_depend_info::@8::@10 flags
size_t len
Definition: kmp.h:2515
kmp_intptr_t base_addr
Definition: kmp.h:2514
kmp_uint8 flag
Definition: kmp.h:2517
unsigned set
Definition: kmp.h:2531
unsigned out
Definition: kmp.h:2529
unsigned in
Definition: kmp.h:2528
unsigned unused
Definition: kmp.h:2532
unsigned all
Definition: kmp.h:2533
Definition: kmp.h:2567
kmp_depnode_t * last_out
Definition: kmp.h:2569
kmp_uint8 last_flag
Definition: kmp.h:2572
kmp_lock_t * mtx_lock
Definition: kmp.h:2573
kmp_dephash_entry_t * next_in_bucket
Definition: kmp.h:2574
kmp_intptr_t addr
Definition: kmp.h:2568
kmp_depnode_list_t * last_set
Definition: kmp.h:2570
kmp_depnode_list_t * prev_set
Definition: kmp.h:2571
kmp_depnode_t * last_all
Definition: kmp.h:2580
kmp_dephash_entry_t ** buckets
Definition: kmp.h:2578
kmp_uint32 nconflicts
Definition: kmp.h:2583
size_t generation
Definition: kmp.h:2581
size_t size
Definition: kmp.h:2579
kmp_uint32 nelements
Definition: kmp.h:2582
kmp_depnode_list_t * next
Definition: kmp.h:2542
kmp_depnode_t * node
Definition: kmp.h:2541
size_t ds_stacksize
Definition: kmp.h:2310
void * ds_stackbase
Definition: kmp.h:2309
int ds_stackgrow
Definition: kmp.h:2311
kmp_thread_t ds_thread
Definition: kmp.h:2312
volatile int ds_tid
Definition: kmp.h:2313
int ds_gtid
Definition: kmp.h:2314
Definition: kmp.h:4467
kmp_int64 up
Definition: kmp.h:4469
kmp_int64 lo
Definition: kmp.h:4468
kmp_int64 st
Definition: kmp.h:4470
Definition: kmp.h:2077
void(* th_dxo_fcn)(int *gtid, int *cid, ident_t *)
Definition: kmp.h:2081
kmp_int32 th_doacross_buf_idx
Definition: kmp.h:2088
volatile kmp_uint32 * th_doacross_flags
Definition: kmp.h:2089
dispatch_private_info_t * th_dispatch_pr_current
Definition: kmp.h:2084
kmp_uint32 th_disp_index
Definition: kmp.h:2087
kmp_int64 * th_doacross_info
Definition: kmp.h:2090
dispatch_private_info_t * th_disp_buffer
Definition: kmp.h:2086
void(* th_deo_fcn)(int *gtid, int *cid, ident_t *)
Definition: kmp.h:2079
dispatch_shared_info_t * th_dispatch_sh_current
Definition: kmp.h:2083
kmp_tas_lock_t lock
Definition: kmp.h:2603
kmp_task_t * task
Definition: kmp.h:2605
kmp_event_type_t type
Definition: kmp.h:2602
int offset
Definition: kmp.h:1017
int num
Definition: kmp.h:1016
int task_thread_limit
Definition: kmp.h:2184
kmp_int8 dynamic
Definition: kmp.h:2173
kmp_proc_bind_t proc_bind
Definition: kmp.h:2188
kmp_r_sched_t sched
Definition: kmp.h:2187
struct kmp_internal_control * next
Definition: kmp.h:2190
int serial_nesting_level
Definition: kmp.h:2171
kmp_int32 default_device
Definition: kmp.h:2189
kmp_int8 bt_set
Definition: kmp.h:2176
int max_active_levels
Definition: kmp.h:2185
void * reduce_data
Definition: kmp.h:2341
PACKED_REDUCTION_METHOD_T packed_reduction_method
Definition: kmp.h:2357
volatile int this_construct
Definition: kmp.h:2340
kmp_proc_bind_t * bind_types
Definition: kmp.h:978
struct kmp_old_threads_list_t * next
Definition: kmp.h:3299
kmp_info_t ** threads
Definition: kmp.h:3298
int length[KMP_MAX_FIELDS]
Definition: kmp.h:1705
char * text
Definition: kmp.h:1703
int offset[KMP_MAX_FIELDS]
Definition: kmp.h:1704
unsigned unused
Definition: kmp.h:1890
unsigned contains_last
Definition: kmp.h:1887
unsigned ordered
Definition: kmp.h:1885
unsigned use_hier
Definition: kmp.h:1888
unsigned use_hybrid
Definition: kmp.h:1889
unsigned nomerge
Definition: kmp.h:1886
long minflt
Definition: kmp.h:1725
long inblock
Definition: kmp.h:1728
long majflt
Definition: kmp.h:1726
long nvcsw
Definition: kmp.h:1730
long oublock
Definition: kmp.h:1729
long nswap
Definition: kmp.h:1727
long nivcsw
Definition: kmp.h:1731
long maxrss
Definition: kmp.h:1724
void * async_handle
Definition: kmp.h:2752
struct kmp_task_affinity_info::@11 flags
kmp_int32 reserved
Definition: kmp.h:2592
kmp_intptr_t base_addr
Definition: kmp.h:2587
kmp_int32 priority
Definition: kmp.h:2841
kmp_task_pri * next
Definition: kmp.h:2842
kmp_thread_data_t td
Definition: kmp.h:2840
kmp_task_team_list_t * next
Definition: kmp.h:2887
kmp_task_team_t * task_team
Definition: kmp.h:2886
Definition: kmp.h:2472
void * shareds
pointer to block of pointers to shared vars
Definition: kmp.h:2473
kmp_int32 part_id
part id for the task
Definition: kmp.h:2476
kmp_cmplrdata_t data2
Definition: kmp.h:2479
kmp_cmplrdata_t data1
Definition: kmp.h:2478
kmp_routine_entry_t routine
pointer to routine to call for executing task
Definition: kmp.h:2475
kmp_uint32 td_taskwait_counter
Definition: kmp.h:2767
ident_t * td_taskwait_ident
Definition: kmp.h:2766
kmp_int32 td_level
Definition: kmp.h:2762
kmp_team_t * td_team
Definition: kmp.h:2758
kmp_task_team_t * td_task_team
Definition: kmp.h:2782
kmp_dephash_t * td_dephash
Definition: kmp.h:2779
kmp_taskdata_t * td_parent
Definition: kmp.h:2761
std::atomic< kmp_int32 > td_incomplete_child_tasks
Definition: kmp.h:2775
std::atomic< kmp_int32 > td_untied_count
Definition: kmp.h:2763
kmp_taskgroup_t * td_taskgroup
Definition: kmp.h:2777
kmp_int32 td_task_id
Definition: kmp.h:2756
kmp_info_p * td_alloc_thread
Definition: kmp.h:2759
ident_t * td_ident
Definition: kmp.h:2764
kmp_depnode_t * td_depnode
Definition: kmp.h:2781
kmp_int32 td_taskwait_thread
Definition: kmp.h:2768
kmp_tasking_flags_t td_flags
Definition: kmp.h:2757
kmp_taskdata_t * td_last_tied
Definition: kmp.h:2788
KMP_ALIGN_CACHE kmp_internal_control_t td_icvs
Definition: kmp.h:2770
kmp_event_t td_allow_completion_event
Definition: kmp.h:2793
size_t td_size_alloc
Definition: kmp.h:2783
kmp_target_data_t td_target_data
Definition: kmp.h:2801
KMP_ALIGN_CACHE std::atomic< kmp_int32 > td_allocated_child_tasks
Definition: kmp.h:2772
std::atomic< kmp_int32 > cancel_request
Definition: kmp.h:2491
uintptr_t * gomp_data
Definition: kmp.h:2496
std::atomic< kmp_int32 > count
Definition: kmp.h:2489
void * reduce_data
Definition: kmp.h:2494
struct kmp_taskgroup * parent
Definition: kmp.h:2492
kmp_int32 reduce_num_data
Definition: kmp.h:2495
unsigned target
Definition: kmp.h:2741
unsigned priority_specified
Definition: kmp.h:2721
unsigned detachable
Definition: kmp.h:2723
unsigned task_serial
Definition: kmp.h:2729
unsigned merged_if0
Definition: kmp.h:2715
unsigned complete
Definition: kmp.h:2738
unsigned freed
Definition: kmp.h:2739
unsigned executing
Definition: kmp.h:2737
unsigned reserved
Definition: kmp.h:2725
unsigned tasking_ser
Definition: kmp.h:2730
unsigned team_serial
Definition: kmp.h:2732
unsigned native
Definition: kmp.h:2740
unsigned reserved31
Definition: kmp.h:2746
unsigned tiedness
Definition: kmp.h:2713
unsigned started
Definition: kmp.h:2736
unsigned destructors_thunk
Definition: kmp.h:2717
unsigned proxy
Definition: kmp.h:2719
unsigned tasktype
Definition: kmp.h:2728
unsigned final
Definition: kmp.h:2714
unsigned hidden_helper
Definition: kmp.h:2724
kmp_int32 nteams
Definition: kmp.h:2910
kmp_int32 nth
Definition: kmp.h:2911
omp_alloctrait_key_t key
Definition: kmp.h:1093
omp_uintptr_t value
Definition: kmp.h:1094
size_t cmn_size
Definition: kmp.h:1832
void * gbl_addr
Definition: kmp.h:1830
struct private_common * next
Definition: kmp.h:1828
struct private_common * link
Definition: kmp.h:1829
void * par_addr
Definition: kmp.h:1831
struct private_data * next
Definition: kmp.h:1821
void * data
Definition: kmp.h:1822
int more
Definition: kmp.h:1823
size_t size
Definition: kmp.h:1824
kmpc_dtor_vec dtorv
Definition: kmp.h:1850
union shared_common::@4 cct
kmpc_dtor dtor
Definition: kmp.h:1849
kmpc_cctor_vec cctorv
Definition: kmp.h:1846
union shared_common::@3 ct
kmpc_ctor ctor
Definition: kmp.h:1841
void * obj_init
Definition: kmp.h:1838
kmpc_cctor cctor
Definition: kmp.h:1845
union shared_common::@5 dt
struct private_data * pod_init
Definition: kmp.h:1837
struct shared_common * next
Definition: kmp.h:1836
kmpc_ctor_vec ctorv
Definition: kmp.h:1842
void * gbl_addr
Definition: kmp.h:1839
size_t cmn_size
Definition: kmp.h:1854
size_t vec_len
Definition: kmp.h:1852
int is_vec
Definition: kmp.h:1853
struct shared_common * data[KMP_HASH_TABLE_SIZE]
Definition: kmp.h:1869
dispatch_private_info64_t p64
Definition: kmp.h:2021
dispatch_private_info32_t p32
Definition: kmp.h:2020
dispatch_shared_info64_t s64
Definition: kmp.h:2060
dispatch_shared_info32_t s32
Definition: kmp.h:2059
kmp_uint64 b_arrived
Definition: kmp.h:2245
double b_align
Definition: kmp.h:2233
kmp_bstate_t bb
Definition: kmp.h:2235
kmp_routine_entry_t destructors
Definition: kmp.h:2464
kmp_int32 priority
priority specified by user for the task
Definition: kmp.h:2462
double dn_align
Definition: kmp.h:2562
kmp_base_depnode_t dn
Definition: kmp.h:2564
Definition: kmp.h:2333
double ds_align
Definition: kmp.h:2334
kmp_desc_base_t ds
Definition: kmp.h:2336
double g_align
Definition: kmp.h:3259
kmp_base_global_t g
Definition: kmp.h:3258
Definition: kmp.h:3085
kmp_base_info_t th
Definition: kmp.h:3088
double th_align
Definition: kmp.h:3086
double dt_align
Definition: kmp.h:3104
kmp_base_data_t dt
Definition: kmp.h:3106
int chunk
Definition: kmp.h:509
enum sched_type r_sched_type
Definition: kmp.h:508
kmp_int64 sched
Definition: kmp.h:511
Definition: kmp.h:3283
kmp_base_root_t r
Definition: kmp.h:3284
double r_align
Definition: kmp.h:3285
kmp_base_data_t dt
Definition: kmp.h:3100
double dt_align
Definition: kmp.h:3098
double tt_align
Definition: kmp.h:2881
kmp_base_task_team_t tt
Definition: kmp.h:2880
Definition: kmp.h:3233
double t_align
Definition: kmp.h:3235
kmp_base_team_t t
Definition: kmp.h:3234
double td_align
Definition: kmp.h:2835
kmp_base_thread_data_t td
Definition: kmp.h:2834
double dt_align
Definition: kmp.h:3240
kmp_base_data_t dt
Definition: kmp.h:3242
kmp_uint64 __kmp_ticks_per_usec
void __kmp_reap_monitor(kmp_info_t *th)
kmp_uint64 __kmp_ticks_per_msec
int __kmp_read_from_file(char const *path, char const *format,...)
void __kmp_register_atfork(void)
void __kmp_free_handle(kmp_thread_t tHandle)
void __kmp_affinity_determine_capable(const char *env_var)
void __kmp_affinity_bind_thread(int proc)
int __kmp_get_load_balance(int max)
int __kmp_still_running(kmp_info_t *th)
void __kmp_initialize_system_tick(void)
int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val)